blob: 26b7e12507906f8cb515094f38a6cc42de5f8885 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
import logging
import argparse
from json.decoder import JSONDecodeError
import datetime
from .api import MarkitAPI
from .quotes import MarkitQuoteKind
logger = logging.getLogger(__name__)
def run(start_from):
for asset_class in (
"ABS",
"CD",
"TRS",
):
after = int((start_from + datetime.timedelta(days=1)).strftime("%s")) * 1000
already_uploaded = MarkitQuoteKind[asset_class].already_uploaded()
while True:
try:
if data := MarkitAPI.get_data(asset_class, after):
for key, quotes in data:
quotes = list(quotes)
# Don't try to insert into DB if already uploaded
if key["id"] in already_uploaded:
row = quotes[-1]
else:
for row in quotes:
try:
quote = MarkitQuoteKind[
asset_class
].from_markit_line(row)
except ValueError as e:
MarkitQuoteKind[asset_class].clear()
logger.error(f"Couldn't parse {msg_id}: {e}")
continue
else:
quote.stage()
quote.commit()
# The after is specific so that we can avoid skipping any quotes
# We would also get stuck sometimes without the quoteid being specified
last_val = (
f"{row['receiveddatetime']},{asset_class}-9480-{row['quoteid']}"
)
if after == last_val:
break
else:
already_uploaded.add(key["id"])
after = last_val
else:
break
except JSONDecodeError:
logger.error(f"Issue with {asset_class}: {after}")
except AttributeError:
MarkitAPI.update_api_key()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"start_from",
type=datetime.date.fromisoformat,
default=datetime.date.today(),
nargs="?",
)
args = parser.parse_args()
while True:
run(args.start_from)
|
