diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/api_quotes/__main__.py | 10 | ||||
| -rw-r--r-- | python/api_quotes/api.py | 8 |
2 files changed, 10 insertions, 8 deletions
diff --git a/python/api_quotes/__main__.py b/python/api_quotes/__main__.py index 6fb48842..14a1d6eb 100644 --- a/python/api_quotes/__main__.py +++ b/python/api_quotes/__main__.py @@ -16,11 +16,11 @@ def process_asset_class(asset_class, after): while True: try: if data := MarkitAPI.get_data(asset_class, after): - for key, quotes in data: + for (quoteid, receiveddatetime), quotes in data: quotes = list(quotes) # Don't try to insert into DB if already uploaded - if key["id"] in already_uploaded: - row = quotes[-1] + if quoteid in already_uploaded: + continue else: for row in quotes: try: @@ -36,9 +36,7 @@ def process_asset_class(asset_class, after): quote.commit() # The after is specific so that we can avoid skipping any quotes # We would also get stuck sometimes without the quoteid being specified - last_val = ( - f"{row['receiveddatetime']},{asset_class}-9480-{row['quoteid']}" - ) + last_val = f"{receiveddatetime},{asset_class}-9480-{quoteid}" if after == last_val: break else: diff --git a/python/api_quotes/api.py b/python/api_quotes/api.py index cf716f14..7d6e36bc 100644 --- a/python/api_quotes/api.py +++ b/python/api_quotes/api.py @@ -43,8 +43,12 @@ class MarkitAPI: path = posixpath.join("parsing", "Quote", service) url = urljoin(cls.base_url, path) r = requests.get(url, params) - for msg_id, data in groupby(json.loads(r.text), key=lambda x: x["message"]): - yield msg_id, map(lowercase_keys, data) + grouped_data = groupby( + json.loads(r.text), + key=lambda x: (x["message"]["id"], x["receivedDateTime"]), + ) + for key, data in grouped_data: + yield key, map(lowercase_keys, data) @classmethod def update_api_key(cls): |