diff options
Diffstat (limited to 'python/api_quotes')
| -rw-r--r-- | python/api_quotes/__main__.py | 22 | ||||
| -rw-r--r-- | python/api_quotes/api.py | 4 | ||||
| -rw-r--r-- | python/api_quotes/quotes.py | 4 |
3 files changed, 21 insertions, 9 deletions
diff --git a/python/api_quotes/__main__.py b/python/api_quotes/__main__.py index 4342c0bb..98655762 100644 --- a/python/api_quotes/__main__.py +++ b/python/api_quotes/__main__.py @@ -1,5 +1,6 @@ import logging import argparse + from json.decoder import JSONDecodeError import datetime @@ -28,14 +29,19 @@ if __name__ == "__main__": while True: try: if data := MarkitAPI.get_data(asset_class, after): - for row in data: - try: - quote = MarkitQuoteKind[asset_class].from_markit_line(row) - except ValueError as e: - logger.error(f"Couldn't parse {row['quoteid']}: {e}") - else: - quote.stage() - quote.commit() + for msg_id, quotes in data: + for row in quotes: + try: + quote = MarkitQuoteKind[asset_class].from_markit_line( + row + ) + except ValueError as e: + MarkitQuoteKind[asset_class].clear() + logger.error(f"Couldn't parse {msg_id}: {e}") + continue + else: + quote.stage() + quote.commit() # The after is specific so that we can avoid skipping any quotes # We would also get stuck sometimes without the quoteid being specified last_val = ( diff --git a/python/api_quotes/api.py b/python/api_quotes/api.py index b9e32226..cf716f14 100644 --- a/python/api_quotes/api.py +++ b/python/api_quotes/api.py @@ -3,6 +3,7 @@ from urllib.parse import urljoin import posixpath import requests +from itertools import groupby from serenitas.utils.misc import get_credential_path @@ -42,7 +43,8 @@ class MarkitAPI: path = posixpath.join("parsing", "Quote", service) url = urljoin(cls.base_url, path) r = requests.get(url, params) - return map(lowercase_keys, json.loads(r.text)) + for msg_id, data in groupby(json.loads(r.text), key=lambda x: x["message"]): + yield msg_id, map(lowercase_keys, data) @classmethod def update_api_key(cls): diff --git a/python/api_quotes/quotes.py b/python/api_quotes/quotes.py index daa3decb..fd5d1c2e 100644 --- a/python/api_quotes/quotes.py +++ b/python/api_quotes/quotes.py @@ -49,6 +49,10 @@ class MarkitQuote(Deal, table_name=None, deal_type=None): } return base_attributes + @classmethod + def clear(cls): + cls._insert_queue.clear() + # TODO # @property # def message(self): |