diff options
Diffstat (limited to 'python/api_quotes')
| -rw-r--r-- | python/api_quotes/__main__.py | 6 | ||||
| -rw-r--r-- | python/api_quotes/quotes.py | 6 |
2 files changed, 11 insertions, 1 deletions
diff --git a/python/api_quotes/__main__.py b/python/api_quotes/__main__.py index 98655762..7bf3be69 100644 --- a/python/api_quotes/__main__.py +++ b/python/api_quotes/__main__.py @@ -26,10 +26,13 @@ if __name__ == "__main__": after = ( int((args.start_from + datetime.timedelta(days=1)).strftime("%s")) * 1000 ) + already_uploaded = MarkitQuoteKind[asset_class].already_uploaded() while True: try: if data := MarkitAPI.get_data(asset_class, after): - for msg_id, quotes in data: + for key, quotes in data: + if key["id"] in already_uploaded: + continue for row in quotes: try: quote = MarkitQuoteKind[asset_class].from_markit_line( @@ -50,6 +53,7 @@ if __name__ == "__main__": if after == last_val: break else: + already_uploaded.add(key["id"]) after = last_val else: break diff --git a/python/api_quotes/quotes.py b/python/api_quotes/quotes.py index fd5d1c2e..8d0b0231 100644 --- a/python/api_quotes/quotes.py +++ b/python/api_quotes/quotes.py @@ -53,6 +53,12 @@ class MarkitQuote(Deal, table_name=None, deal_type=None): def clear(cls): cls._insert_queue.clear() + @classmethod + def already_uploaded(cls): + with cls._conn.cursor() as c: + c.execute(f"SELECT distinct msg_id as msg_id FROM {cls._table_name}") + return set(row.msg_id for row in c) + # TODO # @property # def message(self): |