diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/api_quotes/__main__.py | 25 |
1 files changed, 18 insertions, 7 deletions
diff --git a/python/api_quotes/__main__.py b/python/api_quotes/__main__.py index 14a1d6eb..35ed2172 100644 --- a/python/api_quotes/__main__.py +++ b/python/api_quotes/__main__.py @@ -5,17 +5,21 @@ from json.decoder import JSONDecodeError import datetime import concurrent.futures +from serenitas.analytics.dates import bus_day + from .api import MarkitAPI from .quotes import MarkitQuoteKind logger = logging.getLogger(__name__) -def process_asset_class(asset_class, after): +def process_asset_class(asset_class, start_from, end): already_uploaded = MarkitQuoteKind[asset_class].already_uploaded() while True: + if end and (start_from < end): + break try: - if data := MarkitAPI.get_data(asset_class, after): + if data := MarkitAPI.get_data(asset_class, start_from): for (quoteid, receiveddatetime), quotes in data: quotes = list(quotes) # Don't try to insert into DB if already uploaded @@ -37,14 +41,14 @@ def process_asset_class(asset_class, after): # The after is specific so that we can avoid skipping any quotes # We would also get stuck sometimes without the quoteid being specified last_val = f"{receiveddatetime},{asset_class}-9480-{quoteid}" - if after == last_val: + if start_from == last_val: break else: - after = last_val + start_from = last_val else: break except JSONDecodeError: - logger.error(f"Issue with {asset_class}: {after}") + logger.error(f"Issue with {asset_class}: {start_from}") except AttributeError: MarkitAPI.update_api_key() @@ -57,14 +61,21 @@ if __name__ == "__main__": default=datetime.date.today(), nargs="?", ) + parser.add_argument( + "-b", "--backfill", action="store_true", help="short an old date" + ) args = parser.parse_args() with concurrent.futures.ThreadPoolExecutor() as executor: + start = ( + int((args.start_from + datetime.timedelta(days=1)).strftime("%s")) * 1000 + ) + end = int((args.start_from - bus_day * 1).strftime("%s")) * 1000 futures = [ executor.submit( process_asset_class, asset_class, - int((args.start_from + datetime.timedelta(days=1)).strftime("%s")) - * 1000, + start, + end if not args.backfill else None, ) for asset_class in ["ABS", "CD", "TRS"] ] |