diff options
Diffstat (limited to 'python/quote_parsing')
| -rw-r--r-- | python/quote_parsing/__main__.py | 18 | ||||
| -rw-r--r-- | python/quote_parsing/parse_emails.py | 22 |
2 files changed, 15 insertions, 25 deletions
diff --git a/python/quote_parsing/__main__.py b/python/quote_parsing/__main__.py index 74b18bf0..a0e35510 100644 --- a/python/quote_parsing/__main__.py +++ b/python/quote_parsing/__main__.py @@ -61,9 +61,9 @@ for f in emails: swaption_stack[key] = pd.concat(option_stack, names=["expiry", "strike"]) fwd_index["msg_id"] = int(msg_id, 16) index_data = index_data.append(fwd_index) - already_uploaded[msg_id] = key[0] -if index_data.empty: - sys.exit() + # already_uploaded[msg_id] = key[0] +# if index_data.empty: +# sys.exit() for col in ["fwdbpv", "fwdprice", "fwdspread", "ref"]: if col in index_data: index_data[col] = pd.to_numeric(index_data[col]) @@ -72,18 +72,18 @@ index_data["index"] = index_data["index"].astype("category") index_names = ["quotedate", "index", "series", "quote_source"] swaption_stack = pd.concat(swaption_stack, names=index_names, sort=False) dup = swaption_stack.index.duplicated() -if dup.any(): - logger.warning("duplicated data") - swaption_stack = swaption_stack[~dup] +# if dup.any(): +# logger.warning("duplicated data") +# swaption_stack = swaption_stack[~dup] swaption_stack = swaption_stack.reset_index().set_index( - ["quotedate", "index", "series", "expiry", "quote_source"] + ["quotedate", "index", "series", "expiry", "quote_source", "version"] ) swaption_stack = swaption_stack.sort_index() index_data = index_data.reset_index() index_data = index_data.drop_duplicates( - ["quotedate", "index", "series", "expiry", "quote_source"] + ["quotedate", "index", "series", "expiry", "quote_source", "version"] ) - +index_data = index_data.sort_values(by=["expiry", "version"]) write_todb(swaption_stack, index_data, conn) serenitas_pool.putconn(conn) diff --git a/python/quote_parsing/parse_emails.py b/python/quote_parsing/parse_emails.py index d6704750..ec36973a 100644 --- a/python/quote_parsing/parse_emails.py +++ b/python/quote_parsing/parse_emails.py @@ -671,11 +671,9 @@ def parse_jpm_useu(fh, index_desc): fwd_index.append({**index_desc, **d}) try: - option_stack[index_desc["version"]] = {} - ( - option_stack[index_desc["version"]][d["expiry"]], - line, - ) = parse_jpm_block(fh, index_desc["index"]) + option_stack[(d["expiry"], d["version"])], line = parse_jpm_block( + fh, index_desc["index"] + ) except IndexError as e: logger.debug(traceback.format_exc()) logger.error(f"Something is wrong with file {Path(fh.name).stem}") @@ -689,7 +687,7 @@ def parse_jpm_useu(fh, index_desc): line = next(fh).strip() except StopIteration: break - return option_stack, fwd_index, versions + return option_stack, fwd_index # subject_baml = re.compile(r"(?:Fwd:){0,2}(?:BAML )?(\D{2})(\d{1,2})\s") @@ -766,7 +764,6 @@ def parse_email(email: Path, date_received: datetime.date, conn): if version is None: version = get_version(indextype, series, quotedate) parse_fun = globals()[f"parse_{source.lower()}_{region.lower()}"] - # key = (quotedate, indextype, series, source) index_desc = { "quotedate": quotedate, "index": indextype, @@ -777,15 +774,8 @@ def parse_email(email: Path, date_received: datetime.date, conn): index_desc["ref"] = ref if source in ["BNP", "SG"]: index_desc["expiration_dates"] = list_imm_dates(quotedate) - # Returning different versions and creating a list of keys - if source == "JPM": - key = [] - option_stack, fwd_index, versions = parse_fun(fh, index_desc) - for version in versions: - key.append((quotedate, indextype, series, source, version)) - else: - option_stack, fwd_index = parse_fun(fh, index_desc) - key = (quotedate, indextype, series, source) + option_stack, fwd_index = parse_fun(fh, index_desc) + key = (quotedate, indextype, series, source) if fwd_index: fwd_index = pd.DataFrame.from_records(fwd_index, index="quotedate") fwd_index["quote_source"] = source |