diff options
| -rw-r--r-- | python/quote_parsing/__init__.py | 4 | ||||
| -rw-r--r-- | python/quote_parsing/__main__.py | 6 | ||||
| -rw-r--r-- | python/quote_parsing/download_emails.py | 10 | ||||
| -rw-r--r-- | python/quote_parsing/parse_emails.py | 8 | ||||
| -rw-r--r-- | python/utils/__init__.py | 10 |
5 files changed, 30 insertions, 8 deletions
diff --git a/python/quote_parsing/__init__.py b/python/quote_parsing/__init__.py index 6724b2af..9b60344f 100644 --- a/python/quote_parsing/__init__.py +++ b/python/quote_parsing/__init__.py @@ -1,5 +1,7 @@ import logging import sys + sys.path.append("..") -from utils import SerenitasFileHandler +from utils import SerenitasRotatingFileHandler + logger = logging.getLogger(__name__) diff --git a/python/quote_parsing/__main__.py b/python/quote_parsing/__main__.py index 5fef3463..50daf2b2 100644 --- a/python/quote_parsing/__main__.py +++ b/python/quote_parsing/__main__.py @@ -7,12 +7,12 @@ import sys from env import DATA_DIR from googleapiclient import errors -from . import SerenitasFileHandler +from . import SerenitasRotatingFileHandler from . import logger from .download_emails import save_emails from .parse_emails import parse_email, write_todb -fh = SerenitasFileHandler("emails_parsing.log") +fh = SerenitasRotatingFileHandler("emails_parsing.log", 1_000_000) logger.addHandler(fh) logger.setLevel(logging.WARNING) @@ -22,7 +22,7 @@ except (errors.HttpError, FileNotFoundError) as e: logger.error(e) save_emails(update=False) -emails = [f for f in (DATA_DIR / "swaptions").iterdir() if f.is_file()] +emails = [f for f in (DATA_DIR / "swaptions").glob("????-??/*") if f.is_file()] swaption_stack = {} index_data = pd.DataFrame() diff --git a/python/quote_parsing/download_emails.py b/python/quote_parsing/download_emails.py index 24d87601..e61f6637 100644 --- a/python/quote_parsing/download_emails.py +++ b/python/quote_parsing/download_emails.py @@ -38,8 +38,8 @@ def save_emails(update=True): last_history_id = int((DATA_DIR / ".lastHistoryId").read_text()) existing_msgs = [] else: - p = DATA_DIR / "swaptions" - existing_msgs = set(str(x).split("_")[1] for x in p.iterdir() if x.is_file()) + p = (DATA_DIR / "swaptions").glob("????-??/*") + existing_msgs = set(str(x).split("_")[1] for x in p if x.is_file()) last_history_id = None gm = GmailMessage() @@ -63,8 +63,10 @@ def save_emails(update=True): logger.error("error decoding " + msg['id']) continue else: - email = (DATA_DIR / "swaptions" / - f"{date:%Y-%m-%d %H-%M-%S}_{msg['id']}") + save_dir = DATA_DIR / "swaptions" / f"{date:%Y-%m}" + if not save_dir.exists(): + save_dir.mkdir() + email = save_dir / f"{date:%Y-%m-%d %H-%M-%S}_{msg['id']}" with email.open("w") as fh: fh.write(subject + "\r\n") fh.write(content) diff --git a/python/quote_parsing/parse_emails.py b/python/quote_parsing/parse_emails.py index 21c1b9f1..fe5b7698 100644 --- a/python/quote_parsing/parse_emails.py +++ b/python/quote_parsing/parse_emails.py @@ -462,6 +462,13 @@ def parse_citi(fh, indextype, series, quotedate): return option_stack, fwd_index +def parse_cs(fh, indextype, series, quotedate): + option_stack = {} + fwd_index = [] + d = {"quotedate": quotedate, "index": indextype, "series": series} + return + + subject_baml = re.compile(r"(?:Fwd:){0,2}(?:BAML )?(\w{2})([0-9]{1,2})\s") subject_ms = re.compile( r"[^$]*\$\$ MS CDX OPTIONS: (IG|HY)(\d{2})[^-]*- REF[^\d]*([\d.]+)" @@ -470,6 +477,7 @@ subject_nom = re.compile(r"(?:Fwd:)?CDX (IG|HY)(\d{2}).*- REF:[^\d]*([\d.]+)") subject_gs = re.compile(r"(?:FW: |Fwd: )?GS (IG|HY)(\d{2}) 5y.*- Ref [^\d]*([\d.]+)") subject_sg = re.compile(r"SG OPTIONS - CDX (IG|HY) S(\d{2}).* REF[^\d]*([\d.]+)") subject_citi = re.compile(r"(?:Fwd:)?Citi Options: (IG|HY)(\d{2}) 5Y") +subject_cs = re.compile(r"CS CDX (IG|HY)(\d{2}) Options - Ref = ([\d.]+)[^\d]*") def parse_email(email, date_received): diff --git a/python/utils/__init__.py b/python/utils/__init__.py index 625e0f10..856e42f1 100644 --- a/python/utils/__init__.py +++ b/python/utils/__init__.py @@ -12,3 +12,13 @@ class SerenitasFileHandler(logging.FileHandler): def __init__(self, log_file): super().__init__(filename=os.path.join(os.getenv("LOG_DIR"), log_file)) self.setFormatter(SerenitasFileHandler._formatter) + + +class SerenitasRotatingFileHandler(logging.RotatingFileHandler): + """simple class that encapsulates where we store our logs""" + + def __init__(self, log_file, maxBytes=0): + super().__init__( + filename=os.path.join(os.getenv("LOG_DIR"), log_file), maxBytes=maxByes + ) + self.setFormatter(SerenitasFileHandler._formatter) |