diff options
Diffstat (limited to 'python/report_ops')
| -rw-r--r-- | python/report_ops/__main__.py | 4 | ||||
| -rw-r--r-- | python/report_ops/admin.py | 106 |
2 files changed, 48 insertions, 62 deletions
diff --git a/python/report_ops/__main__.py b/python/report_ops/__main__.py index fed75561..f8c68eee 100644 --- a/python/report_ops/__main__.py +++ b/python/report_ops/__main__.py @@ -104,8 +104,8 @@ if args.isosel_reports: for fund in ("ISOSEL",): for report in ("isosel_accrued", "citco_reports"): try: - report = CitcoReport[report](cob, fund) - report.to_db() + report = CitcoReport[report] + report.to_db(cob, fund) except MissingDataError as e: logger.warning(e) diff --git a/python/report_ops/admin.py b/python/report_ops/admin.py index 8dce970b..86d8bf02 100644 --- a/python/report_ops/admin.py +++ b/python/report_ops/admin.py @@ -1,17 +1,14 @@ -from functools import partial from typing import ClassVar import datetime from dataclasses import dataclass +import re import pandas as pd -import dateutil.parser as dparser from serenitas.utils.remote import SftpClient -from serenitas.ops.trade_dataclasses import Fund from serenitas.utils.db import dawn_engine, dbconn -from serenitas.analytics.dates import next_business_day -from serenitas.analytics.exceptions import MissingDataError +from serenitas.analytics.dates import next_business_day, prev_business_day -from .misc import dt_from_citco +_suffix = {"ISOSEL": "IM", "CRSE": "IMCG"} @dataclass @@ -22,81 +19,72 @@ class CitcoReport: _conn: dbconn = dbconn("dawndb") _sftp: ClassVar = SftpClient.from_creds("citco", folder="outgoing") date_cols: ClassVar = [] - dtkey: ClassVar + prefix: ClassVar _registry = {} - def __init_subclass__(cls, table, date_cols, dtkey): + def __init_subclass__(cls, table, date_cols, prefix): cls.table = table cls.date_cols = date_cols - cls.dtkey = dtkey + cls.prefix = prefix cls._registry[table] = cls def __class_getitem__(cls, table): return cls._registry[table] - def __post_init__(self): - if not self.report_file_name: - raise MissingDataError( - f"No reports for {self.fund}:{self.table} on {self.date}" - ) - self.knowledge_date = datetime.datetime.combine( - next_business_day(self.date), self.dtkey_fun(self.report_file_name).time() - ) - @staticmethod def get_report_date(cob): return cob @classmethod def get_report(cls, date, fund): - prefix = cls.get_prefix(fund) - dtkey_fun = cls.get_dt_fun(fund) - report_date = cls.get_report_date(date) + reports = [ + f + for f in cls._sftp.client.listdir() + if (cls.prefix.format(fund=fund, suffix=_suffix[fund]) in f) + and cls.get_ts(f).date() == date + ] return max( - [ - f - for f in cls._sftp.client.listdir() - if (prefix in f) and (dtkey_fun(f).date() == report_date) - ], - key=dtkey_fun, + reports, + key=cls.get_ts, + default=None, ) @classmethod def get_df(cls, report_name): with cls._sftp.client.open(report_name) as fh: + ts = cls.get_ts(report_name) df = pd.read_csv(fh, parse_dates=cls.date_cols, infer_datetime_format=True) df["row"] = df.index df.columns = df.columns.str.lower() df.columns = df.columns.str.replace(" ", "_") - df["period_end_date"] = cls.date - df["knowledge_date"] = cls.knowledge_date - df["fund"] = cls.fund + df["period_end_date"] = ts.date() + df["knowledge_date"] = next_business_day(ts) + if "strategy" in df.columns: + df["strategy"] = df["strategy"].str.replace( + "/M_|/SER_", "/", regex=True + ) return df @classmethod def to_db(cls, date, fund): - report_name = cls.get_report(date, fund) - df = cls.get_df(report_name) - with cls._conn.cursor() as c: - c.execute( - f"DELETE FROM {cls.table} WHERE period_end_date=%s AND fund=%s", - ( - date, - fund, - ), - ) - cls._conn.commit() - if "strategy" in df.columns: - df["strategy"] = df["strategy"].str.replace("/M_|/SER_", "/", regex=True) - df.to_sql(self.table, dawn_engine, if_exists="append", index=False) + if report_name := cls.get_report(date, fund): + conn = cls._conn + df = cls.get_df(report_name) + df["fund"] = fund - @classmethod - def get_dt_fun(cls, fund): - def dt_fun(fname, date): - ts = fname.removeprefix(cls.get_prefix(fund)).removesuffix(".csv") - return dparser(ts) + with conn.cursor() as c: + c.execute( + f"DELETE FROM {cls.table} WHERE period_end_date=%s AND fund=%s", + ( + date, + fund, + ), + ) + conn.commit() - return dt_fun + df.to_sql(cls.table, dawn_engine, if_exists="append", index=False) + else: + print(f"{fund}: {cls.table} report not ready for {date}") class AccruedReport( @@ -112,23 +100,21 @@ class AccruedReport( "Start Date", "End Date", ], - dtkey="%Y%m%d%H%M%S", + prefix="100502500_INNOCAP_{fund}_", ): @staticmethod - def get_report_date(cob): - return next_business_day(cob) - - @staticmethod - def get_prefix(fund): - return f"100502500_INNOCAP_{fund}_" + def get_ts(s): + m = re.search(r"\d{8}\d{4}", s) + return prev_business_day(datetime.datetime.strptime(m.group(), "%Y%m%d%H%M%S")) class AllReport( CitcoReport, table="citco_reports", date_cols=["Maturity Date"], - dtkey="%Y%m%d.%H%M%S", + prefix="SPOS4X_INNOCAP_{fund}_D_{suffix}", ): @staticmethod - def get_prefix(fund): - return f"SPOS4X_INNOCAP_{fund}_D_{'IM' if fund =='ISOSEL' else 'IMCG'}." + def get_ts(s): + m = re.search(r"\d{8}\.\d{6}", s) + return datetime.datetime.strptime(m.group(), "%Y%m%d.%H%M%S") |