import logging import pandas as pd from io import BytesIO from pikepdf import Pdf from . import DAILY_DIR from .common import load_pdf, get_col, parse_num logger = logging.getLogger(__name__) paths = { "Serenitas": ["NYops", "Margin Calls JPM"], "BowdSt": ["BowdoinOps", "Margin JPM"], } accounts = { "BowdSt": "909271", "Serenitas": "923550", } passwords = {"BowdSt": "tm64EO", "Serenitas": "JV3RJu"} def load_file(d, fund): try: fname = next( (DAILY_DIR / fund / "JPM_reports").glob( f"CSCFTCSTMT-*-{d:%y%m%d}-{accounts[fund]}_2.pdf" ) ) except StopIteration: raise FileNotFoundError(f"JPM file not found for date {d}") return fname def get_collateral(collat_page): return float(get_col(collat_page, 200, 300, 1000, 1100)[0].replace(",", "")) def load_positions(positions_page): anchor = next(c for c in positions_page if c.text.startswith("Total Product Group")) bottom = int(anchor["top"]) - 30 widths = (10, 160, 300, 380, 450, 500, 550, 635, 700, 780, 850, 960, 1000, 1200) cols = [ get_col(positions_page, 200, bottom, l, r) for l, r in zip(widths, widths[1:]) ] def combine(l): return [f"{l[0]} {l[1]}", *l[2:]] cols = [combine(c) if len(c) == (len(cols[0]) + 1) else c for c in cols] df = pd.DataFrame({c[0]: c[1:] for c in cols}) for col in ["Pay Notional", "Rec Notional", "MTM Amount", "IM Amount"]: df[col] = df[col].apply(parse_num) for col in ["Trade Date", "Maturity Date"]: df[col] = pd.to_datetime(df[col], format="%d-%b-%y") df["Deal ID"] = "810RI" + df["Deal ID"].str.extract(r"([^-]*)-.*") return df def download_files(em, count=20, *, fund="BowdSt", **kwargs): if fund not in paths: return emails = em.get_msgs( path=paths[fund], count=count, subject__contains=accounts[fund] ) DATA_DIR = DAILY_DIR / fund / "JPM_reports" for msg in emails: for attach in msg.attachments: fname = attach.name p = DATA_DIR / fname if not p.exists(): stream = BytesIO(attach.content) pdf = Pdf.open(stream, password=passwords[fund]) pdf.save(p) def collateral(d, dawn_trades, *, fund="BowdSt", **kwargs): pdf_file = load_file(d, fund) pages = load_pdf(pdf_file, pages=True) try: collat = get_collateral(pages[3]) except IndexError: collat = 0.0 df = load_positions(pages[4]) df = df.merge(dawn_trades, how="left", left_on="Deal ID", right_on="cpty_id") missing_ids = df.loc[df.cpty_id.isnull(), "Deal ID"] if not missing_ids.empty: logger.error(f"{missing_ids.tolist()} not in the database") df = df[["folder", "MTM Amount", "IM Amount"]] df = df.groupby("folder", dropna=False).sum() df = df.sum(axis=1).to_frame(name="Amount") df["Currency"] = "USD" df = df.reset_index() df.columns = ["Strategy", "Amount", "Currency"] df = df.append( { "Strategy": "M_CSH_CASH", "Amount": -collat - df.Amount.sum(), "Currency": "USD", }, ignore_index=True, ) df["date"] = d return df.set_index("Strategy")