diff options
| -rw-r--r-- | python/intex/__init__.py | 3 | ||||
| -rw-r--r-- | python/intex/__main__.py | 2 | ||||
| -rw-r--r-- | python/intex/load_indicative.py | 38 | ||||
| -rw-r--r-- | python/intex/load_intex_collateral.py | 12 |
4 files changed, 25 insertions, 30 deletions
diff --git a/python/intex/__init__.py b/python/intex/__init__.py index 168432ea..8b137891 100644 --- a/python/intex/__init__.py +++ b/python/intex/__init__.py @@ -1,4 +1 @@ -import sys -sys.path.append("..") -from utils.db import dbconn diff --git a/python/intex/__main__.py b/python/intex/__main__.py index 2622cf6e..7a4a644f 100644 --- a/python/intex/__main__.py +++ b/python/intex/__main__.py @@ -1,7 +1,7 @@ import sys import datetime from .load_intex_collateral import intex_data -from utils.db import dbconn +from serenitas.utils.db import dbconn import logging logger = logging.getLogger("intex") diff --git a/python/intex/load_indicative.py b/python/intex/load_indicative.py index c9bc5bfd..0a2b9d5f 100644 --- a/python/intex/load_indicative.py +++ b/python/intex/load_indicative.py @@ -1,14 +1,11 @@ import psycopg2 -import os import datetime import csv import pdb +import pathlib from common import sanitize_float -from . import dbconn -from contextlib import closing import logging import re -import sys logger = logging.getLogger(__name__) @@ -65,9 +62,9 @@ def insert_new_cusip(conn, line): conn.commit() -def upload_cusip_data(conn, filename): +def upload_cusip_data(conn, filename: pathlib.Path): dealupdate = {} - with open(filename, "r") as fh: + with filename.open("r") as fh: dr = csv.DictReader(fh, dialect="excel-tab") for line in dr: @@ -145,7 +142,7 @@ def upload_cusip_data(conn, filename): conn.commit() -def upload_deal_data(conn, filename): +def upload_deal_data(conn, filename: pathlib.Path): sqlstr = 'SELECT dealname, array_agg("Latest Update") FROM clo_universe GROUP BY dealname' with conn.cursor() as c: c.execute(sqlstr) @@ -155,7 +152,7 @@ def upload_deal_data(conn, filename): c.execute(sqlstr) deallist2 = set([d for d, in c]) conn.commit() - with open(filename, "r") as fh: + with filename.open("r") as fh: dr = csv.DictReader(fh, dialect="excel-tab") data = [] for line in dr: @@ -260,18 +257,21 @@ def upload_deal_data(conn, filename): if __name__ == "__main__": - if len(sys.argv) > 1: - workdate = sys.argv[1] - else: - workdate = str(datetime.date.today()) - files = [ - os.path.join(os.environ["DATA_DIR"], "Indicative_" + workdate, f) - for f in os.listdir( - os.path.join(os.environ["DATA_DIR"], "Indicative_" + workdate) - ) + from serenitas.utils.db import dbconn + from serenitas.utils.env import DATA_DIR + from contextlib import closing + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("workdate", nargs="?", default=str(datetime.date.today())) + args = parser.parse_args() + + cusip_files = [ + f for f in (DATA_DIR / "Indicative_").iterdir() if "TrInfo" in f.name + ] + deal_files = [ + f for f in (DATA_DIR / "Indicative_").iterdir() if "TrInfo" not in f.name ] - cusip_files = [f for f in files if "TrInfo" in f] - deal_files = [f for f in files if "TrInfo" not in f] with closing(dbconn("etdb")) as etdb: # first load deal data diff --git a/python/intex/load_intex_collateral.py b/python/intex/load_intex_collateral.py index e43a58b1..2e8bad22 100644 --- a/python/intex/load_intex_collateral.py +++ b/python/intex/load_intex_collateral.py @@ -1,10 +1,8 @@ +import csv +import datetime import psycopg2 -import os, csv, datetime -import pdb from common import sanitize_float -from . import dbconn -from pathlib import Path -import sys +from serenitas.utils.env import DATA_DIR import uuid from .load_indicative import upload_cusip_data, upload_deal_data import logging @@ -76,7 +74,7 @@ def upload_data(conn, workdate): sqlstr = "INSERT INTO ET_COLLATERAL({0}) VALUES({1})".format( ",".join(sql_fields), ",".join(["%s"] * len(sql_fields)) ) - basedir = Path(os.environ["DATA_DIR"]) / ("Collaterals_" + workdate) + basedir = DATA_DIR / ("Collaterals_" + workdate) for fname in basedir.iterdir(): if fname.stem.endswith("zip"): continue @@ -214,7 +212,7 @@ def upload_data(conn, workdate): def intex_data(conn, workdate): - basedir = Path(os.environ["DATA_DIR"]) / ("Indicative_" + workdate) + basedir = DATA_DIR / ("Indicative_" + workdate) cusip_files = [f for f in basedir.iterdir() if "TrInfo" in f.name] deal_files = [f for f in basedir.iterdir() if "TrInfo" not in f.name] # first load deal data |