diff options
| -rw-r--r-- | python/intex/load_intex_collateral.py | 30 |
1 files changed, 10 insertions, 20 deletions
diff --git a/python/intex/load_intex_collateral.py b/python/intex/load_intex_collateral.py index 08c780fe..4ee61047 100644 --- a/python/intex/load_intex_collateral.py +++ b/python/intex/load_intex_collateral.py @@ -3,6 +3,7 @@ import os, csv, datetime import pdb from common import sanitize_float from db import dbconn +from pathlib import Path import sys import uuid from .load_indicative import upload_cusip_data, upload_deal_data @@ -21,11 +22,7 @@ fields = ['Asset Name', 'Issuer', 'Contributed Balance', 'Asset Maturity Date', def convertToNone(s): return None if s=='' else s -def windows1252_encoder(fh): - for line in fh: - yield line.decode('windows-1252').encode('utf-8') - -def upload_data(conn, dealnames, workdate): +def upload_data(conn, workdate): sql_fields = ["dealname", "updatedate", "name", "IssuerName", "CurrentBalance", "Maturity", "AssetSubtype", "AssetType", "GrossCoupon", "Spread", "Frequency", "NextPaydate", "SecondLien", "LoanXID", @@ -34,14 +31,10 @@ def upload_data(conn, dealnames, workdate): "Liborfloor", "ReinvFlag", "Currency", "Industry", "Country"] sqlstr = "INSERT INTO ET_COLLATERAL({0}) VALUES({1})".format(",".join(sql_fields), ",".join(["%s"] * len(sql_fields))) - for dealname in dealnames: - basedir = os.path.join(os.environ['DATA_DIR'], "Collaterals_" + workdate) - if sys.version_info[0]==2: - fh = open(os.path.join(basedir, dealname.upper() + "_AD.txt")) - fh = windows1252_encoder(fh) - else: - fh = open(os.path.join(basedir, dealname.upper() + "_AD.txt"), encoding="windows-1252") - + basedir = Path(os.environ['DATA_DIR']) / ("Collaterals_" + workdate) + for fname in basedir.iterdir(): + fh = open(fname, encoding="windows-1252") + dealname = fname.name.split("_", 1)[0].lower() dr = csv.DictReader(fh, dialect = 'excel-tab') missingfields = set(fields).union({'Gross Margin'}) - set(dr.fieldnames) if "LoanX ID" in missingfields: @@ -134,12 +127,9 @@ def upload_data(conn, dealnames, workdate): conn.commit() def intex_data(conn, workdate): - dealnames = [d.replace("_AD.txt", "").lower() for d in - os.listdir(os.path.join(os.environ['DATA_DIR'], "Collaterals_" + workdate))] - basedir = os.path.join(os.environ['DATA_DIR'], "Indicative_" + workdate) - files = [os.path.join(basedir, f) for f in os.listdir(basedir)] - cusip_files = [f for f in files if "TrInfo" in f] - deal_files = [f for f in files if "TrInfo" not in f] + basedir = Path(os.environ['DATA_DIR']) / ("Indicative_" + workdate) + cusip_files = [f for f in basedir.iterdir() if "TrInfo" in f.name] + deal_files = [f for f in basedir.iterdir() if "TrInfo" not in f.name] #first load deal data for deal_file in deal_files: upload_deal_data(conn, deal_file) @@ -147,4 +137,4 @@ def intex_data(conn, workdate): for cusip_file in cusip_files: upload_cusip_data(conn, cusip_file) - upload_data(conn, dealnames, workdate) + upload_data(conn, workdate) |