diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/load_refentity.py | 28 |
1 files changed, 17 insertions, 11 deletions
diff --git a/python/load_refentity.py b/python/load_refentity.py index 5299c7d4..837b5dfb 100644 --- a/python/load_refentity.py +++ b/python/load_refentity.py @@ -5,18 +5,18 @@ from psycopg2.extras import Json def todict(xml, uselist=set()): if len(xml): - if xml.tag in usearray: - return [todict(c, usearray) for c in xml] + if xml.tag in uselist: + return [todict(c, uselist) for c in xml] else: d = {} for c in xml: if c.tag in d: - d[c.tag].append(todict(c, usearray)) + d[c.tag].append(todict(c, uselist)) else: if c.tag in ['cdssuccession', 'creditevent', 'auctionccyrate']: - d[c.tag] = [todict(c, usearray)] + d[c.tag] = [todict(c, uselist)] else: - d[c.tag] = todict(c, usearray) + d[c.tag] = todict(c, uselist) return d else: return xml.text.strip() @@ -26,11 +26,11 @@ def dispatch_parsing(col, uselist): if col.tag == 'ratings': return [el.text for el in col.iterfind(".//tier")] else: - return Json(todict(col, usearray)) + return Json(todict(col, uselist)) else: return col.text -def upload_refentity(fname): +def insert_refentity(fname): tree = etree.parse(fname, parser=parser) conn = dbconn('serenitasdb') @@ -42,21 +42,24 @@ def upload_refentity(fname): # these are tags which enclose a list uselist = set(['events', 'isdatransactiontypes', 'nextredentitycodes', 'prevredentitycodes', 'isdatransactiontypes', 'tiers', 'auctions']) - sql_str = "INSERT INTO RefEntity VALUES({})".format(",".join(["%s"] * len(names))) + sql_str = ("""INSERT INTO RefEntity VALUES({}) + ON CONFLICT(redentitycode) DO UPDATE SET {}""". + format(",".join(["%s"] * len(names)), + ",".join(f"{name}=EXCLUDED.{name}" for name in names[1:]))) skipfirst = True with conn.cursor() as c: for child in tree.getroot(): if skipfirst: skipfirst = False continue - d = {col.tag: dispatch_parsing(col) for col in child} + d = {col.tag: dispatch_parsing(col, uselist) for col in child} c.execute(sql_str, [d.get(name) for name in names]) conn.commit() def parse_prospectus(xml): return Json({c.tag: [e.text for e in c] for c in xml}) -def upload_refobligation(fname): +def insert_refobligation(fname): tree = etree.parse(fname, parser=parser) conn = dbconn('serenitasdb') @@ -93,4 +96,7 @@ def upload_refobligation(fname): conn.commit() if __name__ == "__main__": - upload_refobligation('/home/serenitas/guillaume/V9 Red Obligation1503420865581.xml') + #insert_refobligation('/home/serenitas/guillaume/V9 Red Obligation1503420865581.xml') + from pathlib import Path + base_dir = Path('/home/serenitas/CorpCDOs/Tranche_data/RED_reports') + insert_refentity((base_dir / 'V10 Red Entity Delta1503687306161.xml').as_posix()) |