1 files changed, 17 insertions, 11 deletions
diff --git a/python/load_refentity.py b/python/load_refentity.py
index 5299c7d4..837b5dfb 100644
--- a/python/load_refentity.py
+++ b/python/load_refentity.py
@@ -5,18 +5,18 @@ from psycopg2.extras import Json
 
 def todict(xml, uselist=set()):
     if len(xml):
-        if xml.tag in usearray:
-            return [todict(c, usearray) for c in xml]
+        if xml.tag in uselist:
+            return [todict(c, uselist) for c in xml]
         else:
             d = {}
             for c in xml:
                 if c.tag in d:
-                    d[c.tag].append(todict(c, usearray))
+                    d[c.tag].append(todict(c, uselist))
                 else:
                     if c.tag in ['cdssuccession', 'creditevent', 'auctionccyrate']:
-                        d[c.tag] = [todict(c, usearray)]
+                        d[c.tag] = [todict(c, uselist)]
                     else:
-                        d[c.tag] = todict(c, usearray)
+                        d[c.tag] = todict(c, uselist)
             return d
     else:
         return xml.text.strip()
@@ -26,11 +26,11 @@ def dispatch_parsing(col, uselist):
         if col.tag == 'ratings':
             return [el.text for el in col.iterfind(".//tier")]
         else:
-            return Json(todict(col, usearray))
+            return Json(todict(col, uselist))
     else:
         return col.text
 
-def upload_refentity(fname):
+def insert_refentity(fname):
     tree = etree.parse(fname,
                        parser=parser)
     conn = dbconn('serenitasdb')
@@ -42,21 +42,24 @@ def upload_refentity(fname):
     # these are tags which enclose a list
     uselist = set(['events', 'isdatransactiontypes', 'nextredentitycodes', 'prevredentitycodes',
                    'isdatransactiontypes', 'tiers', 'auctions'])
-    sql_str = "INSERT INTO RefEntity VALUES({})".format(",".join(["%s"] * len(names)))
+    sql_str = ("""INSERT INTO RefEntity VALUES({})
+                 ON CONFLICT(redentitycode) DO UPDATE SET {}""".
+               format(",".join(["%s"] * len(names)),
+                      ",".join(f"{name}=EXCLUDED.{name}" for name in names[1:])))
     skipfirst = True
     with conn.cursor() as c:
         for child in tree.getroot():
             if skipfirst:
                 skipfirst = False
                 continue
-            d = {col.tag: dispatch_parsing(col) for col in child}
+            d = {col.tag: dispatch_parsing(col, uselist) for col in child}
             c.execute(sql_str, [d.get(name) for name in names])
     conn.commit()
 
 def parse_prospectus(xml):
     return Json({c.tag: [e.text for e in c] for c in xml})
 
-def upload_refobligation(fname):
+def insert_refobligation(fname):
     tree = etree.parse(fname,
                        parser=parser)
     conn = dbconn('serenitasdb')
@@ -93,4 +96,7 @@ def upload_refobligation(fname):
     conn.commit()
 
 if __name__ == "__main__":
-    upload_refobligation('/home/serenitas/guillaume/V9 Red Obligation1503420865581.xml')
+    #insert_refobligation('/home/serenitas/guillaume/V9 Red Obligation1503420865581.xml')
+    from pathlib import Path
+    base_dir = Path('/home/serenitas/CorpCDOs/Tranche_data/RED_reports')
+    insert_refentity((base_dir / 'V10 Red Entity Delta1503687306161.xml').as_posix())