diff options
Diffstat (limited to 'python/load_refentity.py')
| -rw-r--r-- | python/load_refentity.py | 22 |
1 files changed, 17 insertions, 5 deletions
diff --git a/python/load_refentity.py b/python/load_refentity.py index 4b3e2864..ea9ca6bd 100644 --- a/python/load_refentity.py +++ b/python/load_refentity.py @@ -31,7 +31,10 @@ def dispatch_parsing(col, uselist): else: return Json(todict(col, uselist)) else: - return col.text + if col.tag == "holdco": + return col.text == "TRUE" + else: + return col.text def insert_refentity(fname): tree = etree.parse(fname, @@ -41,7 +44,8 @@ def insert_refentity(fname): 'entitycusip', 'lei', 'entitytype', 'jurisdiction', 'depthlevel', 'markitsector', 'isdatradingdefinition', 'recorddate', 'ratings', 'entityform', 'companynumber', 'alternativenames', - 'isdatransactiontypes', 'validto', 'validfrom', 'events'] + 'isdatransactiontypes', 'validto', 'validfrom', 'events', + 'holdco', 'country',] # these are tags which enclose a list uselist = set(['events', 'isdatransactiontypes', 'nextredentitycodes', 'prevredentitycodes', 'isdatransactiontypes', 'tiers', 'auctions']) @@ -69,7 +73,8 @@ def insert_refobligation(fname): names_redpair = ['redpaircode', 'role', 'referenceentity', 'redentitycode', 'tier', 'pairiscurrent', 'pairvalidfrom', 'pairvalidto', 'ticker', 'ispreferred', 'preferreddate', 'indexconstituents', - 'recorddate', 'publiccomments', 'myticker'] + 'recorddate', 'publiccomments', 'myticker', + 'subordinationtype', 'holdco', 'preferredremovaldate'] names_refobligation = ['obligationname', 'prospectusinfo', 'refentities', 'type', 'isconvert', 'isperp', 'coupontype', 'ccy', 'maturity', 'issuedate', 'coupon', 'isin', 'cusip', 'event'] @@ -84,6 +89,14 @@ def insert_refobligation(fname): ",".join(["%s"] * len(names_refobligation)), ",".join(f"{name}=EXCLUDED.{name}" for name in names_refobligation[1:]))) skipfirst = True + def simple_parse(e): + if e.tag == 'indexconstituents': + return [ic.text for ic in e] + elif e.tag == 'holdco': + return e.text == "TRUE" + else: + return e.text + with conn.cursor() as c: for child in tree.getroot(): if skipfirst: @@ -92,8 +105,7 @@ def insert_refobligation(fname): d = {col.tag: col if len(col) else col.text for col in child} with conn.cursor() as c2: for el in d['refentities']: - redpair = {e.tag: [ic.text for ic in e] if e.tag == 'indexconstituents' - else e.text for e in el} + redpair = {e.tag: simple_parse(e) for e in el} c2.execute(redpair_insert, [redpair.get(name) for name in names_redpair]) conn.commit() d['refentities'] = [el.text for el in d['refentities'].iterfind('.//redpaircode')] |