diff options
Diffstat (limited to 'python/markit_red.py')
| -rw-r--r-- | python/markit_red.py | 121 |
1 files changed, 79 insertions, 42 deletions
diff --git a/python/markit_red.py b/python/markit_red.py index 35a10f05..d2bd98bb 100644 --- a/python/markit_red.py +++ b/python/markit_red.py @@ -9,11 +9,12 @@ from lxml import etree from utils.db import serenitas_engine import pandas as pd + def request_payload(payload): - r = requests.post('https://products.markit.com/red/export.jsp', params=payload) + r = requests.post("https://products.markit.com/red/export.jsp", params=payload) res = [] - path = os.path.join(os.environ['BASE_DIR'], "Tranche_data", "RED_reports") - if 'Delta' in payload['report']: + path = os.path.join(os.environ["BASE_DIR"], "Tranche_data", "RED_reports") + if "Delta" in payload["report"]: path = os.path.join(path, "Deltas") try: @@ -26,74 +27,110 @@ def request_payload(payload): print(r.content) return res + def download_report(report): - version_mapping = {"REDEntity": 11, - "REDEntityDelta": 11, - "REDEntityMapped": 11, - "REDObligation": 10, - "REDObligationDelta": 10, - "REDObligationMapped": 10, - "REDSROObligation": 10, - "REDobligationpreferred": 10, - "CredIndexAnnex": 10, - "CredIndexAnnexSplit": 10, - "REDIndexCodes": 9, - "redindexclassification": 9, - "redindexclassificationdelta": 9} + version_mapping = { + "REDEntity": 11, + "REDEntityDelta": 11, + "REDEntityMapped": 11, + "REDObligation": 10, + "REDObligationDelta": 10, + "REDObligationMapped": 10, + "REDSROObligation": 10, + "REDobligationpreferred": 10, + "CredIndexAnnex": 10, + "CredIndexAnnexSplit": 10, + "REDIndexCodes": 9, + "redindexclassification": 9, + "redindexclassificationdelta": 9, + } - payload = {'user': 'GuillaumeHorel', - 'password': 'password', - 'version': version_mapping[report], - 'report': report} + payload = { + "user": "GuillaumeHorel", + "password": "password", + "version": version_mapping[report], + "report": report, + } - if report in ['CredIndexAnnex', 'CredIndexAnnexSplit']: - for family in ['CDX', 'ITRAXX-EUROPE']: - payload.update({'family': family}) + if report in ["CredIndexAnnex", "CredIndexAnnexSplit"]: + for family in ["CDX", "ITRAXX-EUROPE"]: + payload.update({"family": family}) yield request_payload(payload) else: yield request_payload(payload) + def update_redcodes(fname): - with open(os.path.join(os.environ['BASE_DIR'], "Tranche_data", "RED_reports", fname)) as fh: + with open( + os.path.join(os.environ["BASE_DIR"], "Tranche_data", "RED_reports", fname) + ) as fh: et = etree.parse(fh) data_version = [] data_maturity = [] - for index in et.iter('index'): - temp = {c.tag: c.text for c in index if c.tag not in ['terms', 'paymentfrequency']} + for index in et.iter("index"): + temp = { + c.tag: c.text for c in index if c.tag not in ["terms", "paymentfrequency"] + } data_version.append(temp) - for term in index.iter('term'): - d = {'redindexcode': temp['redindexcode']} + for term in index.iter("term"): + d = {"redindexcode": temp["redindexcode"]} d.update({c.tag: c.text for c in term}) data_maturity.append(d) df_maturity = pd.DataFrame(data_maturity) df_version = pd.DataFrame(data_version) - df_version['activeversion'] = df_version['activeversion'].map({'Y': True, None: False}) - df_maturity.tenor = df_maturity['tenor'].map(lambda s: s.lower() + 'r') + df_version["activeversion"] = df_version["activeversion"].map( + {"Y": True, None: False} + ) + df_maturity.tenor = df_maturity["tenor"].map(lambda s: s.lower() + "r") df_maturity.coupon = (pd.to_numeric(df_maturity.coupon) * 10000).astype(int) - df_version.to_sql("index_version_markit", serenitas_engine, index=False, if_exists='append') - df_maturity.to_sql("index_maturity_markit", serenitas_engine, index=False, if_exists='append') + df_version.to_sql( + "index_version_markit", serenitas_engine, index=False, if_exists="append" + ) + df_maturity.to_sql( + "index_maturity_markit", serenitas_engine, index=False, if_exists="append" + ) + def update_redindices(fname): - basedir = Path(os.environ['BASE_DIR']) / "Tranche_data" + basedir = Path(os.environ["BASE_DIR"]) / "Tranche_data" with open(basedir / "RED_reports" / fname) as fh: e = etree.parse(fh) root = e.getroot() - headers = ['referenceentity', 'redentitycode', 'role', 'redpaircode', 'jurisdiction', - 'tier', 'pairiscurrent', 'pairvalidto', 'pairvalidfrom', 'ticker', - 'ispreferred', 'isdatransactiontype', 'docclause', 'recorddate', - 'publiccomments', 'weight', 'holdco', 'subordinationtype'] - for c in root.findall('index'): - names = [c.find(tag).text for tag in ['indexsubfamily', 'series', 'version']] - with open( basedir / "RED_indices" / "{0}.{1}.V{2}.csv".format(*names), "w") as fh2: + headers = [ + "referenceentity", + "redentitycode", + "role", + "redpaircode", + "jurisdiction", + "tier", + "pairiscurrent", + "pairvalidto", + "pairvalidfrom", + "ticker", + "ispreferred", + "isdatransactiontype", + "docclause", + "recorddate", + "publiccomments", + "weight", + "holdco", + "subordinationtype", + ] + for c in root.findall("index"): + names = [c.find(tag).text for tag in ["indexsubfamily", "series", "version"]] + with open( + basedir / "RED_indices" / "{0}.{1}.V{2}.csv".format(*names), "w" + ) as fh2: csvwriter = csv.DictWriter(fh2, fieldnames=headers) csvwriter.writeheader() data = [] - for constituent in c.findall('.//originalconstituent'): + for constituent in c.findall(".//originalconstituent"): data.append({l.tag: l.text for l in constituent}) - data = sorted(data, key=lambda x: x['referenceentity']) + data = sorted(data, key=lambda x: x["referenceentity"]) csvwriter.writerows(data) + if __name__ == "__main__": fname = next(download_report("REDIndexCodes")) update_redcodes(fname[0]) |