diff options
| -rw-r--r-- | python/load_refentity.py | 10 | ||||
| -rw-r--r-- | python/markit_red.py | 15 |
2 files changed, 11 insertions, 14 deletions
diff --git a/python/load_refentity.py b/python/load_refentity.py index 2d07bf62..99c51b16 100644 --- a/python/load_refentity.py +++ b/python/load_refentity.py @@ -1,11 +1,11 @@ -from utils.db import dbconn -from pathlib import Path +from serenitas.utils.db import dbconn +from serenitas.utils.env import BASE_DIR import datetime import re import lxml.etree as etree +from psycopg2.extras import Json parser = etree.XMLParser(remove_blank_text=True) -from psycopg2.extras import Json def todict(xml, uselist=set()): @@ -185,7 +185,7 @@ def insert_refobligation(fname): def get_date(f): - m = re.search("(\d*)\.", f.name) + m = re.search(r"(\d*)\.", f.name) if m: timestamp = int(m.groups(0)[0]) return datetime.datetime.fromtimestamp(timestamp / 1000) @@ -194,7 +194,7 @@ def get_date(f): if __name__ == "__main__": from markit_red import download_report - base_dir = Path("/home/serenitas/CorpCDOs/Tranche_data/RED_reports/Deltas") + base_dir = BASE_DIR / "Tranche_data/RED_reports/Deltas" for report in ["REDEntityDelta", "REDObligationDelta"]: g = download_report(report) f = base_dir / next(g)[0] diff --git a/python/markit_red.py b/python/markit_red.py index a6eeaa28..914a09fa 100644 --- a/python/markit_red.py +++ b/python/markit_red.py @@ -1,20 +1,19 @@ import csv import io -import os import requests import zipfile -from pathlib import Path from lxml import etree -from utils.db import serenitas_engine +from serenitas.utils.db import serenitas_engine +from serenitas.utils.env import BASE_DIR import pandas as pd def request_payload(payload): r = requests.post("https://products.ihsmarkit.com/red/export.jsp", params=payload) res = [] - path = os.path.join(os.environ["BASE_DIR"], "Tranche_data", "RED_reports") + path = BASE_DIR / "Tranche_data" / "RED_reports" if "Delta" in payload["report"]: - path = os.path.join(path, "Deltas") + path = path / "Deltas" try: with zipfile.ZipFile(io.BytesIO(r.content)) as z: @@ -60,9 +59,7 @@ def download_report(report): def update_redcodes(fname): - with open( - os.path.join(os.environ["BASE_DIR"], "Tranche_data", "RED_reports", fname) - ) as fh: + with ("BASE_DIR" / "Tranche_data" / "RED_reports" / fname).open() as fh: et = etree.parse(fh) data_version = [] data_maturity = [] @@ -93,7 +90,7 @@ def update_redcodes(fname): def update_redindices(fname): - basedir = Path(os.environ["BASE_DIR"]) / "Tranche_data" + basedir = BASE_DIR / "Tranche_data" with open(basedir / "RED_reports" / fname) as fh: e = etree.parse(fh) root = e.getroot() |