diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/import_quotes.py | 2 | ||||
| -rw-r--r-- | python/markit_cds.py | 15 |
2 files changed, 13 insertions, 4 deletions
diff --git a/python/import_quotes.py b/python/import_quotes.py index 7cf8fe7b..ede3fb0f 100644 --- a/python/import_quotes.py +++ b/python/import_quotes.py @@ -108,7 +108,7 @@ def insert_index(engine, workdate): ['adjcloseprice', 'adjmodelprice'] for f in filenames: if datetime.datetime.fromtimestamp(os.path.getmtime(f)).date()==(workdate+BDay(1)).date(): - data = pd.read_csv(f, skiprows=2, parse_dates=[0,7]) + data = pd.read_csv(f, skiprows=2, parse_dates=[0,7], engine='python') data.rename(columns=colmapping, inplace=True) data.dropna(subset=['closeprice'], inplace=True) data[cols] = data[cols].applymap(lambda x: float(x[:-1]) if x.endswith('%') else x) diff --git a/python/markit_cds.py b/python/markit_cds.py index 4b8c652b..05b7e9a7 100644 --- a/python/markit_cds.py +++ b/python/markit_cds.py @@ -1,4 +1,5 @@ import requests
+import datetime
from common import root
import os
import csv
@@ -27,7 +28,11 @@ def download_cds_data(payload): f1.close()
f2.close()
-def download_composite_data(payload):
+def download_composite_data(payload, historical=False):
+ ## if historical, we want to maintain the invariant mtime(f)== payload['date'] + BDay(1)
+ if historical:
+ ts = datetime.datetime.strptime(payload['date'], "%Y%m%d") + BDay(1)
+ ts = ts.timestamp()
for report in ['COMPOSITES', 'TRANCHE_COMPOSITES']:
for family in ['CDX', 'ITRAXX-EUROPE']:
payload.update({'family': family, 'report': report})
@@ -36,7 +41,9 @@ def download_composite_data(payload): with zipfile.ZipFile(io.BytesIO(r.content)) as z:
for f in z.namelist():
if "csv" in f:
- z.extract(f, path=os.path.join(root, "Tranche_data", "Composite_reports"))
+ path = z.extract(f, path=os.path.join(root, "Tranche_data", "Composite_reports"))
+ if historical:
+ os.utime(path, (ts, ts))
except zipfile.BadZipfile:
print(r.content)
continue
@@ -44,8 +51,10 @@ def download_composite_data(payload): if __name__=="__main__":
if len(sys.argv) > 1:
workdate = pd.datetime.strptime(sys.argv[1], "%Y-%m-%d")
+ historical = True
else:
workdate = pd.datetime.today()-BDay(1)
+ historical = False
strworkdate = pd.datetime.strftime(workdate, "%Y%m%d")
payload = {'user': 'GuillaumeHorel',
'password': 'password',
@@ -56,7 +65,7 @@ if __name__=="__main__": 'type': 'CDS'}
download_cds_data(payload)
payload.update({'type':'CredIndex','version':4})
- download_composite_data(payload)
+ download_composite_data(payload, historical)
engine = create_engine('postgresql://serenitas_user@debian/serenitasdb')
insert_cds(serenitasdb, workdate.date())
insert_index(engine, workdate.date())
|