aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--python/markit/__init__.py0
-rw-r--r--python/markit/__main__.py81
-rw-r--r--python/markit/cds.py (renamed from python/markit_cds.py)43
-rw-r--r--python/markit/import_quotes.py (renamed from python/import_quotes.py)37
-rw-r--r--python/markit/loans.py (renamed from python/markit_loans.py)37
5 files changed, 107 insertions, 91 deletions
diff --git a/python/markit/__init__.py b/python/markit/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/python/markit/__init__.py
diff --git a/python/markit/__main__.py b/python/markit/__main__.py
new file mode 100644
index 00000000..a273860f
--- /dev/null
+++ b/python/markit/__main__.py
@@ -0,0 +1,81 @@
+import argparse
+import datetime
+import logging
+import os
+import pandas as pd
+import sys
+import logging
+import pdb
+
+from common import root
+from .cds import download_cds_data, download_composite_data
+from .import_quotes import insert_cds, insert_index, insert_tranche
+from pandas.tseries.offsets import BDay
+from sqlalchemy import create_engine
+from db import dbconn
+
+## parse arguments
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument("-l", "--loans", action="store_true",
+ help="download markit loan data")
+group.add_argument("-c", "--cds", action="store_true",
+ help="download markit cds data")
+parser.add_argument('workdate', nargs='?', type = lambda s: pd.datetime.strptime(sys.argv[1], "%Y-%m-%d")),
+# ,
+# default = pd.datetime.today() - BDay(1))
+
+args = parser.parse_args()
+historical = True
+
+if args.loans:
+ log_file = os.path.join(root, 'logs', 'markit_loans.log')
+ if args.workdate is None:
+ historical = False
+ workdate = datetime.date.today()
+else:
+ log_file = os.path.join(root, 'logs', 'markit_cds.log')
+ if args.workdate is None:
+ historical = False
+ workdate = pd.datetime.today() - BDay(1)
+
+## set up logging
+logger = logging.getLogger('markit')
+fh = logging.FileHandler(filename=log_file)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh.setFormatter(formatter)
+logger.addHandler(fh)
+logger.setLevel(logging.INFO)
+
+if args.loans:
+ workdate = str(workdate)
+ payload={'LEGALENTITY': 'lmcg',
+ 'USERNAME': 'serecapuser',
+ 'PASSWORD': 'Welcome1'}
+ download_facility(workdate, payload)
+ logger.info('facility downloaded')
+ insert_facility(workdate)
+ logger.info('facility inserted')
+ payload.update({'EOD':'Y'})
+ download_marks(workdate, payload)
+ logger.info('marks downloaded')
+ payload.pop('EOD')
+ update_facility(workdate, payload)
+ logger.info('facility updated')
+else:
+ payload = {'user': 'GuillaumeHorel',
+ 'password': 'password',
+ 'version': '5',
+ 'format': 'csv',
+ 'report': 'FIXED_COUPON',
+ 'date': workdate.strftime("%Y%m%d"),
+ 'type': 'CDS'}
+ download_cds_data(payload)
+ payload.update({'type':'CredIndex','version':4})
+ download_composite_data(payload, historical)
+ engine = create_engine('postgresql://serenitas_user@debian/serenitasdb')
+ serenitasdb = dbconn('serenitasdb')
+ insert_cds(serenitasdb, workdate.date())
+ insert_index(engine, workdate.date())
+ insert_tranche(engine, workdate.date())
+ serenitasdb.close()
diff --git a/python/markit_cds.py b/python/markit/cds.py
index bb2ba2a0..7fede498 100644
--- a/python/markit_cds.py
+++ b/python/markit/cds.py
@@ -1,16 +1,13 @@
-import requests
-import datetime
-from common import root
+import io
+import logging
import os
-import csv
-import sys
-import zipfile, io
+import requests
import shutil
-import pandas as pd
+import zipfile
+from common import root
from pandas.tseries.offsets import BDay
-from db import dbconn
-from import_quotes import insert_cds, insert_index, insert_tranche
-from sqlalchemy import create_engine
+
+logger = logging.getLogger(__name__)
def convertToNone(v):
return v if v else None
@@ -45,29 +42,5 @@ def download_composite_data(payload, historical=False):
if historical:
os.utime(path, (ts, ts))
except zipfile.BadZipfile:
- print(r.content)
+ logger.error(r.content)
continue
-
-if __name__=="__main__":
- if len(sys.argv) > 1:
- workdate = pd.datetime.strptime(sys.argv[1], "%Y-%m-%d")
- historical = True
- else:
- workdate = pd.datetime.today()-BDay(1)
- historical = False
- payload = {'user': 'GuillaumeHorel',
- 'password': 'password',
- 'version': '5',
- 'format': 'csv',
- 'report': 'FIXED_COUPON',
- 'date': workdate.strftime("%Y%m%d"),
- 'type': 'CDS'}
- download_cds_data(payload)
- payload.update({'type':'CredIndex','version':4})
- download_composite_data(payload, historical)
- engine = create_engine('postgresql://serenitas_user@debian/serenitasdb')
- serenitasdb = dbconn('serenitasdb')
- insert_cds(serenitasdb, workdate.date())
- insert_index(engine, workdate.date())
- insert_tranche(engine, workdate.date())
- serenitasdb.close()
diff --git a/python/import_quotes.py b/python/markit/import_quotes.py
index 0e053dd4..7bbb6e89 100644
--- a/python/import_quotes.py
+++ b/python/markit/import_quotes.py
@@ -1,15 +1,15 @@
-import os
-from common import root
import csv
import datetime
-from db import dbconn
-import re, sys
-from pandas.tseries.offsets import BDay
-import pandas as pd
+import logging
import numpy as np
-import psycopg2
-from sqlalchemy import create_engine
+import pandas as pd
+import os
+
from collections import defaultdict
+from common import root
+from pandas.tseries.offsets import BDay
+
+logger = logging.getLogger(__name__)
def convert(x):
try:
@@ -99,7 +99,7 @@ def insert_cds(database, workdate):
for i, t in enumerate(curves)])
tickers_found.add((line['Ticker'], line['Tier']))
database.commit()
- print(all_tickers-tickers_found)
+ logger.warning('missing_quotes for {0}'.format(all_tickers-tickers_found))
def insert_index(engine, workdate=None):
"""insert Markit index quotes into the database
@@ -174,22 +174,3 @@ def insert_tranche(engine, workdate = None):
'upfront_bid', 'upfront_ask', 'upfront_mid',
'tranche_spread', 'index_price'])
df.to_sql('markit_tranche_quotes', engine, if_exists='append', index=False)
-
-if __name__=="__main__":
- if len(sys.argv)>=2:
- workdate = datetime.datetime.strptime(sys.argv[1], "%Y-%m-%d")
- else:
- workdate = datetime.datetime.today()-BDay(1)
- workdate = workdate.date()
- engine = create_engine('postgresql://serenitas_user@debian/serenitasdb')
- serenitasdb = dbconn('serenitasdb')
- insert_cds(serenitasdb, workdate)
- insert_index(engine, workdate)
- serenitasdb.close()
-
- # for f in os.listdir(os.path.join(root, "Tranche_data", "CDS")):
- # if f.endswith("csv"):
- # workdate = datetime.datetime.strptime(f.split(" ")[2].split(".")[0], "%Y%m%d")
- # workdate = workdate.date()
- # insert_cds(serenitasdb, workdate)
- # serenitasdb.close()
diff --git a/python/markit_loans.py b/python/markit/loans.py
index 39d733bb..0b14d60b 100644
--- a/python/markit_loans.py
+++ b/python/markit/loans.py
@@ -1,12 +1,12 @@
+import csv
+import logging
+import os
import requests
+
from common import root
from db import with_connection
-import os
-import datetime
-import csv
-import sys
-import logging
-from psycopg2 import IntegrityError
+
+logger = logging.getLogger(__name__)
def download_facility(workdate, payload):
r = requests.get('https://loans.markit.com/loanx/LoanXFacilityUpdates.csv', params=payload)
@@ -22,7 +22,7 @@ def insert_facility(conn, workdate):
reader = csv.reader(fh)
header = next(reader)
if 'Authentication failed' in header:
- logging.error("Couldn't authenticate")
+ logger.error("Couldn't authenticate")
raise SystemExit
with conn.cursor() as c:
for line in reader:
@@ -30,7 +30,7 @@ def insert_facility(conn, workdate):
try:
c.execute(sqlstring, newline)
except IntegrityError as e:
- logging.error(e)
+ logger.error(e)
conn.rollback()
else:
conn.commit()
@@ -45,7 +45,7 @@ def download_marks(conn, workdate, payload):
with open(marks_filename, "r") as fh:
reader = csv.DictReader(fh)
if 'Authentication failed' in reader.fieldnames[0]:
- logging.error("Couldn't authenticate")
+ logger.error("Couldn't authenticate")
raise SystemExit
with conn.cursor() as c:
for line in reader:
@@ -91,22 +91,3 @@ def update_facility(conn, workdate, payload):
except StopIteration:
pass
conn.close()
-
-if __name__=="__main__":
- if len(sys.argv) > 1:
- workdate = datetime.datetime.strptime(sys.argv[1], "%Y-%m-%d").date()
- else:
- workdate = datetime.date.today()
- workdate = str(workdate)
- logging.basicConfig(filename = '/home/share/CorpCDOs/logs/markit_loans.log',
- level = logging.INFO,
- format = '%(asctime)s %(levelname)s %(message)s')
- payload={'LEGALENTITY': 'lmcg',
- 'USERNAME': 'serecapuser',
- 'PASSWORD': 'Welcome1'}
- download_facility(workdate, payload)
- insert_facility(workdate)
- payload.update({'EOD':'Y'})
- download_marks(workdate, payload)
- payload.pop('EOD')
- update_facility(workdate, payload)
generated by cgit v1.2.3-70-g09d2 (git 2.52.0) at 2025-12-22 05:59:12 +0000