aboutsummaryrefslogtreecommitdiffstats
path: root/python/markit
diff options
context:
space:
mode:
Diffstat (limited to 'python/markit')
-rw-r--r--python/markit/README.md5
-rw-r--r--python/markit/__main__.py7
-rw-r--r--python/markit/cds.py8
-rw-r--r--python/markit/import_quotes.py7
-rw-r--r--python/markit/loans.py11
-rw-r--r--python/markit/rates.py7
6 files changed, 24 insertions, 21 deletions
diff --git a/python/markit/README.md b/python/markit/README.md
new file mode 100644
index 00000000..80ef9a28
--- /dev/null
+++ b/python/markit/README.md
@@ -0,0 +1,5 @@
+Environment variables:
+
+- DATA_DIR
+- BASE_DIR
+- LOG_DIR
diff --git a/python/markit/__main__.py b/python/markit/__main__.py
index 456e02c9..a157ae2a 100644
--- a/python/markit/__main__.py
+++ b/python/markit/__main__.py
@@ -7,7 +7,6 @@ import os
import sys
import logging
-from common import root
from .cds import download_cds_data, download_composite_data
from .loans import download_facility, insert_facility, download_marks, update_facility
from .rates import downloadMarkitIRData
@@ -39,11 +38,11 @@ else:
workdate = args.workdate
if args.loans:
- log_file = os.path.join(root, 'logs', 'markit_loans.log')
+ log_file = os.path.join(os.environ['LOG_DIR'], 'markit_loans.log')
elif args.cds:
- log_file = os.path.join(root, 'logs', 'markit_cds.log')
+ log_file = os.path.join(os.environ['LOG_DIR'], 'markit_cds.log')
elif args.rates:
- log_file = os.path.join(root, 'logs', 'markit_rates.log')
+ log_file = os.path.join(os.environ['LOG_DIR'], 'markit_rates.log')
## set up logging
logger = logging.getLogger('markit')
diff --git a/python/markit/cds.py b/python/markit/cds.py
index 2ae7692f..29e1f623 100644
--- a/python/markit/cds.py
+++ b/python/markit/cds.py
@@ -5,7 +5,6 @@ import requests
import shutil
import zipfile
import time
-from common import root
from pandas.tseries.offsets import BDay
import pandas as pd
logger = logging.getLogger(__name__)
@@ -15,7 +14,8 @@ def convertToNone(v):
def download_cds_data(payload):
r = requests.get('https://www.markit.com/export.jsp', params=payload)
- f2 = open(os.path.join(root, "Tranche_data", "CDS", "cds eod {0}.csv".format(payload['date'])), "wb")
+ f2 = open(os.path.join(os.environ['BASE_DIR'], "Tranche_data", "CDS",
+ "cds eod {0}.csv".format(payload['date'])), "wb")
with zipfile.ZipFile(io.BytesIO(r.content)) as z:
for f in z.namelist():
if "csv" in f:
@@ -39,7 +39,9 @@ def download_composite_data(payload, historical=False):
with zipfile.ZipFile(io.BytesIO(r.content)) as z:
for f in z.namelist():
if "csv" in f:
- path = z.extract(f, path=os.path.join(root, "Tranche_data", "Composite_reports"))
+ path = z.extract(f, path=os.path.join(os.environ['BASE_DIR'],
+ "Tranche_data",
+ "Composite_reports"))
if historical:
os.utime(path, (ts, ts))
except zipfile.BadZipfile:
diff --git a/python/markit/import_quotes.py b/python/markit/import_quotes.py
index 88166dd8..a584be96 100644
--- a/python/markit/import_quotes.py
+++ b/python/markit/import_quotes.py
@@ -6,7 +6,6 @@ import pandas as pd
import os
from collections import defaultdict
-from common import root
from pandas.tseries.offsets import BDay
logger = logging.getLogger(__name__)
@@ -87,7 +86,7 @@ def insert_cds(database, workdate):
with database.cursor() as c:
c.execute("DELETE from cds_quotes where date=%s", (workdate,))
database.commit()
- with open(os.path.join(root, "Tranche_data", "CDS", filename)) as fh:
+ with open(os.path.join(os.environ['BASE_DIR'], "Tranche_data", "CDS", filename)) as fh:
csvreader = csv.DictReader(fh)
with database.cursor() as c:
for line in csvreader:
@@ -110,7 +109,7 @@ def insert_index(engine, workdate=None):
:param workdate: date. If None, we will try to reinsert all files
"""
- basedir = os.path.join(root, 'Tranche_data', 'Composite_reports')
+ basedir = os.path.join(os.environ['BASE_DIR'], 'Tranche_data', 'Composite_reports')
filenames = [os.path.join(basedir, f) for f in os.listdir(basedir) if 'Indices' in f]
name_mapping = {"CDXNAHY":"HY", "CDXNAIG":"IG",'iTraxx Eur': "EU", 'iTraxx Eur Xover': "XO"}
@@ -150,7 +149,7 @@ def insert_tranche(engine, workdate=None):
:type workdate: pd.Timestamp
"""
- basedir = os.path.join(root, 'Tranche_data', 'Composite_reports')
+ basedir = os.path.join(os.environ['BASE_DIR'], 'Tranche_data', 'Composite_reports')
filenames = [os.path.join(basedir, f) for f in os.listdir(basedir) if f.startswith('Tranche Composites')]
index_version = pd.read_sql_table("index_version", engine, index_col='redindexcode')
for f in filenames:
diff --git a/python/markit/loans.py b/python/markit/loans.py
index f038aa38..f9fbe065 100644
--- a/python/markit/loans.py
+++ b/python/markit/loans.py
@@ -3,7 +3,6 @@ import logging
import os
import requests
-from common import root
from db import with_connection
from psycopg2 import IntegrityError
@@ -11,13 +10,13 @@ logger = logging.getLogger(__name__)
def download_facility(workdate, payload):
r = requests.get('https://loans.markit.com/loanx/LoanXFacilityUpdates.csv', params=payload)
- facility_filename = os.path.join(root, "data", "Facility files", "facility_{0}.csv".format(workdate))
+ facility_filename = os.path.join(os.environ['DATA_DIR'], "Facility files", "facility_{0}.csv".format(workdate))
with open( facility_filename, "wb") as fh:
fh.write(r.content)
@with_connection('etdb')
def insert_facility(conn, workdate):
- facility_filename = os.path.join(root, "data", "Facility files", "facility_{0}.csv".format(workdate))
+ facility_filename = os.path.join(os.environ['DATA_DIR'], "Facility files", "facility_{0}.csv".format(workdate))
sqlstring = "INSERT INTO markit_facility VALUES( {0} )".format( ",".join(["%s"] * 13))
with open( facility_filename, "r") as fh:
reader = csv.reader(fh)
@@ -39,7 +38,7 @@ def insert_facility(conn, workdate):
@with_connection('etdb')
def download_marks(conn, workdate, payload):
r = requests.get('https://loans.markit.com/loanx/LoanXMarks.csv', params=payload)
- marks_filename = os.path.join(root, "data", "markit", "markit_data_{0}.csv".format(workdate))
+ marks_filename = os.path.join(os.environ['DATA_DIR'], "markit", "markit_data_{0}.csv".format(workdate))
with open(marks_filename, "wb") as fh:
fh.write(r.content)
sqlstring = "INSERT INTO markit_prices VALUES( {0} )".format( ",".join([ "%s" ] * 5))
@@ -60,14 +59,14 @@ def download_marks(conn, workdate, payload):
def update_facility(conn, workdate, payload):
#we update the missing facility loanxids
sqlstring = "SELECT loanxid FROM markit_prices EXCEPT SELECT loanxid FROM markit_facility";
- facility_diff_filename = os.path.join(root, "data", "Facility files",
+ facility_diff_filename = os.path.join(os.environ['DATA_DIR'], "Facility files",
"facility_diff_{0}.csv".format(workdate))
with open( facility_diff_filename, "wb") as fh:
flag = False
with conn.cursor() as c:
c.execute(sqlstring)
for loanxid in c:
- payload.update({'LOANXID':loanxid[0]})
+ payload.update({'LOANXID': loanxid[0]})
r = requests.get('https://loans.markit.com/loanx/LoanXOneFacility.csv', params=payload)
if flag:
fh.write(r.content.split('\n')[1] + "\n")
diff --git a/python/markit/rates.py b/python/markit/rates.py
index 07d1e5f1..7d96bbb4 100644
--- a/python/markit/rates.py
+++ b/python/markit/rates.py
@@ -1,5 +1,4 @@
from db import dbconn
-from common import root
import datetime
from io import BytesIO
import os
@@ -15,17 +14,17 @@ def downloadMarkitIRData(download_date=datetime.date.today(),
conn = dbconn("serenitasdb")
## T+1 rates are published in the evening
effective_date = download_date + datetime.timedelta(days=1)
- basedir = os.path.join(root, "data", "Yield Curves")
+ basedir = os.path.join(os.environ['DATA_DIR'], "Yield Curves")
filename = "InterestRates_{0}_{1:%Y%m%d}".format(currency, effective_date)
if not os.path.exists(os.path.join(basedir, filename + '.xml')):
r = requests.get('http://www.markit.com/news/{0}.zip'.format(filename))
if "zip" in r.headers['content-type']:
with zipfile.ZipFile(BytesIO(r.content)) as z:
- z.extractall(path = os.path.join(root, "data", "Yield Curves"))
+ z.extractall(path = os.path.join(os.environ['DATA_DIR'], "Yield Curves"))
else:
return downloadMarkitIRData(download_date - datetime.timedelta(days=1))
- tree = ET.parse(os.path.join(root, "data", "Yield Curves", filename + '.xml'))
+ tree = ET.parse(os.path.join(os.environ['DATA_DIR'], "Yield Curves", filename + '.xml'))
deposits = zip([e.text for e in tree.findall('./deposits/*/tenor')],
[float(e.text) for e in tree.findall('./deposits/*/parrate')])
swaps = zip([e.text for e in tree.findall('./swaps/*/tenor')],
generated by cgit v1.2.3-70-g09d2 (git 2.52.0) at 2025-12-21 21:09:13 +0000