import csv
from lxml import etree
import requests, io, zipfile, shutil
from common import root
import os

user = 'GuillaumeHorel'
password = 'password'


for report in ['REDEntity', 'REDObligation', 'REDEntityDelta', 'REDObligationDelta',
               'REDObligationPreferred', 'CredIndexAnnex', 'CredIndexAnnexSplit',
               'REDIndexCodes']:
    payload = {'user': user,
               'password': password,
               'report':report,
               'version':9}
    if report in ['CredIndexAnnex', 'CredIndexAnnexSplit']:
        #payload['family']='CDX'
        payload['family'] = 'ITRAXX-EUROPE'
    r = requests.get('https://www.markit.com/export.jsp', params=payload)
    with zipfile.ZipFile(io.BytesIO(r.content)) as z:
        for f in z.namelist():
            if "xml" in f:
                z.extract(f, path=os.path.join(root, "Tranche_data", "RED_reports"))


basedir = os.path.join(root, "Tranche_Data", "Red")
fh = open(os.path.join(root, "Tranche_Data", "Red_reports", "V9 Red Indices1416860143432.xml"))
e = etree.parse(fh)
root = e.getroot()

for c in root.findall('index'):
    with open( os.path.join(basedir, "{0}.{1}.V{2}.csv".format(c.find('indexsubfamily').text,
                                                               c.find('series').text,
                                                               c.find('version').text)), "w") as fh2:
        headers = ['referenceentity', 'redentitycode', 'role', 'redpaircode', 'jurisdiction',
                   'tier','pairiscurrent', 'pairvalidto', 'pairvalidfrom', 'ticker',
                   'ispreferred', 'isdatransactiontype', 'docclause','recorddate','publiccomments','weight']
        csvwriter = csv.DictWriter(fh2, fieldnames=headers)
        csvwriter.writeheader()
        data = []
        for constituent in c.findall('.//originalconstituent'):
            data.append({l.tag: l.text for l in constituent})
            data = sorted(data, key=lambda x: x['referenceentity'])
        csvwriter.writerows(data)
fh.close()