import os import os.path import datetime from ftplib import FTP import gnupg import config import re import logging import argparse import shutil try: import pandas as pd from pandas.tseries.offsets import BDay except ImportError: pass if os.name =='nt': root = "//WDsentinel/share/Daily" elif os.name == 'posix': root = '/home/share/Daily' def get_ped(s): regex = re.search("PED=(.+?)\.", s) if regex: PED = datetime.datetime.strptime(regex.group(1), "%Y-%m-%d").date() else: regex = re.search("(.+?)\.", s) PED = pd.to_datetime(regex.group(1), "%Y%m%d") - BDay(1) PED = PED.date() return PED def key_fun(s): PED = get_ped(s) regex = re.search("KD=(.+?)\.", s) if regex: KD = datetime.datetime.strptime(regex.group(1), "%Y-%m-%d-%H-%M-%S") else: regex = re.search("(.+?\..+?)\.", s) KD = datetime.datetime.strptime(regex.group(1), "%Y%m%d.%H%M%S") return (PED, KD) def download_data(workdate): ftp = FTP('ftp.globeop.com') ftp.login('srntsftp', config.ftp_password) ftp.cwd('outgoing') files = ftp.nlst() pnlfiles = [filename for filename in files if "csv" in filename and \ "Profit" in filename if get_ped(filename) < workdate] valuationfiles = [filename for filename in files if "csv" in filename and \ "Valuation" in filename if get_ped(filename) < workdate] pnlfile = sorted(pnlfiles, key=key_fun, reverse=True)[0] valuationfile = sorted(valuationfiles, key=key_fun, reverse=True)[0] if pnlfile: if not os.path.exists(os.path.join(root, str(workdate), "Reports")): os.makedirs(os.path.join(root, str(workdate), "Reports")) for filename in [pnlfile, valuationfile]: with open(os.path.join(root, str(workdate), "Reports", filename), "wb") as fh: ftp.retrbinary('RETR ' + filename, fh.write) logging.info("downloaded {0}".format(filename)) if os.name=='nt': gpg = gnupg.GPG(gpgbinary = r'"c:\\Program Files (x86)\\GNU\\GnuPG\\gpg2.exe"', gnupghome = os.path.join(os.getenv('APPDATA'), "gnupg")) elif os.name == 'posix': gpg = gnupg.GPG(gnupghome = '/home/guillaume/.gnupg') gpg.encoding = 'utf8' for filename in [pnlfile, valuationfile]: if "Profit" in filename: newfilename = "Pnl.csv" else: newfilename = "Valuation_Report.csv" with open(os.path.join(root, str(workdate), "Reports", filename), "rb") as fh: gpg.decrypt_file(fh, output = os.path.join(root, str(workdate), "Reports", newfilename), passphrase=config.key_password) os.remove(os.path.join(root, str(workdate), "Reports", filename)) def upload_data(startdate): for i in range(10): workdate = startdate - datetime.timedelta(days=i) workdatestr = str(workdate) try: filelist = [(f, os.stat(os.path.join(root, workdatestr, f)).st_ctime) \ for f in os.listdir(os.path.join(root, workdatestr)) if f.startswith("securitiesNpv")] except OSError: continue filelist = sorted(filelist, key = lambda x: x[1], reverse = True) if filelist: file_to_upload = filelist[0][0] newfile_to_upload = file_to_upload if workdate < startdate: newfile_to_upload = "securitiesNpv{0}.csv".format( datetime.datetime.strftime(datetime.datetime.today(), "%Y%m%d_%H%M%S")) # due to the way the drive is mounted, we get an exception when copy # tries to change permissions try: shutil.copy(os.path.join(root, workdatestr, file_to_upload), os.path.join(root, str(startdate), newfile_to_upload)) except OSError: pass logging.info("moved file from {0}".format(workdatestr)) ftp = FTP('ftp.globeop.com') ftp.login('srntsftp', config.ftp_password) ftp.cwd('incoming') with open(os.path.join(root, str(startdate), newfile_to_upload), "rb") as fh: ftp.storbinary('STOR ' + newfile_to_upload, fh) break logging.info("upload done") if __name__=="__main__": logging.basicConfig(filename='/home/share/CorpCDOs/logs/globeop.log', level=logging.INFO, format='%(asctime)s %(message)s') def date_from_string(s): return datetime.datetime.strptime(s, "%Y-%m-%d").date() parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=True) ## options are technically not exclusive, but we will be running them ## at different times of the day group.add_argument("-d", "--download", action="store_true", help="download reports from GlobeOp") group.add_argument("-u", "--upload", action="store_true", help="upload marks to GlobeOp") parser.add_argument("date", nargs='?', type=date_from_string, default=datetime.date.today()) args = parser.parse_args() if args.download: download_data(args.date) elif args.upload: upload_data(args.date)