diff options
Diffstat (limited to 'python/task_server/globeop.py')
| -rw-r--r-- | python/task_server/globeop.py | 46 |
1 files changed, 13 insertions, 33 deletions
diff --git a/python/task_server/globeop.py b/python/task_server/globeop.py index 295509c0..cb7b0404 100644 --- a/python/task_server/globeop.py +++ b/python/task_server/globeop.py @@ -8,6 +8,7 @@ import logging import argparse
import shutil
import pandas as pd
+from sqlalchemy import create_engine
try:
import pandas as pd
@@ -106,45 +107,23 @@ def download_data(workdate): df.to_csv(os.path.join(reports_dir, "CDS_Report.csv"), index=False)
os.remove(os.path.join(reports_dir, "CDS_Report.xls"))
-def upload_data(startdate):
- for i in range(10):
- workdate = startdate - datetime.timedelta(days=i)
- workdatestr = str(workdate)
- try:
- filelist = [(f, os.stat(os.path.join(root, workdatestr, f)).st_ctime) \
- for f in os.listdir(os.path.join(root, workdatestr)) if f.startswith("securitiesNpv")]
- except OSError:
- continue
-
- filelist = sorted(filelist, key = lambda x: x[1], reverse = True)
- if filelist:
- file_to_upload = filelist[0][0]
- newfile_to_upload = file_to_upload
- if workdate < startdate:
- newfile_to_upload = "securitiesNpv{0:%Y%m%d_%H%M%S}.csv".\
- format(datetime.datetime.today())
- # due to the way the drive is mounted, we get an exception when copy
- # tries to change permissions
- try:
- shutil.copy(os.path.join(root, workdatestr, file_to_upload),
- os.path.join(root, str(startdate), newfile_to_upload))
- except OSError:
- pass
- logging.info("moved file from {0}".format(workdatestr))
- ftp = FTP('ftp.globeop.com')
- ftp.login('srntsftp', config.ftp_password)
- ftp.cwd('incoming')
- with open(os.path.join(root, str(startdate), newfile_to_upload), "rb") as fh:
- ftp.storbinary('STOR ' + newfile_to_upload, fh)
- break
+def upload_data(workdate):
+ df = pd.read_sql_query("SELECT * from list_marks(%s)", engine, params = (workdate.date(),))
+ df.rename(columns = {'identifier': 'IDENTIFIER',
+ 'price': 'Price'}, inplace=True)
+ filename = 'securitiesNpv{0:%Y%m%d_%H%M%S}.csv'.format(workdate)
+ fullpath = os.path.join(root, str(workdate.date()), filename)
+ df.to_csv(fullpath, index=False)
+ ftp = get_ftp('incomping')
+ with open(fullpath, "rb") as fh:
+ ftp.storbinary('STOR ' + filename, fh)
logging.info("upload done")
if __name__=="__main__":
logging.basicConfig(filename='/home/share/CorpCDOs/logs/globeop.log',
level=logging.INFO,
format='%(asctime)s %(message)s')
- def date_from_string(s):
- return datetime.datetime.strptime(s, "%Y-%m-%d").date()
+
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
## options are technically not exclusive, but we will be running them
@@ -160,4 +139,5 @@ if __name__=="__main__": if args.download:
download_data(args.date)
elif args.upload:
+ engine = create_engine('postgresql://dawn_user@debian/dawndb')
upload_data(args.date)
|