diff options
Diffstat (limited to 'python/task_server/globeop.py')
| -rw-r--r-- | python/task_server/globeop.py | 49 |
1 files changed, 24 insertions, 25 deletions
diff --git a/python/task_server/globeop.py b/python/task_server/globeop.py index d0ffac78..c49d240c 100644 --- a/python/task_server/globeop.py +++ b/python/task_server/globeop.py @@ -1,10 +1,13 @@ import datetime +import json import os -import os.path +import pandas as pd from . import DAILY_DIR -from ftplib import FTP +from pathlib import Path + +from serenitas.utils.remote import FtpClient +from serenitas.analytics.utils import prev_business_day import gnupg -from . import config import re import logging import sys @@ -14,19 +17,14 @@ import load_globeop_report logger = logging.getLogger(__name__) -try: - import pandas as pd - from pandas.tseries.offsets import BDay -except ImportError: - pass - def get_ped(s): if m := re.search("PED=([^.]+)", s): PED = datetime.date.fromisoformat(m.group(1)) elif m := re.search("([^.]+)", s): - PED = pd.to_datetime(m.group(1), format="%Y%m%d") - BDay(1) - PED = PED.date() + PED = prev_business_day( + datetime.datetime.strptime(m.group(1), format="%Y%m%d").date() + ) else: raise ValueError return PED @@ -52,9 +50,8 @@ def run_date(s): def get_ftp(folder): - ftp = FTP("ftp.globeop.com") - ftp.login("srntsftp", config.ftp_password) - ftp.cwd(folder) + ftp = FtpClient.from_creds("globeop") + ftp.client.cwd(folder) return ftp @@ -65,7 +62,7 @@ def get_gpg(): gnupghome=os.path.join(os.getenv("APPDATA"), "gnupg"), ) elif os.name == "posix": - gpg = gnupg.GPG(gnupghome=os.path.join(os.environ["HOME"], ".gnupg")) + gpg = gnupg.GPG(gnupghome=Path.home() / ".gnupg") gpg.encoding = "utf8" return gpg @@ -80,8 +77,9 @@ def convert_to_csv(f): def download_data(engine, workdate: datetime.date): - ftp = get_ftp("outgoing") - files = ftp.nlst() + ftp = FtpClient.from_creds("globeop") + ftp.client.cwd("outgoing") + files = ftp.client.nslt() pnlfiles = [ filename for filename in files @@ -118,7 +116,7 @@ def download_data(engine, workdate: datetime.date): for filename in available_files: with (reports_dir / filename).open("wb") as fh: - ftp.retrbinary("RETR " + filename, fh.write) + ftp.client.retrbinary("RETR " + filename, fh.write) logger.info(f"downloaded {filename}") gpg = get_gpg() @@ -130,10 +128,11 @@ def download_data(engine, workdate: datetime.date): else: newfilename = "CDS_Report.xls" with (reports_dir / filename).open("rb") as fh: + creds = json.load((Path.home() / ".credentials" / "gpg-key.json").open()) dec = gpg.decrypt_file( fh, output=(reports_dir / newfilename).as_posix(), - passphrase=config.key_password, + passphrase=creds["password"], always_trust=True, ) logger.info(f"{filename}: {dec.status}") @@ -183,9 +182,9 @@ def upload_bond_marks(engine, workdate: datetime.datetime): df.rename(columns={"identifier": "IDENTIFIER", "price": "Price"}, inplace=True) fullpath = DAILY_DIR / str(d) / f"securitiesNpv{workdate:%Y%m%d_%H%M%S}.csv" df.to_csv(fullpath, index=False) - ftp = get_ftp("incoming") - with fullpath.open("rb") as fh: - ftp.storbinary("STOR " + fullpath.name, fh) + ftp = FtpClient.from_creds("globeop") + ftp.client.cwd("incoming") + ftp.put(fullpath) logger.info("upload bond marks done") @@ -200,9 +199,9 @@ JOIN cds USING (security_id)""", ) fullpath = DAILY_DIR / str(d) / f"otcNpv{workdate:%Y%m%d}.csv" df.to_csv(fullpath, index=False) - ftp = get_ftp("incoming") - with fullpath.open("rb") as fh: - ftp.storbinary("STOR " + fullpath.name, fh) + ftp = FtpClient.from_creds("globeop") + ftp.client.cwd("incoming") + ftp.puth(fullpath) logger.info("upload cds marks done") |