diff options
Diffstat (limited to 'python/process_queue.py')
| -rw-r--r-- | python/process_queue.py | 44 |
1 files changed, 10 insertions, 34 deletions
diff --git a/python/process_queue.py b/python/process_queue.py index a588cea8..42e666a3 100644 --- a/python/process_queue.py +++ b/python/process_queue.py @@ -2,7 +2,6 @@ import argparse import blpapi import csv import datetime -import json import logging import psycopg2 import pathlib @@ -15,14 +14,12 @@ from io import StringIO from analytics import CreditIndex from analytics.utils import bus_day from itertools import groupby -from pathlib import Path from pickle import loads -from ftplib import FTP from bbg_helpers import init_bbg_session, retrieve_data, BBG_IP from common import get_redis_queue from functools import partial -from paramiko import Transport, SFTPClient from pyisda.date import previous_twentieth +from remote import FtpClient, SftpClient from utils.db import dbconn from quantlib.time.api import pydate_from_qldate, UnitedStates, Days, Date from gmail_helpers import GmailMessage @@ -293,26 +290,6 @@ HEADERS = { } -def load_credentials(name): - return json.load((Path(".credentials") / f"{name}.json").open()) - - -def get_sftp_client(host, port, username, password, folder=None): - transport = Transport((host, port)) - transport.connect(username=username, password=password) - sftp = SFTPClient.from_transport(transport) - if folder is not None: - sftp.chdir(folder) - return sftp - - -def get_ftp_client(host, username, password, folder=None): - ftp = FTP(host, username, password) - if folder is not None: - ftp.cwd(folder) - return ftp - - def get_headers(trade_type, fund): headers = HEADERS[trade_type] if trade_type == "bond" and fund == "BOWDST": @@ -860,23 +837,22 @@ def get_filepath( return ( base_dir / str(timestamp.date()) - / f"Bowdst.ALL.{timestamp:%Y%m%d.%H%M%S}.{d[trade_type]}.csv" + / f"LMC01CFE.Bowdst.ALL.{timestamp:%Y%m%d.%H%M%S}.{d[trade_type]}.csv" ) def upload_file(file_path: pathlib.Path) -> None: if "BBH" in file_path.name: - return + sftp = SftpClient.from_creds("bbh") + sftp.put(file_path) elif file_path.name.startswith("Serenitas"): - ftp = get_ftp_client(**{**load_credentials("globeop"), "folder": "incoming"}) - cmd = f"STOR {file_path.name}" - with file_path.open("rb") as fh: - ftp.storbinary(cmd, fh) + ftp = FtpClient.from_creds("globeop") + ftp.client.cwd("incoming") + ftp.put(file_path) elif file_path.name.startswith("Bowdst"): - sftp = get_sftp_client(**load_credentials("bony")) - sftp.chdir("/inbound/cfe/") - with file_path.open("rb") as fh: - sftp.putfo(fh, f"LMC01CFE.{file_path.name}") + sftp = SftpClient.from_creds("bony") + sftp.client.chdir("/inbound/cfe/") + sftp.put(file_path) def write_buffer( |