diff options
Diffstat (limited to 'python/collateral_calc.py')
| -rw-r--r-- | python/collateral_calc.py | 39 |
1 files changed, 36 insertions, 3 deletions
diff --git a/python/collateral_calc.py b/python/collateral_calc.py index db58a2ff..a3efd523 100644 --- a/python/collateral_calc.py +++ b/python/collateral_calc.py @@ -1,5 +1,6 @@ import logging import pandas as pd +import socket import sys from db import dawn_engine @@ -15,6 +16,11 @@ from pandas.tseries.offsets import BDay from paramiko import Transport, SFTPClient, RSAKey from parse_citi_pdf import get_citi_collateral from sqlalchemy.exc import IntegrityError +from ssh2.session import Session +from ssh2.utils import wait_socket +from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN +from ssh2.sftp import (LIBSSH2_FXF_READ, LIBSSH2_SFTP_S_IRUSR, + LIBSSH2_SFTP_S_IFDIR, LIBSSH2_SFTP_S_IFREG) logging.basicConfig(filename=LOG_DIR / 'collateral_calc.log', level=logging.WARNING, @@ -36,6 +42,33 @@ def get_wells_sftp_client(): transport.connect(username='LMCHsWC6EP', password='0lvK+7xL') return SFTPClient.from_transport(transport) +def get_wells_sftp_client2(): + start = timer() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('axst.wellsfargo.com', 10022)) + session = Session() + session.handshake(sock) + session.userauth_password('LMCHsWC6EP', '0lvK+7xL') + sftp = session.sftp_init() + return sftp + +def download_wells_files2(d=None): + DATA_DIR = DAILY_DIR / "Wells_reports" + sftp = get_wells_sftp_client2() + files = [] + with sftp.opendir("/RECEIVE/339425_DATO2") as fh: + for size, buf, attrs in fh.readdir(): + if attrs.permissions & LIBSSH2_SFTP_S_IFREG: + files.append(buf.decode()) + for f in files: + local_file = DATA_DIR / f + if not local_file.exists(): + with sftp.open(f"/RECEIVE/339425_DATO2/{f}", + LIBSSH2_FXF_READ, LIBSSH2_SFTP_S_IRUSR) as remote_handle, + local_file.open("wb") as local_handle: + for size, data in remote_handle: + local_handle.write(data) + def download_baml_files(d=None): DATA_DIR = DAILY_DIR / "BAML_reports" sftp = get_baml_sftp_client() @@ -47,7 +80,8 @@ def download_wells_files(d=None): sftp = get_wells_sftp_client() base_dir = "/RECEIVE/339425_DATO2" for f in sftp.listdir(base_dir): - sftp.get(f"{base_dir}/{f}", localpath=DATA_DIR / f) + if not (DATA_DIR / f).exists(): + sftp.get(f"{base_dir}/{f}", localpath=DATA_DIR / f) def download_sftp_files(d=None, @@ -446,10 +480,9 @@ if __name__ == "__main__": download_ms_emails() download_gs_emails() download_citi_emails() - #download_sftp_files(d) download_baml_files() - download_wells_files() + download_wells_files2() dawn_trades = get_dawn_trades() d = (pd.Timestamp.today() - BDay()).normalize() try: |