import logging import datetime from contextlib import contextmanager import re from selenium import webdriver from selenium.webdriver.firefox.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from serenitas.utils.env import DAILY_DIR from serenitas.analytics.dates import prev_business_day logger = logging.getLogger(__name__) def download_report(account_username, password, report_dir, workdate): with create_driver(report_dir) as driver: login(driver, account_username, password) navigate_to_inbox(driver, workdate) attachment_element = get_attachment_element(driver) attachment_element.click() fname = attachment_element.text.split()[0] return fname def login(driver, account_username, password): driver.get("https://web1.mail.scotiabank.com/webmail/do/Home") wait = WebDriverWait(driver, 10) username_element = wait.until(EC.presence_of_element_located((By.ID, "username"))) username_element.send_keys(account_username) password_element = wait.until(EC.presence_of_element_located((By.ID, "passwordid"))) password_element.send_keys(password) login_button = driver.find_element(By.ID, "loginbutton") login_button.click() def navigate_to_inbox(driver, workdate): wait = WebDriverWait(driver, 10) login_button = wait.until( EC.presence_of_element_located((By.ID, "leftNavFolderLink")) ) login_button.click() inbox_link = wait.until( EC.presence_of_element_located((By.ID, "leftNavInboxFolderLink")) ) inbox_link.click() wait.until(EC.presence_of_element_located((By.CLASS_NAME, "zfolder-msg"))) for msg in driver.find_elements(By.CLASS_NAME, "zfolder-msg"): date_string = re.search( r"\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{4}", msg.text ).group() date = datetime.datetime.strptime(date_string, "%Y-%m-%d").date() if date == workdate: msg.click() return raise ValueError(f"Could not find a file for this date {workdate}") def get_attachment_element(driver): wait = WebDriverWait(driver, 10) attach = wait.until( EC.presence_of_element_located((By.CLASS_NAME, "zmessage-attachment-link")) ) return driver.find_element(By.CLASS_NAME, "zmessage-attachment-link") @contextmanager def create_driver(download_dir): options = Options() options.add_argument("--headless") fp = webdriver.FirefoxProfile() fp.set_preference("browser.download.dir", str(download_dir)) fp.set_preference("browser.download.folderList", 2) fp.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/octet-stream" ) driver = webdriver.Firefox( firefox_profile=fp, options=options, service_log_path="/home/serenitas/CorpCDOs/logs/selenium_scotia.log", ) yield driver driver.quit() def download_scotia_report(workdate): username, password = ("selene-ops@lmcg.com", "oeujG*UF!53o") REPORT_DIR = DAILY_DIR / "Selene" / "Scotia_reports" try: fname = next( REPORT_DIR.glob( f"IsoSelene_{prev_business_day(workdate):%d-%b-%Y}_*_xlsx.JOAAPKO3.JOAAPKO1" ) ) logger.info(f"{fname} already exists in {REPORT_DIR}") except StopIteration: # File doesn't exist, let's get it" fname = download_report(username, password, REPORT_DIR, workdate) logger.info(f"Downloaded Scotia Report for {workdate}: {fname}")