import re from selenium import webdriver from selenium.webdriver.firefox.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By import logging import datetime from serenitas.utils.env import DAILY_DIR from serenitas.analytics.dates import prev_business_day logger = logging.getLogger(__name__) def wait(driver): WebDriverWait(driver, 10).until( lambda driver: driver.execute_script("return document.readyState") == "complete" ) def download_report(account_username, password, report_dir, workdate): driver = create_driver(report_dir) login(driver, account_username, password) navigate_to_inbox(driver, workdate) attachment_element = get_attachment_element(driver) attachment_element.click() fname = attachment_element.text.split()[0] driver.quit() return fname def login(driver, account_username, password): driver.get("https://web1.mail.scotiabank.com/webmail/do/Home") wait(driver) username_element = driver.find_element_by_id("username") username_element.send_keys(account_username) password_element = driver.find_element_by_id("passwordid") password_element.send_keys(password) login_button = driver.find_element_by_id("loginbutton") login_button.click() wait(driver) def navigate_to_inbox(driver, workdate): wait = WebDriverWait(driver, 10) login_button = wait.until( EC.presence_of_element_located((By.ID, "leftNavFolderLink")) ) login_button.click() inbox_link = wait.until( EC.presence_of_element_located((By.ID, "leftNavInboxFolderLink")) ) inbox_link.click() wait.until(EC.presence_of_element_located((By.CLASS_NAME, "zfolder-msg"))) for msg in driver.find_elements_by_class_name("zfolder-msg"): date_string = re.search( r"\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{4}", msg.text ).group() date = datetime.datetime.strptime(date_string, "%Y-%m-%d").date() if date == workdate: msg.click() return raise ValueError(f"Could not find a file for this date {workdate}") def get_attachment_element(driver): wait = WebDriverWait(driver, 10) attach = wait.until( EC.presence_of_element_located((By.CLASS_NAME, "zmessage-attachment-link")) ) return driver.find_elements_by_class_name("zmessage-attachment-link")[0] def create_driver(download_dir): options = Options() options.add_argument("--headless") fp = webdriver.FirefoxProfile() fp.set_preference("browser.download.dir", str(download_dir)) fp.set_preference("browser.download.folderList", 2) fp.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/octet-stream" ) return webdriver.Firefox(firefox_profile=fp, options=options) def download_scotia_report(workdate): scotia_login = {"selene-ops@lmcg.com": "oeujG*UF!53o"} for username, password in scotia_login.items(): REPORT_DIR = DAILY_DIR / "Selene" / "Scotia_reports" try: fname = next( REPORT_DIR.glob( f"IsoSelene_{prev_business_day(workdate):%d-%b-%Y}_*_xlsx.JOAAPKO3.JOAAPKO1" ) ) logger.info(f"{fname} already exists in {REPORT_DIR}") except StopIteration: # File doesn't exist, let's get it" fname = download_report(username, password, REPORT_DIR, workdate) logger.info(f"Downloaded Scotia Report for {workdate}: {fname}")