1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
import logging
import datetime
from contextlib import contextmanager
import re
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from serenitas.utils.env import DAILY_DIR
from serenitas.analytics.dates import prev_business_day
logger = logging.getLogger(__name__)
@contextmanager
def quit_driver(driver):
try:
yield driver
finally:
driver.quit()
def download_report(account_username, password, report_dir, workdate):
with quit_driver(create_driver(report_dir)) as driver:
login(driver, account_username, password)
navigate_to_inbox(driver, workdate)
attachment_element = get_attachment_element(driver)
attachment_element.click()
fname = attachment_element.text.split()[0]
return fname
def login(driver, account_username, password):
driver.get("https://web1.mail.scotiabank.com/webmail/do/Home")
wait = WebDriverWait(driver, 10)
username_element = wait.until(EC.presence_of_element_located((By.ID, "username")))
username_element.send_keys(account_username)
password_element = wait.until(EC.presence_of_element_located((By.ID, "passwordid")))
password_element.send_keys(password)
login_button = driver.find_element(By.ID, "loginbutton")
login_button.click()
def navigate_to_inbox(driver, workdate):
wait = WebDriverWait(driver, 10)
login_button = wait.until(
EC.presence_of_element_located((By.ID, "leftNavFolderLink"))
)
login_button.click()
inbox_link = wait.until(
EC.presence_of_element_located((By.ID, "leftNavInboxFolderLink"))
)
inbox_link.click()
wait.until(EC.presence_of_element_located((By.CLASS_NAME, "zfolder-msg")))
for msg in driver.find_elements(By.CLASS_NAME, "zfolder-msg"):
date_string = re.search(
r"\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{4}", msg.text
).group()
date = datetime.datetime.strptime(date_string, "%Y-%m-%d").date()
if date == workdate:
msg.click()
return
raise ValueError(f"Could not find a file for this date {workdate}")
def get_attachment_element(driver):
wait = WebDriverWait(driver, 10)
attach = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, "zmessage-attachment-link"))
)
return driver.find_element(By.CLASS_NAME, "zmessage-attachment-link")
def create_driver(download_dir):
options = Options()
options.add_argument("--headless")
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.dir", str(download_dir))
fp.set_preference("browser.download.folderList", 2)
fp.set_preference(
"browser.helperApps.neverAsk.saveToDisk", "application/octet-stream"
)
return webdriver.Firefox(
firefox_profile=fp,
options=options,
service_log_path="/home/serenitas/CorpCDOs/logs/selenium_scotia.log",
)
def download_scotia_report(workdate):
scotia_login = {"selene-ops@lmcg.com": "oeujG*UF!53o"}
for username, password in scotia_login.items():
REPORT_DIR = DAILY_DIR / "Selene" / "Scotia_reports"
try:
fname = next(
REPORT_DIR.glob(
f"IsoSelene_{prev_business_day(workdate):%d-%b-%Y}_*_xlsx.JOAAPKO3.JOAAPKO1"
)
)
logger.info(f"{fname} already exists in {REPORT_DIR}")
except StopIteration: # File doesn't exist, let's get it"
fname = download_report(username, password, REPORT_DIR, workdate)
logger.info(f"Downloaded Scotia Report for {workdate}: {fname}")
|
