1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
import selenium
from selenium.webdriver import Firefox, PhantomJS, FirefoxProfile
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException, ElementNotInteractableException
import time, os, shutil
from task_server.config import markit_login, markit_password
from task_server.insert_tranche_quotes import insert_quotes
import logging
import sys
logging.basicConfig(filename='/home/serenitas/CorpCDOs/logs/download_tranche_quotes.log',
level=logging.INFO,
format='%(asctime)s %(message)s')
##profile was created as so: firefox -CreateProfile "Selenium /home/guillaume/ffprofile"
profile = FirefoxProfile()
profile.set_preference('browser.download.folderList', 2) # custom location
profile.set_preference('browser.download.manager.showWhenStarting', False)
profile.set_preference('browser.download.dir', '/tmp')
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv')
driver = Firefox(profile)
driver.get("https://products.markit.com/")
time.sleep(1)
driver.find_element_by_id("username").send_keys(markit_login)
driver.find_element_by_id("password").send_keys(markit_password)
driver.find_element_by_id("submit").click()
try:
menu = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.ID,
"QUOTES.Quotes.Quotes")))
except TimeoutException:
logging.info("timeout after login")
driver.quit()
sys.exit()
else:
time.sleep(1)
menu.click()
hiddenmenu = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.ID,
"QUOTES.Quotes.Tranches")))
time.sleep(5)
hiddenmenu.click()
WebDriverWait(driver, 15).until(EC.frame_to_be_available_and_switch_to_it((By.NAME, "MFrame")))
try:
link = WebDriverWait(driver, 10).until(EC.element_to_be_clickable(
(By.CSS_SELECTOR,
".searchBg_bottom span:nth-child(1) table:nth-child(1) tbody:nth-child(1) " \
"tr:nth-child(1) td:nth-child(2) a:nth-child(1)")))
link.click()
except TimeoutException:
logging.info("timeout before being able to click")
driver.quit()
sys.exit()
total_wait = 0
while (not os.path.exists("/tmp/Quotes.csv") or total_wait==10):
time.sleep(1)
total_wait += 1
if total_wait==10:
logging.info("failed to download quotes")
else:
logging.info("download worked")
insert_quotes(quote_dir="/tmp")
driver.quit()
|
