from xvfbwrapper import Xvfb from selenium import webdriver from selenium.common.exceptions import ElementNotVisibleException,\ NoSuchElementException, StaleElementReferenceException from time import sleep from bs4 import BeautifulSoup from celery import Celery, Task from urllib2 import urlopen app = Celery('tasks', broker='amqp://guest@horel.org//') app.conf.CELERY_RESULT_BACKEND = 'rpc' app.conf.CELERY_ENABLE_UTC = True drivers = [None] def normalize(url): if "profile.php" in url: basename = url.split("&")[0] fname = basename.split("=")[-1] getname = basename + "&sk=friends" else: basename = url.split("?")[0] fname = basename.split("/")[-1] getname = basename + "/friends" return basename, fname, getname class ListFollowers(Task): @property def driver(self): if drivers[0] is None: uname, passwd = urlopen("http://horel.org:8080/").readline().strip().split() vdisplay = Xvfb() vdisplay.start() driver = webdriver.Chrome() driver.get("https://facebook.com") driver.find_element_by_id("email").send_keys(uname) elem = driver.find_element_by_id("pass") elem.send_keys(passwd) elem.submit() drivers[0] = driver return drivers[0] def run(self, url): self.driver.get(url) while True: for i in xrange(5): try: footer = self.driver.find_element_by_class_name("_359") except (NoSuchElementException, ElementNotVisibleException): sleep(0.1) else: break else: break try: footer.click() except StaleElementReferenceException: sleep(0.1) for i in xrange(5): try: div = self.driver.find_element_by_class_name("_30f") except NoSuchElementException: sleep(0.1) else: break else: return {"friends": [], "for": url} soup = BeautifulSoup(div.get_attribute("outerHTML")) return {"friends": [li.a["href"] for li in soup.findAll("li", class_="_698")], "for": url} class NumFollowers(Task): @property def driver(self): if drivers[0] is None: uname, passwd = urlopen("http://horel.org:8080/").readline().strip().split() vdisplay = Xvfb() vdisplay.start() driver = webdriver.Chrome() driver.get("https://facebook.com") driver.find_element_by_id("email").send_keys(uname) elem = driver.find_element_by_id("pass") elem.send_keys(passwd) elem.submit() drivers[0] = driver return drivers[0] def run(self, url): self.driver.get(url) for i in xrange(5): try: box = self.driver.find_element_by_class_name("_1f8g") except (NoSuchElementException, ElementNotVisibleException): sleep(0.1) else: break else: return {"nfriends": 0, "for": url} soup = BeautifulSoup(box.get_attribute("outerHTML")) a = soup.find("a", class_="uiLinkSubtle") return {"nfriends": int(a.string.replace(",", "")), "for": url}