donor_scraper/donor_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.opensecrets.org/indivs/search.php?name=Yoon&cycle=All&sort=R&state=&zip=&employ=&cand=&submit=%EC%A7%88%EC%9D%98+%EB%B3%B4%EB%82%B4%EA%B8%B0\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=2\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=2\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=3\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=3\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=4\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=4\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=5\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=5\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=6\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=6\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=7\n",
      "Next \n",
      "Is end page? False\n",
      "http://www.opensecrets.org/indivs/search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=7\n",
      "search.php?&name=Yoon&employ=&cand=&state=&cycle=All&soft=&zip=&sort=R&page=8\n",
      "Next \n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from lxml import etree\n",
    "from StringIO import StringIO\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "import csv\n",
    "import os\n",
    "\n",
    "def init_driver():\n",
    "    driver = webdriver.Firefox()\n",
    "    driver.wait = WebDriverWait(driver,1)\n",
    "    return driver\n",
    "                                \n",
    "def lookup(driver, query):\n",
    "    driver.get(\"http://www.opensecrets.org/indivs/\")\n",
    "    try:\n",
    "        button = driver.wait.until(EC.element_to_be_clickable(\n",
    "            (By.ID, \"name\")))  #find donor search box\n",
    "        button.click()           #Click input box\n",
    "        _input = driver.wait.until(EC.element_to_be_clickable(\n",
    "                (By.ID, \"name\")))    #Iput text saving space\n",
    "        _input.send_keys(query)  #Send \"Yoon\" text\n",
    "        _id = driver.wait.until(EC.element_to_be_clickable(\n",
    "                (By.NAME, \"submit\"))) #Find search botton\n",
    "        _id.click()                     #Click saerch botton\n",
    "    except TimeoutException:     #Error handling\n",
    "        print(\"Box or Button not found in google.com\")\n",
    "    \n",
    "def updateDriver(driver,root):\n",
    "    isEnd = True\n",
    "    for child in root:\n",
    "        url = child.xpath(\"@href\")\n",
    "        if len(url) == 1:\n",
    "           isEnd = True\n",
    "           text = child.text.strip()\n",
    "           if text.strip() == \"Next\":\n",
    "             isEnd = False\n",
    "             print url[0]\n",
    "             print child.text\n",
    "             url = \"http://www.opensecrets.org/indivs/\"+url[0]\n",
    "             driver.get(url)  \n",
    "    return isEnd\n",
    "\n",
    "def getXML(driver):\n",
    "    parser = etree.HTMLParser()\n",
    "    try:\n",
    "        html = driver.execute_script(\"return document.documentElement.outerHTML\")\n",
    "        tree = etree.parse(StringIO(html), parser)\n",
    "        root = tree.find(\"//*[@class='pageCtrl']\")\n",
    "    except NoSuchElementException:\n",
    "        driver.quit()\n",
    "        print \"Name not found\"\n",
    "        sys.exit(0)\n",
    "    return root\n",
    "\n",
    "def scrap(driver):\n",
    "    driver.current_url  #Getting current url\n",
    "    data = []           #Container for table data\n",
    "    for tr in driver.find_elements_by_xpath('//table[@id=\"top\"]//tr'): #loop table id top\n",
    "        tds = tr.find_elements_by_tag_name('td')\n",
    "        if tds:\n",
    "            data.append([td.text for td in tds])\n",
    "    return data\n",
    "\n",
    "def iter_scrap(driver):\n",
    "    container = []\n",
    "    endPage = False\n",
    "    while not endPage:\n",
    "        root = getXML(driver)\n",
    "        print driver.current_url\n",
    "        container.append(scrap(driver))\n",
    "        endPage = updateDriver(driver,root)\n",
    "        print 'Is end page? %s' %endPage\n",
    "    return container\n",
    "\n",
    "def flatten(xs):\n",
    "    result = []\n",
    "    if isinstance(xs, (list, tuple)):\n",
    "        for x in xs:\n",
    "            result.extend(flatten(x))\n",
    "    else:\n",
    "        result.append(xs)\n",
    "    return result\n",
    "\n",
    "def save_file(name,data):\n",
    "    save_root = \"./save\"\n",
    "    if not os.path.exists(save_root):\n",
    "        os.makedirs(save_root)\n",
    "    name = name+\".csv\"\n",
    "    csvfile = \"./save/\"+name\n",
    "\n",
    "    with open(csvfile, \"w\") as output:\n",
    "        for infos in data:\n",
    "            for info in infos:\n",
    "                info[0] = info[0].replace('\\n',' ')\n",
    "            writer = csv.writer(output, lineterminator='\\n')\n",
    "            writer.writerows(infos)\n",
    "    print 'Saved as'+name\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    driver = init_driver()\n",
    "    name = 'yark'\n",
    "    lookup(driver, name)\n",
    "    data = iter_scrap(driver)\n",
    "    driver.quit()\n",
    "    save_file(name,data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}