0. Acquiring Speech and Transriptions.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:6ea03f61631ab4a99a774a93e47210b50b54119426e1f8184a269b8d53fde082"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from bs4 import BeautifulSoup\n",
      "import requests\n",
      "import time\n",
      "from selenium import webdriver\n",
      "import lxml.html\n",
      "import cPickle as pickle\n",
      "import os\n",
      "import urllib2\n",
      "from IPython.display import clear_output"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "url_base = 'https://librivox.org/search' + \\\n",
      "            '?title=&author=&reader=&keywords=&genre_id=0&status=complete&project_type=either&recorded_language=1&sort_order=catalog_date&search_form=advanced' +\\\n",
      "            '&search_page='"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "url_page = 1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Scrape the Search Pages"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def parse_search_page(source):\n",
      "    tree = lxml.html.fromstring(source)\n",
      "    elements = tree.find_class(\"catalog-result\")\n",
      "\n",
      "    # URLs of books to download\n",
      "\n",
      "    book_page_urls = []\n",
      "    for element in elements:\n",
      "        if 'title-icon' in lxml.html.tostring(element):\n",
      "            meta_data_html = lxml.html.tostring(element.find_class('book-meta')[0])\n",
      "\n",
      "            # only interested in Solo or Collaborative submissions at this point\n",
      "            #  Solo: lots of audio (hours) from one reader\n",
      "            #  Collaborative: a fair amount of data (minutes to a few hours) from multiple readers\n",
      "            if 'Solo' in meta_data_html or 'Collaborative' in meta_data_html:\n",
      "                #print meta_data_html\n",
      "\n",
      "                raw_book_html = lxml.html.tostring(element)\n",
      "\n",
      "\n",
      "                try:\n",
      "                    url = raw_book_html.split('<h3><a href=\"')[1].split('\">')[0]\n",
      "                    print url\n",
      "                    book_page_urls.append(url)\n",
      "                except IndexError:\n",
      "                    # These are results that are a part of a 'compilation'\n",
      "                    # The main compilation page will appear in the search results at some other point\n",
      "                    pass\n",
      "    return book_page_urls"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "target_pages = []"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "browser = webdriver.Firefox()\n",
      "\n",
      "url_page = 610\n",
      "\n",
      "while True:\n",
      "    # load page\n",
      "    browser.get(url_base + str(url_page))\n",
      "    \n",
      "    # Wait 7 seconds\n",
      "    time.sleep(7)\n",
      "\n",
      "    # save page_source\n",
      "    source = browser.page_source\n",
      "    if 'No results found' in  source:   # Hit end of results\n",
      "        break\n",
      "    \n",
      "    target_pages += parse_search_page(source)\n",
      "    \n",
      "    url_page += 1\n",
      "    \n",
      "\n",
      "# close when done\n",
      "browser.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 1,
       "text": [
        "\"browser = webdriver.Firefox()\\n\\nurl_page = 610\\n\\nwhile True:\\n    # load page\\n    browser.get(url_base + str(url_page))\\n    \\n    # Wait 7 seconds\\n    time.sleep(7)\\n\\n    # save page_source\\n    source = browser.page_source\\n    if 'No results found' in  source:   # Hit end of results\\n        break\\n    \\n    target_pages += parse_search_page(source)\\n    \\n    url_page += 1\\n    \\n\\n# close when done\\nbrowser.close()\""
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(target_pages)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "6294"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "unique_target_pages = []\n",
      "for page in target_pages:\n",
      "    if page not in unique_target_pages:\n",
      "        unique_target_pages.append(page)\n",
      "print len(unique_target_pages)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5339\n"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Save the Results"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#f = open('pages_to_scrape.pkl', 'wb')\n",
      "#pickle.dump(unique_target_pages, f)\n",
      "#f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5339\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Load Saved Results"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open('pages_to_scrape.pkl', 'rb')\n",
      "unique_target_pages = pickle.load(f)\n",
      "print len(unique_target_pages)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5339\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Scrape Individual Book Pages"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def parse_book_page(source):\n",
      "    tree = lxml.html.fromstring(source)\n",
      "    elements = tree.find_class('book-page-sidebar')\n",
      "    \n",
      "    if len(elements)==3:\n",
      "        download_section = lxml.html.tostring(elements[0])\n",
      "        online_text_section = lxml.html.tostring(elements[2])\n",
      "        \n",
      "        if '.zip' not in download_section or 'Online text' not in online_text_section:\n",
      "            return None\n",
      "        \n",
      "        \n",
      "        # Get download URL\n",
      "        tree2 = lxml.html.fromstring(download_section)\n",
      "        elements2 = tree2.find_class('book-download-btn')\n",
      "        if len(elements2) == 4:\n",
      "            download_url = download_section.split('<a class=\"book-download-btn\" href=\"')[1].split('\">Download')[0]\n",
      "            if not download_url.endswith('.zip'): return None\n",
      "            \n",
      "        \n",
      "        # Get online text URL (Gutenberg)\n",
      "        return download_url, online_text_section.split('\">Online text')[0].split('a href=\"')[-1]\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "book_urls_not_parsed = []\n",
      "audio_text_pairs = []"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "while len(book_urls_not_parsed)+len(audio_text_pairs)-1 < len(unique_target_pages)-1:\n",
      "    \n",
      "    count = 0\n",
      "    \n",
      "    browser = webdriver.Firefox()\n",
      "\n",
      "    for url in unique_target_pages[len(book_urls_not_parsed)+len(audio_text_pairs)-1:]:\n",
      "        browser.get(url)\n",
      "\n",
      "        # Wait 7 seconds\n",
      "        time.sleep(7)\n",
      "\n",
      "        # save page_source\n",
      "        source = browser.page_source\n",
      "\n",
      "        result = parse_book_page(source)\n",
      "        print result\n",
      "        if result is None:\n",
      "            book_urls_not_parsed.append(url)\n",
      "        else:\n",
      "            audio_text_pairs.append(result)\n",
      "            \n",
      "        count += 1\n",
      "        if count == 50:\n",
      "            break\n",
      "\n",
      "    # close when done\n",
      "    browser.close()\n",
      "    \n",
      "    print len(audio_text_pairs)\n",
      "    print len(book_urls_not_parsed)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Save the Results"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#f = open('download_urls.pkl', 'wb')\n",
      "#pickle.dump(audio_text_pairs, f)\n",
      "#f.close()\n",
      "\n",
      "#f = open('poorly_formatted_books.pkl', 'wb')\n",
      "#pickle.dump(book_urls_not_parsed, f)\n",
      "#f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 76
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Load the Saved Results"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open('download_urls.pkl', 'rb')\n",
      "audio_text_pairs = pickle.load(f)\n",
      "f.close()\n",
      "\n",
      "f = open('poorly_formatted_books.pkl', 'rb')\n",
      "book_urls_not_parsed = pickle.load(f)\n",
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Download Books and Transcriptions (From Gutenberg Mirror)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "download_directory = '/media/pat/sda3/speech_downloaded/'\n",
      "gutenberg_mirror = 'http://gutenberg.pglaf.org/'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print audio_text_pairs[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "('http://www.archive.org/download//eugenie_grandet_1504_librivox/eugenie_grandet_1504_librivox_64kb_mp3.zip', 'http://www.gutenberg.org/ebooks/1715')\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Get total size of files to be downloaded (for curiosities sake)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "gutenberg_count = 0\n",
      "total_size_bytes = 0\n",
      "for url, text_url in audio_text_pairs:\n",
      "\n",
      "    try:\n",
      "        if 'gutenberg.org/ebooks/' in text_url:\n",
      "            u = urllib2.urlopen(url)\n",
      "            meta = u.info()\n",
      "            total_size_bytes += int(meta.getheaders(\"Content-Length\")[0])\n",
      "            gutenberg_count +=1\n",
      "\n",
      "            if gutenberg_count % 1 == 0:\n",
      "                print total_size_bytes/1024./1024./1024.\n",
      "            #print \"Downloading: %s Bytes: %s\" % (file_name, file_size)\n",
      "    except urllib2.URLError:\n",
      "        pass\n",
      "print gutenberg_count, 'books'\n",
      "print total_size_bytes/1024./1024./1024., 'GB'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Download"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def get_text_mirror_url(text_url):\n",
      "    text_url_download = gutenberg_mirror\n",
      "    book_id = text_url.split('/')[-1]\n",
      "    for i in xrange(len(book_id)-1):\n",
      "        text_url_download += book_id[i] + '/'\n",
      "        #print book_id[i]\n",
      "    return text_url_download + book_id + '/' + book_id + '.txt'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#successfully_downloaded = []\n",
      "f = open('successfully_downloaded.pkl', 'rb')\n",
      "successfully_downloaded = pickle.load(f)\n",
      "f.close()\n",
      "\n",
      "for url, text_url in audio_text_pairs:\n",
      "    if not (url, text_url) in successfully_downloaded:\n",
      "        try:\n",
      "            if 'gutenberg.org/ebooks/' in text_url:\n",
      "\n",
      "                # download audiobook\n",
      "                file_name = url.split('/')[-1]\n",
      "                u = urllib2.urlopen(url)\n",
      "                f = open(download_directory + file_name, 'wb')\n",
      "                meta = u.info()\n",
      "                file_size = int(meta.getheaders(\"Content-Length\")[0])\n",
      "                print \"Downloading: %s Bytes: %s\" % (file_name, file_size)\n",
      "\n",
      "                file_size_dl = 0\n",
      "                block_sz = 8192*2*2*2*2\n",
      "                while True:\n",
      "                    buffer = u.read(block_sz)\n",
      "                    if not buffer:\n",
      "                        break\n",
      "\n",
      "                    file_size_dl += len(buffer)\n",
      "                    f.write(buffer)\n",
      "                    status = r\"%s   [%3.2f%%] of %3.2fMB\" % (file_name, file_size_dl * 100. / file_size, file_size/1024./1024.)\n",
      "                    status = status + chr(8)*(len(status)+1)\n",
      "                    clear_output(True)\n",
      "                    print status\n",
      "                f.close()\n",
      "\n",
      "\n",
      "\n",
      "                # download corresponding text (from gutenberg mirror)\n",
      "                text_download_url = get_text_mirror_url(text_url)\n",
      "                text_file_name = file_name.replace('.zip', '.txt')\n",
      "                u = urllib2.urlopen(text_download_url)\n",
      "                f = open(download_directory + text_file_name, 'wb')\n",
      "                meta = u.info()\n",
      "                file_size = int(meta.getheaders(\"Content-Length\")[0])\n",
      "                print \"Downloading: %s Bytes: %s\" % (text_file_name, file_size)\n",
      "\n",
      "                file_size_dl = 0\n",
      "                block_sz = 8192*2*2\n",
      "                while True:\n",
      "                    buffer = u.read(block_sz)\n",
      "                    if not buffer:\n",
      "                        break\n",
      "\n",
      "                    file_size_dl += len(buffer)\n",
      "                    f.write(buffer)\n",
      "                    status = r\"%s   [%3.2f%%] of %3.2fMB\" % (text_file_name, file_size_dl * 100. / file_size, file_size/1024./1024.)\n",
      "                    status = status + chr(8)*(len(status)+1)\n",
      "                    clear_output(True)\n",
      "                    print status\n",
      "                f.close()\n",
      "\n",
      "                # must not have been any errors, lets save it\n",
      "                successfully_downloaded.append((url, text_url))\n",
      "\n",
      "\n",
      "        except urllib2.URLError:\n",
      "            pass\n",
      "    \n",
      "f = open('successfully_downloaded.pkl', 'wb')\n",
      "pickle.dump(successfully_downloaded, f)\n",
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "charlie_to_the_rescue_1503_librivox_64kb_mp3.zip   [1.75%] of 243.16MB\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\n"
       ]
      },
      {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-51-2cbf7cb3175c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     20\u001b[0m                 \u001b[0mblock_sz\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m8192\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m                 \u001b[1;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m                     \u001b[0mbuffer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mu\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblock_sz\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m                         \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/socket.pyc\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, size)\u001b[0m\n\u001b[0;32m    378\u001b[0m                 \u001b[1;31m# fragmentation issues on many platforms.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    379\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 380\u001b[1;33m                     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    381\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    382\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mEINTR\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/httplib.pyc\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, amt)\u001b[0m\n\u001b[0;32m    565\u001b[0m         \u001b[1;31m# connection, and the user is reading more bytes than will be provided\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m         \u001b[1;31m# (for example, reading in 1k chunks)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 567\u001b[1;33m         \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    568\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    569\u001b[0m             \u001b[1;31m# Ideally, we would raise IncompleteRead if the content-length\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/socket.pyc\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, size)\u001b[0m\n\u001b[0;32m    378\u001b[0m                 \u001b[1;31m# fragmentation issues on many platforms.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    379\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 380\u001b[1;33m                     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    381\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    382\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mEINTR\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open('successfully_downloaded.pkl', 'wb')\n",
      "pickle.dump(successfully_downloaded, f)\n",
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 52
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "x = np.arange(20)\n",
      "print x"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19]\n"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print x[0::4]\n",
      "print x[1::4]\n",
      "print x[2::4]\n",
      "print x[3::4]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 0  4  8 12 16]\n",
        "[ 1  5  9 13 17]\n",
        "[ 2  6 10 14 18]\n",
        "[ 3  7 11 15 19]\n"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}