scraping_mongo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Contents\n",
    "\n",
    "[Scraping](#scraping)<br>\n",
    "[MongoDB](#mongo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='scraping'></a>\n",
    "### Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "website = 'http://www.snackdata.com'\n",
    "html = requests.get(website)\n",
    "soup = BeautifulSoup(html.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#get list of snack urls\n",
    "\n",
    "#get single div that holds a list\n",
    "indexholder = soup.findAll(id='indexholder')\n",
    "#get all \"li\"s from the div\n",
    "lis = indexholder[0].findAll('li')\n",
    "urls = []\n",
    "for li in lis:\n",
    "    path = li.find('a')['href']\n",
    "    urls.append(website + path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.snackdata.com/corn_dog\n"
     ]
    }
   ],
   "source": [
    "print urls[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def parse_snack(url):\n",
    "    html = requests.get(url)\n",
    "    soup = BeautifulSoup(html.content, 'html.parser')\n",
    "    \n",
    "    rightstuff = soup.findAll(id='rightstuff')\n",
    "    name = rightstuff[0].findAll('h1')\n",
    "    \n",
    "    snack = dict()\n",
    "    snack['name'] = name[0].text\n",
    "    snack['desc'] = rightstuff[0].text\n",
    "    return snack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='mongo'></a>\n",
    "### MongoDB\n",
    "\n",
    "https://docs.mongodb.com/manual/reference/mongo-shell/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mc = pymongo.MongoClient()\n",
    "db = mc['snacks']\n",
    "collection = db['snack_data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in urls:\n",
    "    snack = parse_snack(url)\n",
    "    collection.insert_one(snack)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'_id': ObjectId('577ddcb690bae047aa9649e9'), u'name': u'Icing', u'desc': u\"\\nIcing\\n\\nIcing is a sweet spread that is usually spread upon other sweet foods (like cake or  cupcake). Sometimes it's called frosting, and sometimes it has food coloring added to it. The application of icing is best handled by a professional. If it's applied poorly, it can detract from the presentation of what its trying to enhance (thereby ruining the birthday or special event that it was intended for). \\nTasteIcing is sweet because it's made mostly of sugar. It can taste anywhere from terrible to good, depending on the quality of its ingredients. It usually makes a mess, and it's up to the eater to decide whether they like that or not. \\n\\nIcing became an official snack on April 2nd 2015.\\n\\n \\u2191 Index\\n\"}\n"
     ]
    }
   ],
   "source": [
    "print collection.find_one({'name': 'Icing'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    }
   ],
   "source": [
    "print collection.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mc.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}