Activities/web_scraping_beautiful_soup_activity_00.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Web-Scraping-with-BeautifulSoup\" data-toc-modified-id=\"Web-Scraping-with-BeautifulSoup-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Web Scraping with BeautifulSoup</a></span></li><li><span><a href=\"#Activity---Adventure-Time-Transcripts\" data-toc-modified-id=\"Activity---Adventure-Time-Transcripts-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Activity - Adventure Time Transcripts</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping with BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are going to get some data that is a little dirty; we have to scrape up some delicious data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activity - Adventure Time Transcripts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transcripts: https://adventuretime.fandom.com/wiki/Category_talk%3ATranscripts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes web developers just don't write what we expect...\n",
    "\n",
    "![](images/tables_in_tables.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_html('http://www.espn.com/nhl/statistics/player/_/stat/points/sort/points/year/2015&#47;seasontype/2',skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_page_url = 'https://adventuretime.fandom.com/wiki/Slumber_Party_Panic/Transcript'\n",
    "response = requests.get(main_page_url)\n",
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_section = soup.find(id='mw-content-text')\n",
    "main_section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcript_lines = main_section.find_all('dl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Go through each line and pull dd\n",
    "for line in transcript_lines[:10]:\n",
    "    try:\n",
    "        dd_line = line.find('dde').text.strip()\n",
    "        print('newline:\\n\\t',dd_line)\n",
    "    except:\n",
    "        print('Something went wrong')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(1+'0')\n",
    "\n",
    "print('important info')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    print(1/0)#+'0')\n",
    "except TypeError:\n",
    "    print('there was an error for division')\n",
    "except ZeroDivisionError:\n",
    "    print('there was an error for ')    \n",
    "except:\n",
    "   print('some other error') \n",
    "\n",
    "    \n",
    "print('important info')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcript_lines = main_section.find_all('dl')\n",
    "\n",
    "lines_with_info = []\n",
    "for line in transcript_lines:\n",
    "    line_str = line.find('dd').text.strip()\n",
    "    \n",
    "    # Test if it's extra info (starts with '[') → 'extra_info'\n",
    "    if(line_str[0] == '['):\n",
    "        person = 'extra_info'\n",
    "    # If not, assume a person → that character's name\n",
    "    else:\n",
    "        split_info = line_str.split(':')\n",
    "        person = split_info[0]\n",
    "        line_str = split_info[1].strip()\n",
    "        \n",
    "    lines_with_info.append((person,line_str))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_arr = np.array(lines_with_info)\n",
    "\n",
    "list(n_arr[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(n_arr,columns=['speaking','spoken'])\n",
    "df.speaking.value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}