BigDataPipeLine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\kadss\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\kadss\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\kadss\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import unicodedata\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import re\n",
    "import math\n",
    "import spotlight\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.tag import StanfordNERTagger\n",
    "from nltk.tokenize import word_tokenize\n",
    "from io import StringIO\n",
    "import os\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from collections import Counter\n",
    "from pprint import pprint\n",
    "import en_core_web_sm\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.metrics import accuracy_score, precision_score, classification_report, confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from dateparser.search import search_dates\n",
    "import datetime\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "java_path = \"C:\\\\Program Files\\\\Java\\\\jdk1.8.0_231\\\\bin\\\\java.exe\"\n",
    "os.environ['JAVAHOME'] = java_path\n",
    "\n",
    "nlp = en_core_web_sm.load()\n",
    "nltk.download('wordnet')\n",
    "nltk.download(\"stopwords\")\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file load, cleaning and unicode conversion\n",
    "\n",
    "file_name='Boston2C.csv'\n",
    "\n",
    "for_pd = StringIO()\n",
    "with open('Data/2CVTweets/'+file_name,encoding=\"utf8\") as f:\n",
    "    for line in f:\n",
    "        new_line = re.sub(r'NO;', 'NO',line)  ##cleaning 'NO'\n",
    "        print (new_line, file=for_pd)\n",
    "\n",
    "for_pd.seek(0)\n",
    "\n",
    "df_city = pd.read_csv(for_pd, sep=';', header=None)\n",
    "df_city = df_city.drop(df_city.columns[0], axis=1) ## index column\n",
    "df_city.columns=['tweet','Y']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>Y</th>\n",
       "      <th>tweet_org</th>\n",
       "      <th>tweet_tok_lem</th>\n",
       "      <th>no_ques_marks</th>\n",
       "      <th>no_excl_marks</th>\n",
       "      <th>no_uppercase</th>\n",
       "      <th>annotations</th>\n",
       "      <th>num_same_URI</th>\n",
       "      <th>acc_overlap_types</th>\n",
       "      <th>non_acc_overlap_types</th>\n",
       "      <th>num_locations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT' b'(Wellesley,' b'MA):' b'RT-9' b'East' b'near' b'Cedar' b'St' b'-' b'involving' b'a' b'motorcycle' b'and' b'a' b'car' b'-' b'possible' b'lane' b'restrictions'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT (Wellesley, MA): RT-9 East near Cedar St - involving a motorcycle and a car - possible lane restrictions</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b, ', (, Wellesley, ,, ', b'MA, ), :, ', b'RT-9, ', b'East, ', b'near, ', b'Cedar, ', b'St, ', b'-, ', b'involving, ', b, ', a, ', b'motorcycle, ', b'and, ', b, ', a, ', b'car, ', b'-, ', b'possible, ', b'lane, ', b'restrictions, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Master_of_Arts', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Motorcycle'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}</td>\n",
       "      <td>2</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'I-290' b'East' b'before' b'I-495' b'(Exit' b'#26)' b'-' b'2-car' b'accident' b'with' b'possible' b'lane' b'blockages'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT: I-290 East before I-495 (Exit #26) - 2-car accident with possible lane blockages</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b, ', I-290, ', b'East, ', b'before, ', b, ', I-495, ', b, ', (, Exit, ', b, ', #, 26, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'possible, ', b'lane, ', b'blockages, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Interstate_290_(Illinois)', 'http://dbpedia.org/resource/Falmouth_Spur'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b'Opened' b'report' b'via' b'iPhone' b'at' D b'Standard' b'St' b'URL</td>\n",
       "      <td>NO</td>\n",
       "      <td>Opened report via iPhone at 19 Standard St http://t.co/8vc761aCHT. Road kill on sidewalk.</td>\n",
       "      <td>[b'Opened, ', b'report, ', b'via, ', b'iPhone, ', b'at, ', D, b'Standard, ', b'St, ', b'URL]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/IPhone', 'http://dbpedia.org/resource/Twitter', 'http://dbpedia.org/resource/Roadkill', 'http://dbpedia.org/resource/Sidewalk'], 'types': ['Wikidata:Q1067263,DBpedia:InformationAppliance,DBpedia:Device', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', '', '']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.021739</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT' b'REPORTED:' b'I-93' b'North' b'before' b'Granite' b'Av' b'(Exit' b'#11)' b'-' b'police' b'investigating'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT REPORTED: I-93 North before Granite Av (Exit #11) - police investigating</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b'REPORTED, :, ', b, ', I-93, ', b'North, ', b'before, ', b'Granite, ', b'Av, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b'police, ', b'investigating, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Interstate_93'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b'Duduk' b'lebih' b'dari' D b'jam' b'sehari' b'berisiko' b'depresi' D b'persen' b'lebih' b'tinggi' b'dibandingkan' b'yang' b'hanya' b'duduk' D b'jam' b'sehari' b'[HuffingtonPost]'</td>\n",
       "      <td>NO</td>\n",
       "      <td>Duduk lebih dari 7 jam sehari berisiko depresi 47 persen lebih tinggi dibandingkan yang hanya duduk 4 jam sehari [HuffingtonPost]</td>\n",
       "      <td>[b'Duduk, ', b'lebih, ', b'dari, ', D, b'jam, ', b'sehari, ', b'berisiko, ', b'depresi, ', D, b'persen, ', b'lebih, ', b'tinggi, ', b'dibandingkan, ', b'yang, ', b'hanya, ', b'duduk, ', D, b'jam, ', b'sehari, ', b, ', [, HuffingtonPost, ], ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Duduk', 'http://dbpedia.org/resource/Dopamine_reuptake_inhibitor', 'http://dbpedia.org/resource/Duduk', 'http://dbpedia.org/resource/The_Huffington_Post'], 'types': ['', '', '', 'Wikidata:Q386724,Schema:WebPage,Schema:CreativeWork,DBpedia:Work,DBpedia:Website']}</td>\n",
       "      <td>2</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT' b'(Providence,' b'RI):' b'I-195' b'West' b'near' b'Gano' b'St' b'-' b'right' b'shoulder' b'blocked'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT (Providence, RI): I-195 West near Gano St - right shoulder blocked</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b, ', (, Providence, ,, ', b'RI, ), :, ', b, ', I-195, ', b'West, ', b'near, ', b'Gano, ', b'St, ', b'-, ', b'right, ', b'shoulder, ', b'blocked, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Providence_Friars_men's_basketball', 'http://dbpedia.org/resource/Rhode_Island', 'http://dbpedia.org/resource/Interstate_195_(Rhode_Island–Massachusetts)'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', 'Wikidata:Q3455524,Schema:Place,Schema:AdministrativeArea,DBpedia:Region,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:AdministrativeRegion', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.028986</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>b'Dedham' b'Police' b'Looking' b'for' b'Suspects' b'After' b'Car' b'Crashes' b'into' b'Verizon' b'Store' b'URL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Dedham Police Looking for Suspects After Car Crashes into Verizon Store http://t.co/u3zrVjliqB</td>\n",
       "      <td>[b'Dedham, ', b'Police, ', b'Looking, ', b'for, ', b'Suspects, ', b'After, ', b'Car, ', b'Crashes, ', b'into, ', b'Verizon, ', b'Store, ', b'URL]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Dedham_station', 'http://dbpedia.org/resource/Verizon_Communications', 'http://dbpedia.org/resource/Twitter'], 'types': ['', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>b'@NotifyBoston' b'We' b'need' b'a' b\"don''t\" b'block' b'the' b'box' b'ordinance...' b'And' b'traffic' b'needs' b'to' b'be' b'fixed' b'on' b'Atlantic' b'Ave' b'btwn' b'SStation' b'and' b'the' b'offramps'</td>\n",
       "      <td>NO</td>\n",
       "      <td>@NotifyBoston We need a don''t block the box ordinance... And traffic needs to be fixed on Atlantic Ave btwn SStation and the offramps</td>\n",
       "      <td>[b, ', @, NotifyBoston, ', b'We, ', b'need, ', b, ', a, ', b, '', don, '', t, '', b'block, ', b'the, ', b'box, ', b'ordinance, ..., ', b'And, ', b'traffic, ', b'needs, ', b'to, ', b'be, ', b'fixed, ', b'on, ', b'Atlantic, ', b'Ave, ', b'btwn, ', b'SStation, ', b'and, ', b'the, ', b'offramps, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Atlantic_Ocean'], 'types': ['Schema:Place,Schema:BodyOfWater,DBpedia:Place,DBpedia:NaturalPlace,DBpedia:Location,DBpedia:BodyOfWater']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007246</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>b'#' b'T-MOBILE' b'NOWE' b'HORYZONTY:' b\"Punk''s\" b'not' b'dead' b'-' b'Wielkie' b'ogladanie' b'filmow' b'we' b'Wrocawiu' b'trwa' b'w' b'najleps...' b'URL</td>\n",
       "      <td>NO</td>\n",
       "      <td># T-MOBILE NOWE HORYZONTY: Punk''s not dead - Wielkie oglądanie filmów we Wrocławiu trwa w najleps... http://t.co/UhtgxQES8x</td>\n",
       "      <td>[b, ', #, ', b'T-MOBILE, ', b'NOWE, ', b'HORYZONTY, :, ', b, '', Punk, '', s, '', b'not, ', b'dead, ', b'-, ', b'Wielkie, ', b'ogladanie, ', b'filmow, ', b'we, ', b'Wrocawiu, ', b'trwa, ', b, ', w, ', b'najleps, ..., ', b'URL]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/T-Mobile', 'http://dbpedia.org/resource/Gmina_Nowe', 'http://dbpedia.org/resource/Twitter'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q486972,Schema:Place,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>b'ACCIDENT' b'REPORTED' b'(Danvers,' b'MA):' b'RT-1' b'North' b'off-ramp' b'to' b'I-95' b'North' b'-' b'use' b'caution'</td>\n",
       "      <td>YES</td>\n",
       "      <td>ACCIDENT REPORTED (Danvers, MA): RT-1 North off-ramp to I-95 North - use caution</td>\n",
       "      <td>[b'ACCIDENT, ', b'REPORTED, ', b, ', (, Danvers, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'off-ramp, ', b'to, ', b, ', I-95, ', b'North, ', b'-, ', b'use, ', b'caution, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Interstate_95_in_Massachusetts'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011905</td>\n",
       "      <td>0.007246</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT' b'REPORTED' b'(Danvers,' b'MA):' b'RT-1' b'North' b'off-ramp' b'to' b'I-95' b'North' b'-' b'use' b'caution'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT REPORTED (Danvers, MA): RT-1 North off-ramp to I-95 North - use caution</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b'REPORTED, ', b, ', (, Danvers, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'off-ramp, ', b'to, ', b, ', I-95, ', b'North, ', b'-, ', b'use, ', b'caution, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Interstate_95_in_Massachusetts'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011905</td>\n",
       "      <td>0.007246</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>b'Look,' b'your' b'voice' b'is' b'lovely' b'but' b'that' b'breath' b'is' b'not.' b'Please' b'stop' b'serenading' b'the' b'subway' b'car' b'at' b'8am' b'with' b'that' b'vial' b'stench.' b'#commuterproblems'</td>\n",
       "      <td>NO</td>\n",
       "      <td>Look, your voice is lovely but that breath is not. Please stop serenading the subway car at 8am with that vial stench. #commuterproblems</td>\n",
       "      <td>[b'Look, ,, ', b'your, ', b'voice, ', b'is, ', b'lovely, ', b'but, ', b'that, ', b'breath, ', b'is, ', b'not, ., ', b'Please, ', b'stop, ', b'serenading, ', b'the, ', b'subway, ', b'car, ', b'at, ', b'8am, ', b'with, ', b'that, ', b'vial, ', b'stench, ., ', b, ', #, commuterproblems, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Look_(American_magazine)', 'http://dbpedia.org/resource/Rapid_transit'], 'types': ['Wikidata:Q41298,Wikidata:Q386724,Wikidata:Q1092563,Schema:CreativeWork,DBpedia:WrittenWork,DBpedia:Work,DBpedia:PeriodicalLiterature,DBpedia:Magazine', '']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>b'Opened' b'report' b'via' b'Android' b'at' b'146-184' b'South' b'St' b'Boston' b'URL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Opened report via Android at 146-184 South St Boston http://t.co/izfT7Oww65. Car with smashed window. heard that 3 or four more in the area.</td>\n",
       "      <td>[b'Opened, ', b'report, ', b'via, ', b'Android, ', b'at, ', b'146-184, ', b'South, ', b'St, ', b'Boston, ', b'URL]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Android_software_development', 'http://dbpedia.org/resource/Boston', 'http://dbpedia.org/resource/Twitter'], 'types': ['Wikidata:Q7397,Wikidata:Q386724,Schema:CreativeWork,DBpedia:Work,DBpedia:Software', 'Wikidata:Q486972,Schema:Place,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.021739</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>b'ACCIDENT:' b'RT-128' b'North' b'before' b'US-3' b'N/Middlesex' b'Tpk' b'(Exit' b'#32)' b'-' b'2-car' b'accident' b'with' b'reported' b'minor' b'injury' b'-' b'police' b'on' b'scene'</td>\n",
       "      <td>YES</td>\n",
       "      <td>ACCIDENT: RT-128 North before US-3 N/Middlesex Tpk (Exit #32) - 2-car accident with reported minor injury - police on scene</td>\n",
       "      <td>[b'ACCIDENT, :, ', b'RT-128, ', b'North, ', b'before, ', b'US-3, ', b, ', N/Middlesex, ', b'Tpk, ', b, ', (, Exit, ', b, ', #, 32, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'reported, ', b'minor, ', b'injury, ', b'-, ', b'police, ', b'on, ', b'scene, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/AEC_Regent_III_RT', 'http://dbpedia.org/resource/U.S._Route_3', 'http://dbpedia.org/resource/Middlesex_County_Cricket_Club'], 'types': ['DUL:DesignedArtifact,Schema:Product,DBpedia:MeanOfTransportation,DBpedia:Automobile', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:CricketTeam,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.021739</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'RT-128' b'North' b'before' b'US-3' b'N/Middlesex' b'Tpk' b'(Exit' b'#32)' b'-' b'2-car' b'accident' b'with' b'reported' b'minor' b'injury' b'-' b'police' b'on' b's...'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT: RT-128 North before US-3 N/Middlesex Tpk (Exit #32) - 2-car accident with reported minor injury - police on s…</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b'RT-128, ', b'North, ', b'before, ', b'US-3, ', b, ', N/Middlesex, ', b'Tpk, ', b, ', (, Exit, ', b, ', #, 32, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'reported, ', b'minor, ', b'injury, ', b'-, ', b'police, ', b'on, ', b, 's, ..., ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/U.S._Route_3', 'http://dbpedia.org/resource/Middlesex_County_Cricket_Club'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:CricketTeam,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>b'ACCIDENT:' b'I-90' b'(Mass' b'Pike)' b'East' b'after' b'RT-122' b'(Exit' b'#11)' b'-' b'2-car' b'accident' b'in' b'the' b'breakdown' b'lane'</td>\n",
       "      <td>YES</td>\n",
       "      <td>ACCIDENT: I-90 (Mass Pike) East after RT-122 (Exit #11) - 2-car accident in the breakdown lane</td>\n",
       "      <td>[b'ACCIDENT, :, ', b, ', I-90, ', b, ', (, Mass, ', b'Pike, ), ', b'East, ', b'after, ', b'RT-122, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'in, ', b'the, ', b'breakdown, ', b'lane, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Shoulder_(road)'], 'types': ['', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}</td>\n",
       "      <td>2</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'I-90' b'(Mass' b'Pike)' b'East' b'after' b'RT-122' b'(Exit' b'#11)' b'-' b'2-car' b'accident' b'in' b'the' b'breakdown' b'lane'</td>\n",
       "      <td>YES</td>\n",
       "      <td>RT @CommuterBoston: ACCIDENT: I-90 (Mass Pike) East after RT-122 (Exit #11) - 2-car accident in the breakdown lane</td>\n",
       "      <td>[b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b, ', I-90, ', b, ', (, Mass, ', b'Pike, ), ', b'East, ', b'after, ', b'RT-122, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'in, ', b'the, ', b'breakdown, ', b'lane, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Shoulder_(road)'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}</td>\n",
       "      <td>4</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>b'DISABLED' b'VEHICLE' b'(Chelsea,' b'MA):' b'RT-1' b'North' b'near' b'Carter' b'St' b'-' b'left' b'lane' b'blocked'</td>\n",
       "      <td>YES</td>\n",
       "      <td>DISABLED VEHICLE (Chelsea, MA): RT-1 North near Carter St - left lane blocked</td>\n",
       "      <td>[b'DISABLED, ', b'VEHICLE, ', b, ', (, Chelsea, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'near, ', b'Carter, ', b'St, ', b'-, ', b'left, ', b'lane, ', b'blocked, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Chelsea,_Massachusetts'], 'types': ['Wikidata:Q515,Wikidata:Q486972,Schema:Place,Schema:City,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:City']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011905</td>\n",
       "      <td>0.007246</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>b'Is' b'Tyler' b'Zeller' b'a' b'Legitimate' b'Building' b'Block' b'for' b'Boston' b\"Celtics''\" b'Future?' b'-' b'Bleacher' b'Report' b'URL#NBA' b'#BostonCeltics'</td>\n",
       "      <td>NO</td>\n",
       "      <td>Is Tyler Zeller a Legitimate Building Block for Boston Celtics'' Future? - Bleacher Report http://t.co/60v8mmseIX #NBA #BostonCeltics</td>\n",
       "      <td>[b'Is, ', b'Tyler, ', b'Zeller, ', b, ', a, ', b'Legitimate, ', b'Building, ', b'Block, ', b'for, ', b'Boston, ', b, '', Celtics, '', '', b'Future, ?, ', b'-, ', b'Bleacher, ', b'Report, ', b'URL, #, NBA, ', b, ', #, BostonCeltics, ']</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Tyler_Zeller', 'http://dbpedia.org/resource/Boston_Celtics', 'http://dbpedia.org/resource/Bleacher_Report', 'http://dbpedia.org/resource/Twitter', 'http://dbpedia.org/resource/National_Basketball_Association'], 'types': ['Http://xmlns.com/foaf/0.1/Person,Wikidata:Q5,Wikidata:Q3665646,Wikidata:Q24229398,Wikidata:Q215627,DUL:NaturalPerson,DUL:Agent,Schema:Person,DBpedia:Person,DBpedia:BasketballPlayer,DBpedia:Athlete,DBpedia:Agent', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:BasketballTeam,DBpedia:Agent', 'Wikidata:Q386724,Schema:WebPage,Schema:CreativeWork,DBpedia:Work,DBpedia:Website', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q623109,Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:SportsLeague,DBpedia:Organisation,DBpedia:BasketballLeague,DBpedia:Agent']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.036232</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>b'ROADWORK' b'(Groton,' b'CT):' b'I-95' b'SB' b'between' b'Clarence' b'B.' b'Sharp' b'Hwy' b'(Exit' b'#87)' b'and' b'U.S.' b'Sub' b'Base/Gales' b'Ferry' b'(Exit' b'#86)' b'-three' b'right' b'lanes' b'blocked'</td>\n",
       "      <td>NO</td>\n",
       "      <td>ROADWORK (Groton, CT): I-95 SB between Clarence B. Sharp Hwy (Exit #87) and U.S. Sub Base/Gales Ferry (Exit #86) -three right lanes blocked</td>\n",
       "      <td>[b'ROADWORK, ', b, ', (, Groton, ,, ', b'CT, ), :, ', b, ', I-95, ', b'SB, ', b'between, ', b'Clarence, ', b, ', B, ., ', b'Sharp, ', b'Hwy, ', b, ', (, Exit, ', b, ', #, 87, ), ', b'and, ', b, ', U.S, ., ', b'Sub, ', b'Base/Gales, ', b'Ferry, ', b, ', (, Exit, ', b, ', #, 86, ), ', b'-three, ', b'right, ', b'lanes, ', b'blocked, ']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'URI': ['http://dbpedia.org/resource/Groton_School', 'http://dbpedia.org/resource/Connecticut', 'http://dbpedia.org/resource/Interstate_95_in_Connecticut', 'http://dbpedia.org/resource/Bachelor_of_Science', 'http://dbpedia.org/resource/Clarence_Football_Club', 'http://dbpedia.org/resource/United_States', 'http://dbpedia.org/resource/Gales_Ferry,_Connecticut'], 'types': ['Wikidata:Q43229,Wikidata:Q3914,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:School,Schema:Organization,Schema:EducationalOrganization,DBpedia:School,DBpedia:Organisation,DBpedia:EducationalInstitution,DBpedia:Agent', 'Wikidata:Q3455524,Schema:Place,Schema:AdministrativeArea,DBpedia:Region,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:AdministrativeRegion', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:Agent', 'Wikidata:Q6256,Schema:Place,Schema:Country,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:Country', 'Schema:Place,Schema:LandmarksOrHistoricalBuildings,DBpedia:Place,DBpedia:Location,DBpedia:HistoricPlace']}</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.050725</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                               tweet  \\\n",
       "0   b'RT' b'@CommuterBoston:' b'ACCIDENT' b'(Wellesley,' b'MA):' b'RT-9' b'East' b'near' b'Cedar' b'St' b'-' b'involving' b'a' b'motorcycle' b'and' b'a' b'car' b'-' b'possible' b'lane' b'restrictions'               \n",
       "1   b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'I-290' b'East' b'before' b'I-495' b'(Exit' b'#26)' b'-' b'2-car' b'accident' b'with' b'possible' b'lane' b'blockages'                                                     \n",
       "2   b'Opened' b'report' b'via' b'iPhone' b'at' D b'Standard' b'St' b'URL                                                                                                                                               \n",
       "3   b'RT' b'@CommuterBoston:' b'ACCIDENT' b'REPORTED:' b'I-93' b'North' b'before' b'Granite' b'Av' b'(Exit' b'#11)' b'-' b'police' b'investigating'                                                                    \n",
       "4   b'Duduk' b'lebih' b'dari' D b'jam' b'sehari' b'berisiko' b'depresi' D b'persen' b'lebih' b'tinggi' b'dibandingkan' b'yang' b'hanya' b'duduk' D b'jam' b'sehari' b'[HuffingtonPost]'                                \n",
       "5   b'RT' b'@CommuterBoston:' b'ACCIDENT' b'(Providence,' b'RI):' b'I-195' b'West' b'near' b'Gano' b'St' b'-' b'right' b'shoulder' b'blocked'                                                                          \n",
       "6   b'Dedham' b'Police' b'Looking' b'for' b'Suspects' b'After' b'Car' b'Crashes' b'into' b'Verizon' b'Store' b'URL                                                                                                     \n",
       "7   b'@NotifyBoston' b'We' b'need' b'a' b\"don''t\" b'block' b'the' b'box' b'ordinance...' b'And' b'traffic' b'needs' b'to' b'be' b'fixed' b'on' b'Atlantic' b'Ave' b'btwn' b'SStation' b'and' b'the' b'offramps'        \n",
       "8   b'#' b'T-MOBILE' b'NOWE' b'HORYZONTY:' b\"Punk''s\" b'not' b'dead' b'-' b'Wielkie' b'ogladanie' b'filmow' b'we' b'Wrocawiu' b'trwa' b'w' b'najleps...' b'URL                                                         \n",
       "9   b'ACCIDENT' b'REPORTED' b'(Danvers,' b'MA):' b'RT-1' b'North' b'off-ramp' b'to' b'I-95' b'North' b'-' b'use' b'caution'                                                                                            \n",
       "10  b'RT' b'@CommuterBoston:' b'ACCIDENT' b'REPORTED' b'(Danvers,' b'MA):' b'RT-1' b'North' b'off-ramp' b'to' b'I-95' b'North' b'-' b'use' b'caution'                                                                  \n",
       "11  b'Look,' b'your' b'voice' b'is' b'lovely' b'but' b'that' b'breath' b'is' b'not.' b'Please' b'stop' b'serenading' b'the' b'subway' b'car' b'at' b'8am' b'with' b'that' b'vial' b'stench.' b'#commuterproblems'      \n",
       "12  b'Opened' b'report' b'via' b'Android' b'at' b'146-184' b'South' b'St' b'Boston' b'URL                                                                                                                              \n",
       "13  b'ACCIDENT:' b'RT-128' b'North' b'before' b'US-3' b'N/Middlesex' b'Tpk' b'(Exit' b'#32)' b'-' b'2-car' b'accident' b'with' b'reported' b'minor' b'injury' b'-' b'police' b'on' b'scene'                            \n",
       "14  b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'RT-128' b'North' b'before' b'US-3' b'N/Middlesex' b'Tpk' b'(Exit' b'#32)' b'-' b'2-car' b'accident' b'with' b'reported' b'minor' b'injury' b'-' b'police' b'on' b's...'   \n",
       "15  b'ACCIDENT:' b'I-90' b'(Mass' b'Pike)' b'East' b'after' b'RT-122' b'(Exit' b'#11)' b'-' b'2-car' b'accident' b'in' b'the' b'breakdown' b'lane'                                                                     \n",
       "16  b'RT' b'@CommuterBoston:' b'ACCIDENT:' b'I-90' b'(Mass' b'Pike)' b'East' b'after' b'RT-122' b'(Exit' b'#11)' b'-' b'2-car' b'accident' b'in' b'the' b'breakdown' b'lane'                                           \n",
       "17  b'DISABLED' b'VEHICLE' b'(Chelsea,' b'MA):' b'RT-1' b'North' b'near' b'Carter' b'St' b'-' b'left' b'lane' b'blocked'                                                                                               \n",
       "18  b'Is' b'Tyler' b'Zeller' b'a' b'Legitimate' b'Building' b'Block' b'for' b'Boston' b\"Celtics''\" b'Future?' b'-' b'Bleacher' b'Report' b'URL#NBA' b'#BostonCeltics'                                                  \n",
       "19  b'ROADWORK' b'(Groton,' b'CT):' b'I-95' b'SB' b'between' b'Clarence' b'B.' b'Sharp' b'Hwy' b'(Exit' b'#87)' b'and' b'U.S.' b'Sub' b'Base/Gales' b'Ferry' b'(Exit' b'#86)' b'-three' b'right' b'lanes' b'blocked'   \n",
       "\n",
       "      Y  \\\n",
       "0   YES   \n",
       "1   YES   \n",
       "2   NO    \n",
       "3   YES   \n",
       "4   NO    \n",
       "5   YES   \n",
       "6   YES   \n",
       "7   NO    \n",
       "8   NO    \n",
       "9   YES   \n",
       "10  YES   \n",
       "11  NO    \n",
       "12  YES   \n",
       "13  YES   \n",
       "14  YES   \n",
       "15  YES   \n",
       "16  YES   \n",
       "17  YES   \n",
       "18  NO    \n",
       "19  NO    \n",
       "\n",
       "                                                                                                                                       tweet_org  \\\n",
       "0   RT @CommuterBoston: ACCIDENT (Wellesley, MA): RT-9 East near Cedar St - involving a motorcycle and a car - possible lane restrictions          \n",
       "1   RT @CommuterBoston: ACCIDENT: I-290 East before I-495 (Exit #26) - 2-car accident with possible lane blockages                                 \n",
       "2   Opened report via iPhone at 19 Standard St http://t.co/8vc761aCHT. Road kill on sidewalk.                                                      \n",
       "3   RT @CommuterBoston: ACCIDENT REPORTED: I-93 North before Granite Av (Exit #11) - police investigating                                          \n",
       "4   Duduk lebih dari 7 jam sehari berisiko depresi 47 persen lebih tinggi dibandingkan yang hanya duduk 4 jam sehari [HuffingtonPost]              \n",
       "5   RT @CommuterBoston: ACCIDENT (Providence, RI): I-195 West near Gano St - right shoulder blocked                                                \n",
       "6   Dedham Police Looking for Suspects After Car Crashes into Verizon Store http://t.co/u3zrVjliqB                                                 \n",
       "7   @NotifyBoston We need a don''t block the box ordinance... And traffic needs to be fixed on Atlantic Ave btwn SStation and the offramps         \n",
       "8   # T-MOBILE NOWE HORYZONTY: Punk''s not dead - Wielkie oglądanie filmów we Wrocławiu trwa w najleps... http://t.co/UhtgxQES8x                   \n",
       "9   ACCIDENT REPORTED (Danvers, MA): RT-1 North off-ramp to I-95 North - use caution                                                               \n",
       "10  RT @CommuterBoston: ACCIDENT REPORTED (Danvers, MA): RT-1 North off-ramp to I-95 North - use caution                                           \n",
       "11  Look, your voice is lovely but that breath is not. Please stop serenading the subway car at 8am with that vial stench. #commuterproblems       \n",
       "12  Opened report via Android at 146-184 South St Boston http://t.co/izfT7Oww65. Car with smashed window. heard that 3 or four more in the area.   \n",
       "13  ACCIDENT: RT-128 North before US-3 N/Middlesex Tpk (Exit #32) - 2-car accident with reported minor injury - police on scene                    \n",
       "14  RT @CommuterBoston: ACCIDENT: RT-128 North before US-3 N/Middlesex Tpk (Exit #32) - 2-car accident with reported minor injury - police on s…   \n",
       "15  ACCIDENT: I-90 (Mass Pike) East after RT-122 (Exit #11) - 2-car accident in the breakdown lane                                                 \n",
       "16  RT @CommuterBoston: ACCIDENT: I-90 (Mass Pike) East after RT-122 (Exit #11) - 2-car accident in the breakdown lane                             \n",
       "17  DISABLED VEHICLE (Chelsea, MA): RT-1 North near Carter St - left lane blocked                                                                  \n",
       "18  Is Tyler Zeller a Legitimate Building Block for Boston Celtics'' Future? - Bleacher Report http://t.co/60v8mmseIX #NBA #BostonCeltics          \n",
       "19  ROADWORK (Groton, CT): I-95 SB between Clarence B. Sharp Hwy (Exit #87) and U.S. Sub Base/Gales Ferry (Exit #86) -three right lanes blocked    \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                     tweet_tok_lem  \\\n",
       "0   [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b, ', (, Wellesley, ,, ', b'MA, ), :, ', b'RT-9, ', b'East, ', b'near, ', b'Cedar, ', b'St, ', b'-, ', b'involving, ', b, ', a, ', b'motorcycle, ', b'and, ', b, ', a, ', b'car, ', b'-, ', b'possible, ', b'lane, ', b'restrictions, ']                                                 \n",
       "1   [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b, ', I-290, ', b'East, ', b'before, ', b, ', I-495, ', b, ', (, Exit, ', b, ', #, 26, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'possible, ', b'lane, ', b'blockages, ']                                                                                              \n",
       "2   [b'Opened, ', b'report, ', b'via, ', b'iPhone, ', b'at, ', D, b'Standard, ', b'St, ', b'URL]                                                                                                                                                                                                                                                     \n",
       "3   [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b'REPORTED, :, ', b, ', I-93, ', b'North, ', b'before, ', b'Granite, ', b'Av, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b'police, ', b'investigating, ']                                                                                                                           \n",
       "4   [b'Duduk, ', b'lebih, ', b'dari, ', D, b'jam, ', b'sehari, ', b'berisiko, ', b'depresi, ', D, b'persen, ', b'lebih, ', b'tinggi, ', b'dibandingkan, ', b'yang, ', b'hanya, ', b'duduk, ', D, b'jam, ', b'sehari, ', b, ', [, HuffingtonPost, ], ']                                                                                               \n",
       "5   [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b, ', (, Providence, ,, ', b'RI, ), :, ', b, ', I-195, ', b'West, ', b'near, ', b'Gano, ', b'St, ', b'-, ', b'right, ', b'shoulder, ', b'blocked, ']                                                                                                                                     \n",
       "6   [b'Dedham, ', b'Police, ', b'Looking, ', b'for, ', b'Suspects, ', b'After, ', b'Car, ', b'Crashes, ', b'into, ', b'Verizon, ', b'Store, ', b'URL]                                                                                                                                                                                                \n",
       "7   [b, ', @, NotifyBoston, ', b'We, ', b'need, ', b, ', a, ', b, '', don, '', t, '', b'block, ', b'the, ', b'box, ', b'ordinance, ..., ', b'And, ', b'traffic, ', b'needs, ', b'to, ', b'be, ', b'fixed, ', b'on, ', b'Atlantic, ', b'Ave, ', b'btwn, ', b'SStation, ', b'and, ', b'the, ', b'offramps, ']                                          \n",
       "8   [b, ', #, ', b'T-MOBILE, ', b'NOWE, ', b'HORYZONTY, :, ', b, '', Punk, '', s, '', b'not, ', b'dead, ', b'-, ', b'Wielkie, ', b'ogladanie, ', b'filmow, ', b'we, ', b'Wrocawiu, ', b'trwa, ', b, ', w, ', b'najleps, ..., ', b'URL]                                                                                                               \n",
       "9   [b'ACCIDENT, ', b'REPORTED, ', b, ', (, Danvers, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'off-ramp, ', b'to, ', b, ', I-95, ', b'North, ', b'-, ', b'use, ', b'caution, ']                                                                                                                                                                  \n",
       "10  [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, ', b'REPORTED, ', b, ', (, Danvers, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'off-ramp, ', b'to, ', b, ', I-95, ', b'North, ', b'-, ', b'use, ', b'caution, ']                                                                                                                          \n",
       "11  [b'Look, ,, ', b'your, ', b'voice, ', b'is, ', b'lovely, ', b'but, ', b'that, ', b'breath, ', b'is, ', b'not, ., ', b'Please, ', b'stop, ', b'serenading, ', b'the, ', b'subway, ', b'car, ', b'at, ', b'8am, ', b'with, ', b'that, ', b'vial, ', b'stench, ., ', b, ', #, commuterproblems, ']                                                  \n",
       "12  [b'Opened, ', b'report, ', b'via, ', b'Android, ', b'at, ', b'146-184, ', b'South, ', b'St, ', b'Boston, ', b'URL]                                                                                                                                                                                                                               \n",
       "13  [b'ACCIDENT, :, ', b'RT-128, ', b'North, ', b'before, ', b'US-3, ', b, ', N/Middlesex, ', b'Tpk, ', b, ', (, Exit, ', b, ', #, 32, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'reported, ', b'minor, ', b'injury, ', b'-, ', b'police, ', b'on, ', b'scene, ']                                                                     \n",
       "14  [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b'RT-128, ', b'North, ', b'before, ', b'US-3, ', b, ', N/Middlesex, ', b'Tpk, ', b, ', (, Exit, ', b, ', #, 32, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'with, ', b'reported, ', b'minor, ', b'injury, ', b'-, ', b'police, ', b'on, ', b, 's, ..., ']                          \n",
       "15  [b'ACCIDENT, :, ', b, ', I-90, ', b, ', (, Mass, ', b'Pike, ), ', b'East, ', b'after, ', b'RT-122, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'in, ', b'the, ', b'breakdown, ', b'lane, ']                                                                                                                  \n",
       "16  [b'RT, ', b, ', @, CommuterBoston, :, ', b'ACCIDENT, :, ', b, ', I-90, ', b, ', (, Mass, ', b'Pike, ), ', b'East, ', b'after, ', b'RT-122, ', b, ', (, Exit, ', b, ', #, 11, ), ', b'-, ', b, ', 2-car, ', b'accident, ', b'in, ', b'the, ', b'breakdown, ', b'lane, ']                                                                          \n",
       "17  [b'DISABLED, ', b'VEHICLE, ', b, ', (, Chelsea, ,, ', b'MA, ), :, ', b'RT-1, ', b'North, ', b'near, ', b'Carter, ', b'St, ', b'-, ', b'left, ', b'lane, ', b'blocked, ']                                                                                                                                                                         \n",
       "18  [b'Is, ', b'Tyler, ', b'Zeller, ', b, ', a, ', b'Legitimate, ', b'Building, ', b'Block, ', b'for, ', b'Boston, ', b, '', Celtics, '', '', b'Future, ?, ', b'-, ', b'Bleacher, ', b'Report, ', b'URL, #, NBA, ', b, ', #, BostonCeltics, ']                                                                                                       \n",
       "19  [b'ROADWORK, ', b, ', (, Groton, ,, ', b'CT, ), :, ', b, ', I-95, ', b'SB, ', b'between, ', b'Clarence, ', b, ', B, ., ', b'Sharp, ', b'Hwy, ', b, ', (, Exit, ', b, ', #, 87, ), ', b'and, ', b, ', U.S, ., ', b'Sub, ', b'Base/Gales, ', b'Ferry, ', b, ', (, Exit, ', b, ', #, 86, ), ', b'-three, ', b'right, ', b'lanes, ', b'blocked, ']   \n",
       "\n",
       "    no_ques_marks  no_excl_marks  no_uppercase  \\\n",
       "0   0              0              0              \n",
       "1   0              0              0              \n",
       "2   0              0              1              \n",
       "3   0              0              0              \n",
       "4   0              0              3              \n",
       "5   0              0              0              \n",
       "6   0              0              0              \n",
       "7   0              0              0              \n",
       "8   0              0              0              \n",
       "9   0              0              0              \n",
       "10  0              0              0              \n",
       "11  0              0              0              \n",
       "12  0              0              0              \n",
       "13  0              0              0              \n",
       "14  0              0              0              \n",
       "15  0              0              0              \n",
       "16  0              0              0              \n",
       "17  0              0              0              \n",
       "18  1              0              0              \n",
       "19  0              0              0              \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               annotations  \\\n",
       "0   {'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Master_of_Arts', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Motorcycle'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "1   {'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Interstate_290_(Illinois)', 'http://dbpedia.org/resource/Falmouth_Spur'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
       "2   {'URI': ['http://dbpedia.org/resource/IPhone', 'http://dbpedia.org/resource/Twitter', 'http://dbpedia.org/resource/Roadkill', 'http://dbpedia.org/resource/Sidewalk'], 'types': ['Wikidata:Q1067263,DBpedia:InformationAppliance,DBpedia:Device', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', '', '']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "3   {'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Interstate_93'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         \n",
       "4   {'URI': ['http://dbpedia.org/resource/Duduk', 'http://dbpedia.org/resource/Dopamine_reuptake_inhibitor', 'http://dbpedia.org/resource/Duduk', 'http://dbpedia.org/resource/The_Huffington_Post'], 'types': ['', '', '', 'Wikidata:Q386724,Schema:WebPage,Schema:CreativeWork,DBpedia:Work,DBpedia:Website']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       "5   {'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Providence_Friars_men's_basketball', 'http://dbpedia.org/resource/Rhode_Island', 'http://dbpedia.org/resource/Interstate_195_(Rhode_Island–Massachusetts)'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', 'Wikidata:Q3455524,Schema:Place,Schema:AdministrativeArea,DBpedia:Region,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:AdministrativeRegion', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
       "6   {'URI': ['http://dbpedia.org/resource/Dedham_station', 'http://dbpedia.org/resource/Verizon_Communications', 'http://dbpedia.org/resource/Twitter'], 'types': ['', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \n",
       "7   {'URI': ['http://dbpedia.org/resource/Atlantic_Ocean'], 'types': ['Schema:Place,Schema:BodyOfWater,DBpedia:Place,DBpedia:NaturalPlace,DBpedia:Location,DBpedia:BodyOfWater']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       "8   {'URI': ['http://dbpedia.org/resource/T-Mobile', 'http://dbpedia.org/resource/Gmina_Nowe', 'http://dbpedia.org/resource/Twitter'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q486972,Schema:Place,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
       "9   {'URI': ['http://dbpedia.org/resource/Interstate_95_in_Massachusetts'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "10  {'URI': ['http://dbpedia.org/resource/Interstate_95_in_Massachusetts'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "11  {'URI': ['http://dbpedia.org/resource/Look_(American_magazine)', 'http://dbpedia.org/resource/Rapid_transit'], 'types': ['Wikidata:Q41298,Wikidata:Q386724,Wikidata:Q1092563,Schema:CreativeWork,DBpedia:WrittenWork,DBpedia:Work,DBpedia:PeriodicalLiterature,DBpedia:Magazine', '']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
       "12  {'URI': ['http://dbpedia.org/resource/Android_software_development', 'http://dbpedia.org/resource/Boston', 'http://dbpedia.org/resource/Twitter'], 'types': ['Wikidata:Q7397,Wikidata:Q386724,Schema:CreativeWork,DBpedia:Work,DBpedia:Software', 'Wikidata:Q486972,Schema:Place,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
       "13  {'URI': ['http://dbpedia.org/resource/AEC_Regent_III_RT', 'http://dbpedia.org/resource/U.S._Route_3', 'http://dbpedia.org/resource/Middlesex_County_Cricket_Club'], 'types': ['DUL:DesignedArtifact,Schema:Product,DBpedia:MeanOfTransportation,DBpedia:Automobile', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:CricketTeam,DBpedia:Agent']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
       "14  {'URI': ['http://dbpedia.org/resource/U.S._Route_3', 'http://dbpedia.org/resource/Middlesex_County_Cricket_Club'], 'types': ['Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:CricketTeam,DBpedia:Agent']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           \n",
       "15  {'URI': ['http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Shoulder_(road)'], 'types': ['', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
       "16  {'URI': ['http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Massachusetts_Turnpike', 'http://dbpedia.org/resource/Sacramento_Regional_Transit_District', 'http://dbpedia.org/resource/Shoulder_(road)'], 'types': ['Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:BusCompany,DBpedia:Agent', '']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                \n",
       "17  {'URI': ['http://dbpedia.org/resource/Chelsea,_Massachusetts'], 'types': ['Wikidata:Q515,Wikidata:Q486972,Schema:Place,Schema:City,DBpedia:Settlement,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:City']}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "18  {'URI': ['http://dbpedia.org/resource/Tyler_Zeller', 'http://dbpedia.org/resource/Boston_Celtics', 'http://dbpedia.org/resource/Bleacher_Report', 'http://dbpedia.org/resource/Twitter', 'http://dbpedia.org/resource/National_Basketball_Association'], 'types': ['Http://xmlns.com/foaf/0.1/Person,Wikidata:Q5,Wikidata:Q3665646,Wikidata:Q24229398,Wikidata:Q215627,DUL:NaturalPerson,DUL:Agent,Schema:Person,DBpedia:Person,DBpedia:BasketballPlayer,DBpedia:Athlete,DBpedia:Agent', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:BasketballTeam,DBpedia:Agent', 'Wikidata:Q386724,Schema:WebPage,Schema:CreativeWork,DBpedia:Work,DBpedia:Website', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:Organisation,DBpedia:Company,DBpedia:Agent', 'Wikidata:Q623109,Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:Organization,DBpedia:SportsLeague,DBpedia:Organisation,DBpedia:BasketballLeague,DBpedia:Agent']}                                                                                                                                                                                                                                                         \n",
       "19  {'URI': ['http://dbpedia.org/resource/Groton_School', 'http://dbpedia.org/resource/Connecticut', 'http://dbpedia.org/resource/Interstate_95_in_Connecticut', 'http://dbpedia.org/resource/Bachelor_of_Science', 'http://dbpedia.org/resource/Clarence_Football_Club', 'http://dbpedia.org/resource/United_States', 'http://dbpedia.org/resource/Gales_Ferry,_Connecticut'], 'types': ['Wikidata:Q43229,Wikidata:Q3914,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:School,Schema:Organization,Schema:EducationalOrganization,DBpedia:School,DBpedia:Organisation,DBpedia:EducationalInstitution,DBpedia:Agent', 'Wikidata:Q3455524,Schema:Place,Schema:AdministrativeArea,DBpedia:Region,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:AdministrativeRegion', 'Wikidata:Q34442,Schema:Place,DBpedia:RouteOfTransportation,DBpedia:Road,DBpedia:Place,DBpedia:Location,DBpedia:Infrastructure,DBpedia:ArchitecturalStructure', '', 'Wikidata:Q43229,Wikidata:Q24229398,DUL:SocialPerson,DUL:Agent,Schema:SportsTeam,Schema:Organization,DBpedia:SportsTeam,DBpedia:Organisation,DBpedia:Agent', 'Wikidata:Q6256,Schema:Place,Schema:Country,DBpedia:PopulatedPlace,DBpedia:Place,DBpedia:Location,DBpedia:Country', 'Schema:Place,Schema:LandmarksOrHistoricalBuildings,DBpedia:Place,DBpedia:Location,DBpedia:HistoricPlace']}   \n",
       "\n",
       "    num_same_URI  acc_overlap_types  non_acc_overlap_types  num_locations  \n",
       "0   2             0.023810           0.014493               1              \n",
       "1   0             0.023810           0.014493               0              \n",
       "2   0             0.035714           0.021739               0              \n",
       "3   0             0.023810           0.014493               0              \n",
       "4   2             0.023810           0.014493               0              \n",
       "5   0             0.047619           0.028986               0              \n",
       "6   0             0.023810           0.014493               1              \n",
       "7   0             0.000000           0.007246               0              \n",
       "8   0             0.023810           0.014493               0              \n",
       "9   0             0.011905           0.007246               2              \n",
       "10  0             0.011905           0.007246               1              \n",
       "11  0             0.023810           0.014493               0              \n",
       "12  0             0.035714           0.021739               0              \n",
       "13  0             0.035714           0.021739               0              \n",
       "14  0             0.023810           0.014493               0              \n",
       "15  2             0.023810           0.014493               0              \n",
       "16  4             0.023810           0.014493               0              \n",
       "17  0             0.011905           0.007246               0              \n",
       "18  0             0.047619           0.036232               0              \n",
       "19  0             0.071429           0.050725               1              "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_city.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#unicode conversion\n",
    "df_city['tweet_org']=df_city['tweet']\n",
    "df_city['tweet'] = (df_city['tweet'].map(lambda x: unicodedata.normalize('NFKD', x))\n",
    "              .str.encode('ascii', 'ignore'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# slang conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "slang_covert_url='http://www.netlingo.com/acronyms.php'\n",
    "slangdict= {}\n",
    "\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests, json\n",
    "resp = requests.get('http://www.netlingo.com/acronyms.php')\n",
    "soup = BeautifulSoup(resp.text, \"html.parser\")\n",
    "\n",
    "key=\"\"\n",
    "value=\"\"\n",
    "for div in soup.findAll('div', attrs={'class':'list_box3'}):\n",
    "    \n",
    "    for li in div.findAll('li'):\n",
    "        for a in li.findAll('a'):\n",
    "            key =a.text\n",
    "            value = li.text.split(key)[1]\n",
    "            slangdict[key.lower()]=value\n",
    "            \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def slang_to_formal(input):\n",
    "    i=1\n",
    "    formal=[]\n",
    "    for slang in input.split():\n",
    "        i=i+1\n",
    "        if slang.lower() in slangdict:\n",
    "            formal.append(slangdict[slang])\n",
    "        elif slang.isdigit():\n",
    "            \n",
    "            formal.append('D')  #not working  (replacing digits)\n",
    "        else:\n",
    "            formal.append(slang)\n",
    "       \n",
    "        \n",
    "    return  \" \".join(str(x) for x in formal)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_city['tweet'] = df_city.apply(lambda row: slang_to_formal(row['tweet']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#replacing URL and digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_URL(tweet):\n",
    "    return re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', 'URL', tweet)\n",
    "\n",
    "df_city['tweet'] = df_city.apply(lambda row: replace_URL(row['tweet']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove stop words\n",
    "\n",
    "stop = stopwords.words('english')\n",
    "df_city['tweet'] = df_city.apply(lambda x: [item for item in x if item not in stop])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenisation\n",
    "\n",
    "df_city['tweet_tok_lem'] = df_city.apply(lambda row: nltk.word_tokenize(row['tweet']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lemmatize\n",
    "def lemmatize(s):\n",
    "       \n",
    "    s = [lemmatizer.lemmatize(word) for word in s]\n",
    "    return s\n",
    "\n",
    "df_city['tweet_tok_lem'] = df_city.apply(lambda row: lemmatize(row['tweet_tok_lem']), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#syntactic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_city['no_ques_marks']=df_city.apply(lambda row: (row['tweet'].count(\"?\")), axis=1)\n",
    "df_city['no_excl_marks']=df_city.apply(lambda row: (row['tweet'].count(\"!\")), axis=1)\n",
    "df_city['no_uppercase']=df_city.apply(lambda row: sum(map(str.isupper, row['tweet'].split())), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Number same URI (NER-DBPedia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def duplicates(values):\n",
    "    \n",
    "    dups = Counter(values) - Counter(set(values))\n",
    "    return sum(dups.values()) + len(dups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DBPedia api hit\n",
    "def get_annotation(tweet):\n",
    "    try:\n",
    "        annotations = spotlight.annotate('http://api.dbpedia-spotlight.org/en/annotate',tweet,\n",
    "                              confidence=0.4, support=20)\n",
    "    except Exception:\n",
    "        return {'URI':[],'types':[]}\n",
    "    \n",
    "    URI_lst=[]\n",
    "    types_lst=[]\n",
    "    \n",
    "    for ann_dict in annotations:\n",
    "        URI_lst.append(ann_dict['URI'])\n",
    "        types_lst.append(ann_dict['types'])\n",
    "    \n",
    "    return {'URI':URI_lst,'types':types_lst}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#annotation features\n",
    "\n",
    "#global \n",
    "accident_types=set()\n",
    "non_accident_types=set()\n",
    "    \n",
    "def create_accident_types(ann_types,Y):\n",
    "       \n",
    "    for types in ann_types:\n",
    "        if Y=='YES':\n",
    "            accident_types.add(types)\n",
    "        else:\n",
    "            non_accident_types.add(types)\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_city['annotations'] = df_city.apply(lambda row: get_annotation(row['tweet_org']), axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating accident and non-accident frequency distribution\n",
    "for ann,Y in zip(df_city.annotations, df_city.Y):\n",
    "    if 'types' in ann:\n",
    "        create_accident_types(ann['types'],Y)\n",
    "                              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NER features\n",
    "\n",
    "df_city['num_same_URI'] = df_city.apply(lambda row: duplicates(row['annotations']['URI']), axis=1)\n",
    "df_city['acc_overlap_types'] = df_city.apply(lambda row: (len(accident_types.intersection(row['annotations']['types']))/len(accident_types)), axis=1)\n",
    "df_city['non_acc_overlap_types'] = df_city.apply(lambda row: (len(non_accident_types.intersection(row['annotations']['types']))/len(non_accident_types)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I am in LOC LOC DATETIME'"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_location_count(text):\n",
    "    doc = nlp(text)\n",
    "    return sum([1  for X in doc.ents if X.label_=='GPE'])\n",
    "\n",
    "\n",
    "def replace_LOC(text):\n",
    "    doc = nlp(text)\n",
    "    result_tweet=''\n",
    "    abs_dict={X.text:'LOC' for X in doc.ents if X.label_=='GPE'}\n",
    "   # loc_dict={n:'LOC' for n in range(1,10) if n==1}\n",
    "    dates = search_dates(text)\n",
    "    if dates is not None:\n",
    "      \n",
    "        for tok in dates:\n",
    "\n",
    "            if isinstance(tok[1], datetime.datetime):\n",
    "              \n",
    "                text=text.replace(tok[0],'DATETIME')\n",
    "\n",
    "        \n",
    "    for word in text.split():\n",
    "        if word in abs_dict:\n",
    "            word=abs_dict[word]\n",
    "        \n",
    "        result_tweet=result_tweet+' '+word\n",
    "        \n",
    "        \n",
    "    return(result_tweet.strip())\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "replace_LOC('I am in USA Dallas at 10 am')\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_city['num_locations'] = df_city.apply(lambda row: get_location_count(row['tweet']), axis=1)\n",
    "df_city['replaced_twet'] = df_city.apply(lambda row: replace_LOC(row['tweet']), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X =df_city[['tweet','no_ques_marks', 'no_excl_marks', 'no_uppercase', 'annotations',\n",
    "       'num_same_URI', 'acc_overlap_types', 'non_acc_overlap_types',\n",
    "       'num_locations']]\n",
    "Y = df_city.loc[:, 'Y']\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create tweets corpus\n",
    "all_tweets_corpus=df_city['tweet'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pipeline Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#classes to return text and numeric values for transformer\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "class TextSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, field):\n",
    "        self.field = field\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.field]\n",
    "class NumberSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, field):\n",
    "        self.field = field\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[[self.field]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize(s):\n",
    "    s = [lemmatizer.lemmatize(word) for word in s]\n",
    "    return s\n",
    "    \n",
    "def Tokenizer(str_input):\n",
    "    words = re.sub(r\"[^A-Za-z0-9\\-]\", \" \", str_input).lower().split()\n",
    "    return lemmatize(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = Pipeline([\n",
    "    ('features', FeatureUnion([\n",
    "        ('tfidf_reduced', Pipeline([\n",
    "            ('colext', TextSelector('replaced_twet')),\n",
    "            ('tfidf', TfidfVectorizer(tokenizer=Tokenizer, stop_words=stop_words,\n",
    "                     min_df=.0025, max_df=0.25, ngram_range=(1,2))),\n",
    "            ('svd', TruncatedSVD(algorithm='randomized', n_components=300)), \n",
    "        ])),\n",
    "        ('question_marks', Pipeline([\n",
    "            ('wordext', NumberSelector('no_ques_marks')),\n",
    "            ('wscaler', StandardScaler()),\n",
    "        ])),\n",
    "         ('no_excl_marks', Pipeline([\n",
    "            ('wordext', NumberSelector('no_excl_marks')),\n",
    "            ('wscaler', StandardScaler()),\n",
    "        ])),\n",
    "         ('no_uppercase', Pipeline([\n",
    "            ('wordext', NumberSelector('no_uppercase')),\n",
    "            ('wscaler', StandardScaler()),\n",
    "        ])),\n",
    "         ('num_locations', Pipeline([\n",
    "            ('wordext', NumberSelector('num_locations')),\n",
    "            ('wscaler', StandardScaler()),\n",
    "        ])),\n",
    "         ('num_same_URI', Pipeline([\n",
    "            ('wordext', NumberSelector('num_same_URI')),\n",
    "            ('wscaler', StandardScaler()),\n",
    "        ])),\n",
    "         ('acc_overlap_types', Pipeline([\n",
    "            ('wordext', NumberSelector('acc_overlap_types')),\n",
    "        \n",
    "        ])),\n",
    "         ('non_acc_overlap_types', Pipeline([\n",
    "            ('wordext', NumberSelector('non_acc_overlap_types')),\n",
    "           \n",
    "        ])),\n",
    "    ])),\n",
    "#    ('clf', XGBClassifier(max_depth=3, n_estimators=300, learning_rate=0.1)),\n",
    "    ('rf', RandomForestClassifier()),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kadss\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:300: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['doe', 'ha', 'wa'] not in stop_words.\n",
      "  'stop_words.' % sorted(inconsistent))\n",
      "C:\\Users\\kadss\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# fitting & prediction\n",
    "\n",
    "classifier.fit(X_train, y_train)\n",
    "preds = classifier.predict(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8822695035460993\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          NO       0.88      0.99      0.93       571\n",
      "         YES       0.88      0.44      0.59       134\n",
      "\n",
      "    accuracy                           0.88       705\n",
      "   macro avg       0.88      0.71      0.76       705\n",
      "weighted avg       0.88      0.88      0.87       705\n",
      "\n",
      "[[563   8]\n",
      " [ 75  59]]\n",
      "F1-Score: 0.8659099308378649\n"
     ]
    }
   ],
   "source": [
    "# Evaluation\n",
    "\n",
    "print(\"Accuracy:\", accuracy_score(y_test, preds))\n",
    "print(classification_report(y_test, preds))\n",
    "print(confusion_matrix(y_test, preds))\n",
    "print(\"F1-Score:\",f1_score(y_test, preds, average='weighted')) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}