#1. Data Cleaning and Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pares\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import nltk\n",
    "import re\n",
    "import string\n",
    "import spacy\n",
    "import itertools\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import silhouette_score\n",
    "from sklearn.cluster import KMeans\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "import matplotlib.colors as mcolors\n",
    "import time\n",
    "import scipy.sparse\n",
    "from gensim import matutils,models\n",
    "from gensim import corpora\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from nltk import pos_tag\n",
    "import time\n",
    "import numpy as np\n",
    "import multiprocess as mp\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Connecting to the sqlite database and fetching all the data\n",
    "conn = sqlite3.connect(\"C:/Users/pares/Downloads/18_2157_compressed_database.sqlite/database.sqlite\")\n",
    "data = pd.read_sql_query(\"\"\" \n",
    "SELECT *\n",
    "FROM Reviews\n",
    "\"\"\",conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 568454 entries, 0 to 568453\n",
      "Data columns (total 10 columns):\n",
      "Id                        568454 non-null int64\n",
      "ProductId                 568454 non-null object\n",
      "UserId                    568454 non-null object\n",
      "ProfileName               568454 non-null object\n",
      "HelpfulnessNumerator      568454 non-null int64\n",
      "HelpfulnessDenominator    568454 non-null int64\n",
      "Score                     568454 non-null int64\n",
      "Time                      568454 non-null int64\n",
      "Summary                   568454 non-null object\n",
      "Text                      568454 non-null object\n",
      "dtypes: int64(5), object(5)\n",
      "memory usage: 43.4+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>HelpfulnessNumerator</th>\n",
       "      <th>HelpfulnessDenominator</th>\n",
       "      <th>Score</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>B001E4KFG0</td>\n",
       "      <td>A3SGXH7AUHU8GW</td>\n",
       "      <td>delmartian</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1303862400</td>\n",
       "      <td>Good Quality Dog Food</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>B00813GRG4</td>\n",
       "      <td>A1D87F6ZCVE5NK</td>\n",
       "      <td>dll pa</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1346976000</td>\n",
       "      <td>Not as Advertised</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>B000LQOCH0</td>\n",
       "      <td>ABXLMWJIXXAIN</td>\n",
       "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1219017600</td>\n",
       "      <td>\"Delight\" says it all</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>B000UA0QIQ</td>\n",
       "      <td>A395BORC6FGVXV</td>\n",
       "      <td>Karl</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1307923200</td>\n",
       "      <td>Cough Medicine</td>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>B006K2ZZ7K</td>\n",
       "      <td>A1UQRSCLF8GW1T</td>\n",
       "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1350777600</td>\n",
       "      <td>Great taffy</td>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id   ProductId          UserId                      ProfileName  \\\n",
       "0   1  B001E4KFG0  A3SGXH7AUHU8GW                       delmartian   \n",
       "1   2  B00813GRG4  A1D87F6ZCVE5NK                           dll pa   \n",
       "2   3  B000LQOCH0   ABXLMWJIXXAIN  Natalia Corres \"Natalia Corres\"   \n",
       "3   4  B000UA0QIQ  A395BORC6FGVXV                             Karl   \n",
       "4   5  B006K2ZZ7K  A1UQRSCLF8GW1T    Michael D. Bigham \"M. Wassir\"   \n",
       "\n",
       "   HelpfulnessNumerator  HelpfulnessDenominator  Score        Time  \\\n",
       "0                     1                       1      5  1303862400   \n",
       "1                     0                       0      1  1346976000   \n",
       "2                     1                       1      4  1219017600   \n",
       "3                     3                       3      2  1307923200   \n",
       "4                     0                       0      5  1350777600   \n",
       "\n",
       "                 Summary                                               Text  \n",
       "0  Good Quality Dog Food  I have bought several of the Vitality canned d...  \n",
       "1      Not as Advertised  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  \"Delight\" says it all  This is a confection that has been around a fe...  \n",
       "3         Cough Medicine  If you are looking for the secret ingredient i...  \n",
       "4            Great taffy  Great taffy at a great price.  There was a wid...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.info()\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### About the data\n",
    "\n",
    "- We have a total of 568,454 reviews. The data has been collected from Oct 1999-Oct 2012. All we know is, these reviews are about Food products available at Amazon. Looking at the data, we can use the Summary and Text fields to understand more about what the reviews are about.\n",
    "\n",
    "- Lets perform \"Topic Modelling\" on the reviews, and learn the different topics, that can help us categorize these reviews further.\n",
    "\n",
    "- There doesn't seem to be any missing values here. The Id column is not originally present in the data on the website. This must be added later on, to make it a SQL database. \n",
    "\n",
    "- Columns other than Summary and Text are not relevant for this project since, we only require the textual reviews to learn about the \"topics\" present in the data.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lets start understanding more the data and clean it in the process\n",
    "clean_data = data.drop(columns = [\"Id\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    568454.000000\n",
       "mean        436.222083\n",
       "std         445.339741\n",
       "min          12.000000\n",
       "25%         179.000000\n",
       "50%         302.000000\n",
       "75%         527.000000\n",
       "max       21409.000000\n",
       "Name: Text, dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE9FJREFUeJzt3X+s3XV9x/Hn21awQZECctO0zcpm/7BKRLiBLizmDpZScFlZIkkJWTvWpAkpiUaWWeYfOB0JLEEWmLJ1o7E1TOxU0kaLtUFOzBJ+FcWW0mEv2Mm1DQ22IFcjrvreH+dzyeFyzr2fnlbOPafPR3Jyvuf9/Xy/n/N9c+999XzP9xwiM5EkqcY7ev0EJEn9w9CQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklRtdq+fwMl27rnn5qJFi7ra9pe//CVnnHHGyX1CA8LedGZvOrM3nc203jz11FMvZ+b7phs3cKGxaNEidu3a1dW2jUaDkZGRk/uEBoS96czedGZvOptpvYmI/60Z5+kpSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWB+0T4idjzs1f56/Xf7mrbA7d/7CQ/G0maeXylIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpWlVoRMSBiNgTEU9HxK5SOzsidkbE/nI/t9QjIu6OiNGI2B0RF7XsZ3UZvz8iVrfULy77Hy3bxlRzSJJ643heafxpZl6YmcPl8Xrg4cxcDDxcHgNcBSwut7XAvdAMAOBW4FLgEuDWlhC4t4yd2G75NHNIknrgRE5PrQA2leVNwDUt9c3Z9BhwVkTMA64Edmbmkcw8CuwElpd1Z2bmo5mZwOZJ+2o3hySpB2q/eyqB70ZEAv+WmRuAocw8BJCZhyLivDJ2PvBiy7ZjpTZVfaxNnSnmeJOIWEvzlQpDQ0M0Go3Kw3qzoTlw8wXHutq22zn7xfj4+MAfY7fsTWf2prN+7U1taFyWmQfLH+2dEfE/U4yNNrXsol6thNgGgOHh4RwZGTmezd9wz/1buXNPd9/heOD67ubsF41Gg277OujsTWf2prN+7U3V6anMPFjuDwMP0nxP4qVyaolyf7gMHwMWtmy+ADg4TX1BmzpTzCFJ6oFpQyMizoiI90wsA8uAZ4BtwMQVUKuBrWV5G7CqXEW1FHi1nGLaASyLiLnlDfBlwI6y7rWIWFqumlo1aV/t5pAk9UDNuZgh4MFyFexs4D8z8zsR8SSwJSLWAD8Fri3jtwNXA6PAr4AbADLzSER8HniyjPtcZh4pyzcCXwbmAA+VG8DtHeaQJPXAtKGRmS8AH25T/zlwRZt6Aus67GsjsLFNfRfwodo5JEm94SfCJUnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1apDIyJmRcQPI+Jb5fH5EfF4ROyPiK9FxGmlfnp5PFrWL2rZxy2l/lxEXNlSX15qoxGxvqXedg5JUm8czyuNTwD7Wh7fAdyVmYuBo8CaUl8DHM3M9wN3lXFExBJgJfBBYDnwpRJEs4AvAlcBS4Drytip5pAk9UBVaETEAuBjwH+UxwFcDny9DNkEXFOWV5THlPVXlPErgAcy8/XM/AkwClxSbqOZ+UJm/gZ4AFgxzRySpB6ofaXxz8DfAb8rj88BXsnMY+XxGDC/LM8HXgQo618t49+oT9qmU32qOSRJPTB7ugER8efA4cx8KiJGJspthuY06zrV2wXXVOPbPce1wFqAoaEhGo1Gu2HTGpoDN19wbPqBbXQ7Z78YHx8f+GPslr3pzN501q+9mTY0gMuAv4iIq4F3AWfSfOVxVkTMLq8EFgAHy/gxYCEwFhGzgfcCR1rqE1q3aVd/eYo53iQzNwAbAIaHh3NkZKTisN7qnvu3cueempa81YHru5uzXzQaDbrt66CzN53Zm876tTfTnp7KzFsyc0FmLqL5Rvb3MvN64BHg42XYamBrWd5WHlPWfy8zs9RXlqurzgcWA08ATwKLy5VSp5U5tpVtOs0hSeqBE/mcxqeBT0XEKM33H+4r9fuAc0r9U8B6gMzcC2wBngW+A6zLzN+WVxE3ATtoXp21pYydag5JUg8c17mYzGwAjbL8As0rnyaP+TVwbYftbwNua1PfDmxvU287hySpN/xEuCSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqNm1oRMS7IuKJiPhRROyNiH8o9fMj4vGI2B8RX4uI00r99PJ4tKxf1LKvW0r9uYi4sqW+vNRGI2J9S73tHJKk3qh5pfE6cHlmfhi4EFgeEUuBO4C7MnMxcBRYU8avAY5m5vuBu8o4ImIJsBL4ILAc+FJEzIqIWcAXgauAJcB1ZSxTzCFJ6oFpQyObxsvDd5ZbApcDXy/1TcA1ZXlFeUxZf0VERKk/kJmvZ+ZPgFHgknIbzcwXMvM3wAPAirJNpzkkST1Q9Z5GeUXwNHAY2Ak8D7ySmcfKkDFgflmeD7wIUNa/CpzTWp+0Taf6OVPMIUnqgdk1gzLzt8CFEXEW8CDwgXbDyn10WNep3i64phr/FhGxFlgLMDQ0RKPRaDdsWkNz4OYLjk0/sI1u5+wX4+PjA3+M3bI3ndmbzvq1N1WhMSEzX4mIBrAUOCsiZpdXAguAg2XYGLAQGIuI2cB7gSMt9Qmt27SrvzzFHJOf1wZgA8Dw8HCOjIwcz2G94Z77t3LnnuNqyRsOXN/dnP2i0WjQbV8Hnb3pzN501q+9qbl66n3lFQYRMQf4M2Af8Ajw8TJsNbC1LG8rjynrv5eZWeory9VV5wOLgSeAJ4HF5Uqp02i+Wb6tbNNpDklSD9T8s3oesKlc5fQOYEtmfisingUeiIh/BH4I3FfG3wd8JSJGab7CWAmQmXsjYgvwLHAMWFdOexERNwE7gFnAxszcW/b16Q5zSJJ6YNrQyMzdwEfa1F+geeXT5PqvgWs77Os24LY29e3A9to5JEm94SfCJUnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVK1aUMjIhZGxCMRsS8i9kbEJ0r97IjYGRH7y/3cUo+IuDsiRiNid0Rc1LKv1WX8/ohY3VK/OCL2lG3ujoiYag5JUm/UvNI4BtycmR8AlgLrImIJsB54ODMXAw+XxwBXAYvLbS1wLzQDALgVuBS4BLi1JQTuLWMntlte6p3mkCT1wLShkZmHMvMHZfk1YB8wH1gBbCrDNgHXlOUVwOZsegw4KyLmAVcCOzPzSGYeBXYCy8u6MzPz0cxMYPOkfbWbQ5LUA8f1nkZELAI+AjwODGXmIWgGC3BeGTYfeLFls7FSm6o+1qbOFHNIknpgdu3AiHg38A3gk5n5i/K2Q9uhbWrZRb1aRKyleXqLoaEhGo3G8Wz+hqE5cPMFx7ratts5+8X4+PjAH2O37E1n9qazfu1NVWhExDtpBsb9mfnNUn4pIuZl5qFyiulwqY8BC1s2XwAcLPWRSfVGqS9oM36qOd4kMzcAGwCGh4dzZGSk3bBp3XP/Vu7cU52jb3Lg+u7m7BeNRoNu+zro7E1n9qazfu1NzdVTAdwH7MvML7Ss2gZMXAG1GtjaUl9VrqJaCrxaTi3tAJZFxNzyBvgyYEdZ91pELC1zrZq0r3ZzSJJ6oOaf1ZcBfwXsiYinS+3vgduBLRGxBvgpcG1Ztx24GhgFfgXcAJCZRyLi88CTZdznMvNIWb4R+DIwB3io3JhiDklSD0wbGpn537R/3wHgijbjE1jXYV8bgY1t6ruAD7Wp/7zdHJKk3vAT4ZKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSp2rShEREbI+JwRDzTUjs7InZGxP5yP7fUIyLujojRiNgdERe1bLO6jN8fEatb6hdHxJ6yzd0REVPNIUnqnZpXGl8Glk+qrQcezszFwMPlMcBVwOJyWwvcC80AAG4FLgUuAW5tCYF7y9iJ7ZZPM4ckqUemDY3M/D5wZFJ5BbCpLG8Crmmpb86mx4CzImIecCWwMzOPZOZRYCewvKw7MzMfzcwENk/aV7s5JEk90u17GkOZeQig3J9X6vOBF1vGjZXaVPWxNvWp5pAk9cjsk7y/aFPLLurHN2nEWpqnuBgaGqLRaBzvLgAYmgM3X3Csq227nbNfjI+PD/wxdsvedGZvOuvX3nQbGi9FxLzMPFROMR0u9TFgYcu4BcDBUh+ZVG+U+oI246ea4y0ycwOwAWB4eDhHRkY6DZ3SPfdv5c493bXkwPXdzdkvGo0G3fZ10NmbzuxNZ/3am25PT20DJq6AWg1sbamvKldRLQVeLaeWdgDLImJueQN8GbCjrHstIpaWq6ZWTdpXuzkkST0y7T+rI+KrNF8lnBsRYzSvgrod2BIRa4CfAteW4duBq4FR4FfADQCZeSQiPg88WcZ9LjMn3ly/keYVWnOAh8qNKeaQJPXItKGRmdd1WHVFm7EJrOuwn43Axjb1XcCH2tR/3m4OSVLv+IlwSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUbXavn8CgWLT+211ve+D2j53EZyJJvz++0pAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVK1GR8aEbE8Ip6LiNGIWN/r5yNJp7IZHRoRMQv4InAVsAS4LiKW9PZZSdKpa6Z/IvwSYDQzXwCIiAeAFcCzPX1WJ9mJfJoc/ES5pLfPTA+N+cCLLY/HgEt79FxmLL/CRNLbZaaHRrSp5VsGRawF1paH4xHxXJfznQu83OW2fSnuqB56yvXmONibzuxNZzOtN39QM2imh8YYsLDl8QLg4ORBmbkB2HCik0XErswcPtH9DCJ705m96czedNavvZnRb4QDTwKLI+L8iDgNWAls6/FzkqRT1ox+pZGZxyLiJmAHMAvYmJl7e/y0JOmUNaNDAyAztwPb36bpTvgU1wCzN53Zm87sTWd92ZvIfMv7ypIktTXT39OQJM0ghgan7leVRMSBiNgTEU9HxK5SOzsidkbE/nI/t9QjIu4uPdodERe17Gd1Gb8/Ilb36nhOVERsjIjDEfFMS+2k9SMiLi79Hi3btrukfEbq0JvPRsTPys/P0xFxdcu6W8pxPhcRV7bU2/6ulYtdHi89+1q58GXGi4iFEfFIROyLiL0R8YlSH9yfm8w8pW8032B/HvhD4DTgR8CSXj+vt+nYDwDnTqr9E7C+LK8H7ijLVwMP0fzszFLg8VI/G3ih3M8ty3N7fWxd9uOjwEXAM7+PfgBPAH9ctnkIuKrXx3yCvfks8Ldtxi4pv0enA+eX369ZU/2uAVuAlWX5X4Ebe33MlX2ZB1xUlt8D/Lgc/8D+3PhKo+WrSjLzN8DEV5WcqlYAm8ryJuCalvrmbHoMOCsi5gFXAjsz80hmHgV2Asvf7id9MmTm94Ejk8onpR9l3ZmZ+Wg2/xJsbtnXjNehN52sAB7IzNcz8yfAKM3fs7a/a+VfzpcDXy/bt/Z5RsvMQ5n5g7L8GrCP5jdZDOzPjaHR/qtK5vfoubzdEvhuRDxVPlUPMJSZh6D5CwGcV+qd+jTo/TtZ/ZhflifX+91N5TTLxolTMBx/b84BXsnMY5PqfSUiFgEfAR5ngH9uDI3KryoZUJdl5kU0v0V4XUR8dIqxnfp0qvbvePsxiH26F/gj4ELgEHBnqZ9yvYmIdwPfAD6Zmb+YamibWl/1xtCo/KqSQZSZB8v9YeBBmqcPXioviSn3h8vwTn0a9P6drH6MleXJ9b6VmS9l5m8z83fAv9P8+YHj783LNE/TzJ5U7wsR8U6agXF/Zn6zlAf258bQOEW/qiQizoiI90wsA8uAZ2ge+8SVG6uBrWV5G7CqXP2xFHi1vOzeASyLiLnl9MSyUhsUJ6UfZd1rEbG0nMNf1bKvvjTxR7H4S5o/P9DszcqIOD0izgcW03wzt+3vWjlX/wjw8bJ9a59ntPLf8j5gX2Z+oWXV4P7c9PJd+Jlyo3lFw49pXtnxmV4/n7fpmP+Q5tUrPwL2Thw3zfPLDwP7y/3ZpR40/4dYzwN7gOGWff0NzTc7R4Eben1sJ9CTr9I8zfJ/NP+Ft+Zk9gMYpvmH9XngXygfru2HW4fefKUc+26afwzntYz/TDnO52i52qfT71r5eXyi9Oy/gNN7fcyVffkTmqeLdgNPl9vVg/xz4yfCJUnVPD0lSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKna/wMNtFFoTmP6fwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#How long are our reviews?\n",
    "clean_data.Text.apply(len).hist(bins =20)\n",
    "clean_data.Text.apply(len).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mostly, reviews extend from a few lines to short paragraphs. There are a few very long reviews that might extend to several pages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Fuzzy Wuzzy's Summary:<br />*** Somewhat recommended, with reservations and only lukewarm fuzzies.<br /><br />Positives:<br /><br />+ Not too\",\n",
       " \"expensive compared to other bottled water, although it is not cheap either.<br /><br />Negatives:<br /><br />- This bottled water product's labeling,\",\n",
       " 'product descriptions, and health claims are misleading.<br /><br />At first glance, this 1.5-liter bottle of water that I am reviewing looks like',\n",
       " 'another one of the many varieties of bottled water on the market.  There are many kinds of bottled water: \"spring water\" produced from an underground',\n",
       " 'formation where the groundwater naturally flows up to the Earth\\'s surface, \"artesian water\" obtained from a well that taps into an underground',\n",
       " 'aquifer, \"mineral water\" produced from a well or spring that naturally contains trace amounts of minerals that some people consider to provide health',\n",
       " 'benefits, \"ground/well water\" produced from a well that tapes into an aquifer, \"sparkling water\" from a spring or well that is naturally carbonated,',\n",
       " 'and \"purified water\" produced by distillation, deionization, reverse osmosis, or other purification processes.  This Essentia Drinking Water is',\n",
       " '\"purified water\"; this bottle contains municipal water (i.e. tap water that comes from your faucet) that is purified using reverse osmosis and',\n",
       " 'ultraviolet light for disinfection.<br /><br />The label on this Essentia water bottle is filled with marketing hype and technical terms designed to',\n",
       " 'make it sound exotic:<br />* \"Infused with Electrolytes For A Difference You Can Taste\"<br />* \"Silky Smooth Taste\"<br />* \"Hydrates Better\"<br />*',\n",
       " '\"Electrolytes Infused\"<br />* \"Best Available pH at 9.5\"<br />* \"Doctor recommended to protect, improve, and enhance the quality of your health.\"<br',\n",
       " '/>* \"Our Proprietary Technology: Water purified by reverse osmosis; infused with electrolytes for taste; alkalinized and restructured using Ionic',\n",
       " 'Separation Technology.\"<br />* \"Ingredients: purified water, sodium bicarbonate, dipotassium phosphate, magnesium sulfate, calcium chloride.\"<br /><br',\n",
       " '/>There are two ways to evaluate this water product.  One way to review this water is based on how it tastes, how safe are its added ingredients, and',\n",
       " \"its cost.  Since Essentia includes a variety of health claims and marketing statements on the water bottle's label, on the Amazon product listing, and\",\n",
       " \"on the company's Web site, the veracity of this statements must also be scrutinized since the marketing hype of most other brands of bottled water is\",\n",
       " 'just limited to the \"freshness\" or \"purity\" of their taste without making other health-related claims.<br /><br />First, on the basis on its cost,',\n",
       " 'with all of its marketing hype and hoopla, at least this water is not priced far more expensive than most other brands of bottled water.  But there',\n",
       " 'are various other bottled water products on Amazon that cost less on a per-liter basis.  And if you think that this slightly pricey water can be',\n",
       " 'justified based on its health \"merits\", keep reading...<br /><br />As for taste, that can be a highly personal preference, but I like my water to',\n",
       " 'simply taste \"wet\".  :-)  If I want some kind of flavored drink - tea, wine, beer, fruit juice, etc. - I will drink that instead.  But nothing',\n",
       " 'hydrates and quenches as easily as pure and simple water, with no additives or flavorings added to it.  This particular Essentia water does have a',\n",
       " 'smooth texture to it.  Most other bottled waters taste more crisp than this Essentia water.  Again, you may either like the smooth texture of this',\n",
       " 'water or you may think that its texture is rather flat and bland.  Unlike drinking \"distilled water\", or bottled water that was purified using',\n",
       " 'distillation with no additives, this water has a very slight subtle aftertaste that might be considered as either \"chalky\" or \"metallic\".  Those are',\n",
       " 'my highly subjective adjectives that I associate with this water.  I can only surmise that this is due to the small amounts of additives that were',\n",
       " 'used in this water.  This aftertaste is not too objectionable, but if I just want to hydrate myself drinking \"water\", I really prefer my water to be',\n",
       " \"without any identifiable taste, texture, or odor.<br /><br />Next, since this is not just a bottle of pure H2O water, let's consider what was added to\",\n",
       " 'this water: sodium bicarbonate, dipotassium phosphate, magnesium sulfate, and calcium chloride.  Sodium bicarbonate is the same thing as baking soda.',\n",
       " 'This water was made to have a slightly alkaline pH of 9.5 by adding sodium bicarbonate.  People sometimes also adjust and raise the pH of their',\n",
       " 'swimming pool water by adding sodium bicarbonate.  Dipotassium phosphate is sometimes added to various food products as a buffering agent or',\n",
       " 'emulsifier.  In this water, it serves as one of the \"electrolytes\" that is added.  The term \"electrolytes\" is just a subset of minerals that are found',\n",
       " 'in both the Earth and the food that we eat.  Also serving as an electrolyte is the addition of magnesium sulfate.  Magnesium sulfate is sold in drug',\n",
       " 'stores as \"Epsom Salt\", and you can either soak your feet or body in an Epsom Salt bath to soothe sore muscles, or when ingested in larger quantities,',\n",
       " 'magnesium sulfate also can serve as a laxative.  Calcium chloride serves as another electrolyte in this water.  Calcium chloride is sometimes used as',\n",
       " 'a food additive.  The trace amounts of calcium, magnesium, and phosphorus in this water do not qualify as a measurable part of your daily nutritional',\n",
       " 'intake of these minerals, regardless of how much of this water you drink each day.<br /><br />Essentia is definitely not the first bottled water',\n",
       " 'manufacturer to add electrolytes to either bottled water or drinks.  During the 1960s, Gatorade was possibly the first large-scale commercial drink',\n",
       " 'that was produced with the goal of adding electrolytes of sodium, sugar, potassium, and phosphate to hydrate and replenish athletes who lost fluids',\n",
       " 'during physical exertion in hot weather.  On Essentia\\'s Web site, they have an FAQ question/answer of: \"Why should I drink Essentia if I get my',\n",
       " \"electrolytes through sports drinks?  Essentia provides a blend of dissolved minerals that emulate the ratio of the body's own mixture and\",\n",
       " 'concentration - providing the correct proportion of electrolytes without all the sugars and other additives you typically find in sports and energy',\n",
       " 'drinks.\"  There are several problems with that statement.  First, if you are actively exercising and sweating, your body can actually use those extra',\n",
       " 'calories, and if you are just sitting in front of the computer or television, guzzling Gatorade just adds useless calories and electrolytes.  Second,',\n",
       " 'if you are drinking Essentia at home, this product\\'s claim that it provides \"the correct proportion of electrolytes\" implies a one-size-fits-all',\n",
       " 'balance of electrolytes, which is simply not true as we all have different bodies, diets, and lifestyles.<br /><br />None of what Essentia refers to',\n",
       " 'as \"Our Proprietary Technology\" is really that \"proprietary\" or exotic.  The reverse osmosis purification of municipal tap water is commonly used to',\n",
       " 'produce bottled water, and the addition of various electrolytes or minerals to water has been practiced by other bottled water manufacturers for many',\n",
       " 'decades now.  Some people think that drinking plain distilled water tastes boring and, mentally at least, they like the idea that drinking water that',\n",
       " 'is naturally infused with minerals from a spring or well/aquifer adds an extra measure of health benefits.  The addition of trace minerals do add',\n",
       " 'noticeable flavor that mimics natural spring water.  Also, slightly flavoring the water with minerals masks the residual odors that can be left by the',\n",
       " 'plastic water bottle; the plastic odor is still there, but you notice it less.  The same principle applies to using a room deodorizer to mask the',\n",
       " 'existing odors in a house; that old cat urine smell is still present inside the house, but you notice it less and therefore the house smells fresher.',\n",
       " 'The most common electrolytes/minerals in spring water are calcium, magnesium, potassium, sodium, sulfate, and chloride.  The Coca-Cola company markets',\n",
       " 'a \"Dasani\" bottled water that, like this Essentia, is municipal tap water purified via reverse osmosis.  Dasani has magnesium sulfate and potassium',\n",
       " 'chloride added to it.  Perhaps the Dasani taste testing panels decided that their water tastes better without the addition of calcium?  But for',\n",
       " 'whatever reasons, this Essentia water has a slight residual \"chalky\" or \"metallic\" aftertaste, as if it had a tannic undertone like the aftertaste',\n",
       " 'after drinking a dry red wine, that I do not really care for.  I do not want my drinking water to leave a subtle aftertaste in my mouth.<br /><br',\n",
       " '/>Essentia also competes directly with \"Glaceau Smart Water\", which takes an approach similar to Essentia\\'s water.  Glaceau Smart Water is municipal',\n",
       " 'water purified via distillation, and then infused with the similar ingredients of calcium chloride, magnesium chloride, and potassium bicarbonate.<br',\n",
       " '/><br />Not to be outdone, another of Essentia\\'s alkaline/electrolyte water competitors, named \"NEO Water\" (to ride on the futuristic coattails of',\n",
       " '\"The Matrix\" fame??), pumps out even more junk science drivel filled with fear-mongering misleading statements, calling itself \"Alkaline, Electrolyte,',\n",
       " 'Antioxidant Super Hydrator ... ultra purified healthy and refreshing water formulated through proprietary technology to balance, revitalize and propel',\n",
       " 'you unlike any traditional water\", and \"NEO\\'s unique antioxidants neutralize free radicals the body releases when under exertion\".  But when you',\n",
       " \"really dig further into NEO Water's claims, there are *NO* antioxidants in NEO Water... just potassium bicarbonate, calcium chloride, magnesium\",\n",
       " 'chloride, which is similar to the mundane chemicals added to Essentia and Dasani water.  NEO Water\\'s marketing nonsense says that its \"alkalinity',\n",
       " 'allows its molecules to carry more oxygen than normal with an extra attached electron (hydroxyl ions). As a result, these molecules are able to seek',\n",
       " 'and neutralize free radicals in your body and prevent free radical damage to your cells. These natural antioxidant benefits leave your body rich in',\n",
       " 'oxygen and energy.\"  Again, like Essentia, NEO Water hopes to blind and persuade the consumer by overwhelming you with exotic totally false techno-',\n",
       " 'babble; it ascribes its antioxidant benefits entirely to its 9.5 pH alkalinity.  So if you really believe in this nonsense, you could, in theory,',\n",
       " \"obtain far more amazing antioxidant protection simply by dissolving an antacid tablet into your tap water once a day.  As with Essentia's water, NEO\",\n",
       " 'Water\\'s \"ultra purified\" water is simply municipal tap water purified using the common method of reverse osmosis.<br /><br />This Essentia Drinking',\n",
       " 'Water is not the first product to tout the benefits of \"alkaline water\".  There have been previous highly-hyped products based on ionized water,',\n",
       " 'alkaline water, and combination alkaline-ionized water.  There have even been so-called \"liquid oxygen\" drops that supposedly contained \"O4\" molecules',\n",
       " '(instead of the \"O2\" oxygen that is in the air) dissolved in a liquid, and by ingesting 10 drops each day, you feel more energized, but this quackery',\n",
       " 'completely ignores the fact that oxygen is best utilized in the lungs, and not in the stomach.  Just as the \"liquid oxygen\" drops are a distorted',\n",
       " 'version of twisted junk science based on the concept of oxygen bars (at least in oxygen bars, you do breathe in the oxygen instead of putting drops of',\n",
       " 'useless oxygen in your stomach), this \"alkaline water\" concept is a distortion of the \"alkaline diet\" alternative medicine belief that the foods we',\n",
       " 'eat can affect the acidity of urine and blood, which can then affect our well-being, susceptibility to disease, and energy level.  If you are an avid',\n",
       " 'vegan/vegetarian practitioner of an alkaline diet, then this 9.5-pH alkaline water can keep your diet on its alkaline-based theme.  But if you',\n",
       " 'regularly eat meats, poultry, seafood, dairy products, eggs, grains, processed sugar foods, and colas/sodas, all of which significantly boost the',\n",
       " 'acidic levels of your food consumption, then drinking a slightly alkaline water makes absolutely no difference to your health.<br /><br />Your stomach',\n",
       " 'has a healthily acidic pH that ranges between 1.45 to 3.5; your stomach must be very acidic to aid in digestion.  However, blood must always be',\n",
       " \"slightly alkaline, with a pH of 7.35 to 7.45.  If the blood's pH drops below 6.8 (more acidic) or rises above 7.8 (more alkaline), death may occur.\",\n",
       " 'But your blood automatically stays within its healthy pH range regardless of what you eat or drink because your blood utilizes several buffering',\n",
       " 'systems.  The most important buffer for maintaining acidity/alkalinity balance in the blood is called the carbonic-acid-bicarbonate buffer.  To form',\n",
       " 'the bicarbonate in this buffering system, your body uses carbon dioxide... guess what, carbon dioxide is in (over)abundance in the air that you',\n",
       " 'breathe and you do not need sodium bicarbonate baking soda added to your drinking water to maintain a healthy pH in your body.  In fact, consuming too',\n",
       " 'much supplemental sodium bicarbonate can actually lead to constipation, convulsions, diarrhea, frequent urination, and vomiting.  There is only a tiny',\n",
       " 'amount of sodium bicarbonate in each bottle of Essentia water, so you do not have to worry about overdose.  But my point is that all this hoopla and',\n",
       " 'hype about this water being of the \"Best Available pH at 9.5\" is purified nonsense.  Some other fear-mongering alkaline water sales gimmicks claim',\n",
       " 'that, to quote another Web site, \"Excess acidity weakens all our systems and forces the body to take minerals from vital organs and bones to',\n",
       " 'buffer/neutralize the acid and safely remove it from the body.\"  But this is totally false fear-based junk science contrived to mislead you into',\n",
       " \"thinking that your body will rob its vital organs and bones of minerals to neutralize excess acidity.  But our body's carbonic-acid-bicarbonate buffer\",\n",
       " 'simply combines carbon dioxide (CO2) in the air with water to make the blood plasma more acidic if it needs to, and it can prevent excess plasma',\n",
       " 'acidity and raise its pH to be more alkaline by having the lungs convert the bicarbonate back into CO2 which you then breathe out.  A similar',\n",
       " 'phosphate buffering system also helps to maintain the pH of cellular fluids, using phosphate instead of bicarbonate.  Phosphorus is present in nearly',\n",
       " 'all foods, and regardless of whether you like to eat lots of meats or lots of vegetables, you can get plenty of sufficient phosphorus in your diet',\n",
       " 'without needing to add it to your drinking water!<br /><br />Drinking too many acidic drinks such as sodas and colas can have a detrimental effect of',\n",
       " 'eroding your tooth enamel, but guzzling all those colas have absolutely no effect on the pH of your cells and bodily fluids.  And regardless of how',\n",
       " 'much alkalinity you consume and put in your stomach, your body is going to make sure that your stomach stays within a healthy acidic pH range and your',\n",
       " 'body will use its pH buffering systems to keep your blood pH between 7.35 and 7.45 regardless of how much acidic or alkaline food and drink you',\n",
       " 'consume. If you are familiar with logarithms, also realize that the pH scale is logarithmic (pH is not a linear scale) and drinking 9.5 pH water',\n",
       " 'really has no effect on your various bodily fluids compared with drinking tap water or any other bottled water. And if you really could somehow',\n",
       " \"artifically adjust your stomach, blood, and bodily fluids entirely to a 9.5 pH, you would be immediately wheeled into a hospital's emergency room\",\n",
       " 'because your body and its stomach cannot handle that kind of alkalinity.<br /><br />If you suffer from frequent heartburn or acid reflux disease',\n",
       " '(gastroesophageal reflux disease), drinking this 9.5 pH water will NOT help the least bit.  Your stomach acids need to constantly stay very acidic to',\n",
       " 'maintain proper digestive functions, and the slight alkalinity of this water has no effect on the stomach acid that is entering into the esophagus of',\n",
       " 'people with GERD.<br /><br />Trying to market bottled water based on its \"silky smooth taste\" and \"hydrates better\" claims, its over-hyped \"9.5 pH\"',\n",
       " 'attribute, and all its overexaggerated health claims constitute marketing trickery to fool consumers by utilizing the \"if you believe that it can',\n",
       " 'really work, it does indeed work\" placebo effect.  The placebo effect is a very powerful phenomenon that can frequently affect everything from',\n",
       " 'clinical trials of pharmacological substances to faith healing.  Basically, if you have a hopeful mindset of expectancy and conditioning that makes',\n",
       " 'you think that something will work, then that perception can become 100% of your subjectively observed reality and outcome of the treatment.  For',\n",
       " 'examples of other placebo effects: If I told you that you can get stronger bones by breathing milk through your nose instead of drinking it through',\n",
       " 'your mouth, or if I told you that you could be revitalized by just holding broccoli in your hands for two minutes every day instead of eating it, if',\n",
       " 'you really believe or hope that to be true, then after breathing the milk into your nose or holding the broccoli in your hands, you may think to',\n",
       " 'yourself, \"Gee, I feel better, stronger, and more energetic after having done that!\"  Likewise, if you really HOPE to feel more hydrated after',\n",
       " 'drinking this Essentia water, you may actually feel more refreshed simply because you BELIEVE that this water is hydrating you better than drinking',\n",
       " \"other sources of water, when in fact the tap water from your faucet could have hydrated you just as refreshingly.<br /><br />The Amazon 'Product\",\n",
       " 'Description\\' for this Essentia water falsely states even more overexaggerated health claims: \"Essentia Water provides unmatched hydration, health',\n",
       " 'benefits and smooth taste.  Its superior hydrating qualities come from a special electrolyte formula and optimal pH level of 9.5, which gives your',\n",
       " 'body more of what it needs to thrive.  Drinking Essentia Water boosts the antioxidant properties of your immune system and helps bring your body back',\n",
       " 'into balance.  It also helps maintain normal blood pressure, restful sleep, heart health, muscle strength and more.\"  Again... these claims are way',\n",
       " \"wayyyy overexaggerated!  Compared to your overall health, fitness level, and daily food diet, the trace minerals found in this water and this water's\",\n",
       " 'slight alkalinity really make no difference in boosting the \"antioxidant properties of your immune system\", nor does it \"help maintain normal blood',\n",
       " 'pressure, restful sleep, heart health, muscle strength and more\".  With this litany of false health claims, I am surprised that they also did not pile',\n",
       " 'on claims that this water \"increases libido, cures erectile dysfunction, grows hair on balding men, makes women sexy, and fixes the national debt\"',\n",
       " \"too.<br /><br />Fuzzy Wuzzy's Conclusion:<br /><br />I must admit that I personally do not regularly purchase bottled water for home use, even though\",\n",
       " 'I will sometimes buy it while I am away from home, both because I think that filtering my tap water through a Brita filter produces water that is just',\n",
       " 'as tasty and healthy as bottled water and, just as important, I think that too much plastic (and petroleum) is used in the production of bottled water',\n",
       " 'and most plastic bottles end up in our overflowing landfills.  Roughly 1.5 million tons of plastic is used to make water bottles.  And this does not',\n",
       " 'include the energy used to make the plastic and the oil used for transporting all the bottled water.  It takes about two liters of water used during',\n",
       " 'the production process just to make one liter of bottled water.  So behind each 1-liter bottle of water is another two liters of water that was',\n",
       " 'expended during its production.  Essentia states that their plastic bottles are \"safe and recyclable\", but we can all reduce our carbon footprint if',\n",
       " 'we drink less bottled water.  In many developing countries, tap water is either unavailable or unsafe, making bottled water a better option if',\n",
       " 'residents cannot boil their tap water before consumption.  But here in the U.S., some bottled water manufacturers, including this Essentia bottled',\n",
       " 'water, use fear-based advertising claims that their water is healthier than tap water, which it really is not.  The concerns for the taste of tap',\n",
       " 'water are valid, however, especially with tap water that has residual smell or taste of chlorination.  But residual smells and taste issues can be',\n",
       " 'easily and economically remedied by using various activated charcoal/carbon filters at home to treat your tap water.  Anyway, the tap water that comes',\n",
       " 'out of my faucet at home is also slightly alkaline like this Essentia water, and many municipal water supplies are also slightly alkaline, so there is',\n",
       " 'no need to purchase this bottled water that has baking soda added to it.<br /><br />This is not a bad bottled water.  It is priced competitively with',\n",
       " \"other bottled waters.  But it has an odd aftertaste that I do not like.  And the techno-jargon claims that revolve around this water's selling points\",\n",
       " '- 9.5 pH, \"ionically separated water\", \"Infused With Electrolytes\", and even falsely stating that this water boosts your immune system - amount to',\n",
       " 'nothing but purified snake oil hogwash.  To quote William Shakespeare, all this marketing hype about 9.5 pH and electrolytes is really \"much ado about',\n",
       " 'nothing\"!']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Longest review\n",
    "import textwrap as tw\n",
    "tw.wrap(clean_data.loc[clean_data.Text.apply(len).idxmax(clean_data.Text.apply(len))].Text,width =150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- I see a lot of line breakers read in as text. We need to deal with this. Also, there appears words that use appostrophe. Such words will create a problem later on. We need to take care of such words early."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Duplicated Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "281"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_data.duplicated().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have around 281 duplicated reviews. Since, we dont have a unique identifer in the dataset, lets use userID, time, summary and text to identify all the dupicated entries in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "173282"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_data[[\"UserId\",\"ProfileName\",\"Time\",\"Summary\",\"Text\"]].duplicated().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Woah! Thats quite a lot of duplicated reviews. Almost 30% of our reviews are duplicated. We need to remove these reviews in order to indentify the true topics in the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(568454, 5)"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_data = clean_data[[\"UserId\",\"ProfileName\",\"Time\",\"Summary\",\"Text\"]]\n",
    "clean_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_data.drop_duplicates(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(395172, 5)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3SGXH7AUHU8GW</td>\n",
       "      <td>delmartian</td>\n",
       "      <td>1303862400</td>\n",
       "      <td>Good Quality Dog Food</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1D87F6ZCVE5NK</td>\n",
       "      <td>dll pa</td>\n",
       "      <td>1346976000</td>\n",
       "      <td>Not as Advertised</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABXLMWJIXXAIN</td>\n",
       "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
       "      <td>1219017600</td>\n",
       "      <td>\"Delight\" says it all</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A395BORC6FGVXV</td>\n",
       "      <td>Karl</td>\n",
       "      <td>1307923200</td>\n",
       "      <td>Cough Medicine</td>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1UQRSCLF8GW1T</td>\n",
       "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
       "      <td>1350777600</td>\n",
       "      <td>Great taffy</td>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           UserId                      ProfileName        Time  \\\n",
       "0  A3SGXH7AUHU8GW                       delmartian  1303862400   \n",
       "1  A1D87F6ZCVE5NK                           dll pa  1346976000   \n",
       "2   ABXLMWJIXXAIN  Natalia Corres \"Natalia Corres\"  1219017600   \n",
       "3  A395BORC6FGVXV                             Karl  1307923200   \n",
       "4  A1UQRSCLF8GW1T    Michael D. Bigham \"M. Wassir\"  1350777600   \n",
       "\n",
       "                 Summary                                               Text  \n",
       "0  Good Quality Dog Food  I have bought several of the Vitality canned d...  \n",
       "1      Not as Advertised  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  \"Delight\" says it all  This is a confection that has been around a fe...  \n",
       "3         Cough Medicine  If you are looking for the secret ingredient i...  \n",
       "4            Great taffy  Great taffy at a great price.  There was a wid...  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sanity check\n",
    "clean_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning text data\n",
    "\n",
    "- We noticed a lot of html tags and grammer contractions in the text - removing them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "#correcting grammar contractions\n",
    "grammer = {\"can't\": \"can not\",\"won't\": \" will not\",\"n't\":\" not\", \"'s\":\" is\",\"'ve\":\" have\",\"'re\":\" are\",\"'ll\":\" will\",\"'m\":\" am\",\"'d\":\" had\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing html tags\n",
    "TAG_RE = re.compile(r'<[^>]+>')\n",
    "def remove_tags(text):\n",
    "    return TAG_RE.sub('', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to remove html tags and replace grammer contractions from text\n",
    "def clean_review(text):\n",
    "    text = remove_tags(text)\n",
    "    for key in grammer.keys():\n",
    "        text = text.replace(key,grammer[key])\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time taken to clean 00:00:03.98\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "clean_data.Text = clean_data.Text.apply(clean_review)\n",
    "clean_data.Summary = clean_data.Summary.apply(clean_review)\n",
    "end = time.time()\n",
    "#time taken to remove non-frequent words\n",
    "hours, rem = divmod(end-start, 3600)\n",
    "minutes, seconds = divmod(rem, 60)\n",
    "print(\"Total time taken to clean\",\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>ProfileName</th>\n",
       "      <th>Time</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A3SGXH7AUHU8GW</td>\n",
       "      <td>delmartian</td>\n",
       "      <td>1303862400</td>\n",
       "      <td>Good Quality Dog Food</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1D87F6ZCVE5NK</td>\n",
       "      <td>dll pa</td>\n",
       "      <td>1346976000</td>\n",
       "      <td>Not as Advertised</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABXLMWJIXXAIN</td>\n",
       "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
       "      <td>1219017600</td>\n",
       "      <td>\"Delight\" says it all</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A395BORC6FGVXV</td>\n",
       "      <td>Karl</td>\n",
       "      <td>1307923200</td>\n",
       "      <td>Cough Medicine</td>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1UQRSCLF8GW1T</td>\n",
       "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
       "      <td>1350777600</td>\n",
       "      <td>Great taffy</td>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           UserId                      ProfileName        Time  \\\n",
       "0  A3SGXH7AUHU8GW                       delmartian  1303862400   \n",
       "1  A1D87F6ZCVE5NK                           dll pa  1346976000   \n",
       "2   ABXLMWJIXXAIN  Natalia Corres \"Natalia Corres\"  1219017600   \n",
       "3  A395BORC6FGVXV                             Karl  1307923200   \n",
       "4  A1UQRSCLF8GW1T    Michael D. Bigham \"M. Wassir\"  1350777600   \n",
       "\n",
       "                 Summary                                               Text  \n",
       "0  Good Quality Dog Food  I have bought several of the Vitality canned d...  \n",
       "1      Not as Advertised  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  \"Delight\" says it all  This is a confection that has been around a fe...  \n",
       "3         Cough Medicine  If you are looking for the secret ingredient i...  \n",
       "4            Great taffy  Great taffy at a great price.  There was a wid...  "
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sanity check\n",
    "clean_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unique Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "355205"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#How many unique words are there?\n",
    "import itertools\n",
    "words = list(itertools.chain(*list(clean_data.Text.apply(word_tokenize))))\n",
    "len(set(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Right now, we have 355,205 unique tokens. But this also includes punctuations and other stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD8CAYAAACyyUlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGClJREFUeJzt3X+wX3V95/Hny0QsapVfV5YmoYk1bReY7ghXjLU6rLQQtGvoLIygU7I2O5l18UdLdzSsO4MjZQZqZ9kyVXZYkyU4DphhrWQ0mM2Aym7Lj1wEgYgsd5CFW1ACiSyWEYS+94/v5+qXy/fem9yTmy+B52PmO99z3udzzvt8mct95fz43pOqQpKkLl417B2QJB34DBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjozTCRJnRkmkqTOFg57B/aXI444opYuXTrs3ZCkA8rtt9/+eFWNzDbuFRMmS5cuZWxsbNi7IUkHlCT/d0/GeZpLktSZYSJJ6swwkSR1ZphIkjozTCRJnRkmkqTODBNJUmeGiSSpM8NEktTZK+Yb8F0sXfeNee/x4MXvm/cekjRfPDKRJHVmmEiSOjNMJEmdzRomSTYkeSzJPVPqH0tyX5IdSf6yr35+kvG27NS++spWG0+yrq++LMmtSe5P8pUkB7X6a9r8eFu+dLYekqTh2JMjkyuBlf2FJP8SWAX8TlUdC/xVqx8DnAUc29b5QpIFSRYAnwdOA44Bzm5jAS4BLq2q5cBuYE2rrwF2V9VbgEvbuGl77P1HlyTtK7OGSVXdBOyaUv4IcHFVPdPGPNbqq4BrquqZqvohMA6c2F7jVfVAVT0LXAOsShLgPcC1bf2NwOl929rYpq8FTm7jp+shSRqSuV4z+U3gXe3003eSvK3VFwEP942baLXp6ocDP6mq56bUX7CttvzJNn66bUmShmSu3zNZCBwKrADeBmxK8mYgA8YWg0OrZhjPDMtmWucFkqwF1gIcffTRg4ZIkvaBuR6ZTABfrZ7bgH8Cjmj1JX3jFgOPzFB/HDgkycIpdfrXacvfSO9023TbepGquqKqRqtqdGRk1kcYS5LmaK5h8jV61zpI8pvAQfSCYTNwVrsTaxmwHLgN2A4sb3duHUTvAvrmqirgW8AZbburgeva9OY2T1t+Yxs/XQ9J0pDMeporydXAScARSSaAC4ANwIZ2u/CzwOr2i35Hkk3A94HngHOr6vm2nY8CW4EFwIaq2tFafAq4JslfAHcA61t9PfClJOP0jkjOAqiqaXtIkoYjvQx4+RsdHa2xsbE5revf5pL0SpXk9qoanW2c34CXJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqbNYwSbIhyWPtqYpTl/2HJJXkiDafJJclGU9yV5Lj+8auTnJ/e63uq5+Q5O62zmVJ0uqHJdnWxm9LcuhsPSRJw7EnRyZXAiunFpMsAf4AeKivfBq9Z7IvB9YCl7exh9F73O/bgROBCybDoY1Z27feZK91wA1VtRy4oc1P20OSNDyzhklV3UTvGexTXQp8Euh/7u8q4KrquQU4JMlRwKnAtqraVVW7gW3AyrbsDVV1c3uG/FXA6X3b2timN06pD+ohSRqSOV0zSfJ+4B+q6ntTFi0CHu6bn2i1meoTA+oAR1bVowDt/U2z9JAkDcnCvV0hyWuBTwOnDFo8oFZzqM+4C3u6TpK19E6FcfTRR8+yWUnSXM3lyOQ3gGXA95I8CCwGvpvkn9E7SljSN3Yx8Mgs9cUD6gA/njx91d4fa/XptvUiVXVFVY1W1ejIyMhefkxJ0p7a6zCpqrur6k1VtbSqltL75X58Vf0I2Ayc0+64WgE82U5RbQVOSXJou/B+CrC1LXsqyYp2F9c5wHWt1WZg8q6v1VPqg3pIkoZk1tNcSa4GTgKOSDIBXFBV66cZvgV4LzAOPA18GKCqdiW5ENjexn22qiYv6n+E3h1jBwPXtxfAxcCmJGvo3TF25kw9JEnDM2uYVNXZsyxf2jddwLnTjNsAbBhQHwOOG1B/Ajh5QH3aHpKk4fAb8JKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLUmWEiSerMMJEkdWaYSJI6M0wkSZ3NGiZJNiR5LMk9fbXPJflBkruS/G2SQ/qWnZ9kPMl9SU7tq69stfEk6/rqy5LcmuT+JF9JclCrv6bNj7flS2frIUkajj05MrkSWDmltg04rqp+B/g/wPkASY4BzgKObet8IcmCJAuAzwOnAccAZ7exAJcAl1bVcmA3sKbV1wC7q+otwKVt3LQ99vJzS5L2oT15bO9N/UcFrfY/+2ZvAc5o06uAa6rqGeCHScaBE9uy8ap6ACDJNcCqJPcC7wE+2MZsBD4DXN629ZlWvxb4mySZocfNe/aRDyxL131j3ns8ePH75r2HpJe3fXHN5E+A69v0IuDhvmUTrTZd/XDgJ1X13JT6C7bVlj/Zxk+3LUnSkHQKkySfBp4DvjxZGjCs5lCfy7YG7d/aJGNJxnbu3DloiCRpH5hzmCRZDfwh8KGqmvxlPgEs6Ru2GHhkhvrjwCFJFk6pv2BbbfkbgV0zbOtFquqKqhqtqtGRkZG5fExJ0h6YU5gkWQl8Cnh/VT3dt2gzcFa7E2sZsBy4DdgOLG93bh1E7wL65hZC3+KX11xWA9f1bWt1mz4DuLGNn66HJGlIZr0An+Rq4CTgiCQTwAX07t56DbCtd02cW6rq31XVjiSbgO/TO/11blU937bzUWArsADYUFU7WotPAdck+QvgDmB9q68HvtQusO+iF0DM1EOSNBx7cjfX2QPK6wfUJsdfBFw0oL4F2DKg/gC/vOOrv/4z4My96SFJGg6/AS9J6swwkSR1ZphIkjqb9ZqJXtnm+xv4fvteennwyESS1JlhIknqzDCRJHXmNRO9ZHm9RjpweGQiSerMMJEkdWaYSJI6M0wkSZ0ZJpKkzgwTSVJnhokkqTPDRJLU2axhkmRDkseS3NNXOyzJtiT3t/dDWz1JLksynuSuJMf3rbO6jb+/PT9+sn5CkrvbOpelPbpxLj0kScOxJ0cmVwIrp9TWATdU1XLghjYPcBq9Z7IvB9YCl0MvGOg97vft9J6qeMFkOLQxa/vWWzmXHpKk4Zk1TKrqJnrPYO+3CtjYpjcCp/fVr6qeW4BDkhwFnApsq6pdVbUb2AasbMveUFU3V1UBV03Z1t70kCQNyVyvmRxZVY8CtPc3tfoi4OG+cROtNlN9YkB9Lj0kSUOyry/AZ0Ct5lCfS48XD0zWJhlLMrZz585ZNitJmqu5hsmPJ08ttffHWn0CWNI3bjHwyCz1xQPqc+nxIlV1RVWNVtXoyMjIXn1ASdKem2uYbAYm78haDVzXVz+n3XG1AniynaLaCpyS5NB24f0UYGtb9lSSFe0urnOmbGtvekiShmTW55kkuRo4CTgiyQS9u7IuBjYlWQM8BJzZhm8B3guMA08DHwaoql1JLgS2t3GfrarJi/ofoXfH2MHA9e3F3vaQJA3PrGFSVWdPs+jkAWMLOHea7WwANgyojwHHDag/sbc9JEnD4TfgJUmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ7P+bS7plWjpum/M6/YfvPh987p9aX/zyESS1JlhIknqzDCRJHVmmEiSOusUJkn+LMmOJPckuTrJryRZluTWJPcn+UqSg9rY17T58bZ8ad92zm/1+5Kc2ldf2WrjSdb11Qf2kCQNx5zDJMki4OPAaFUdBywAzgIuAS6tquXAbmBNW2UNsLuq3gJc2saR5Ji23rHASuALSRYkWQB8HjgNOAY4u41lhh6SpCHoemvwQuDgJD8HXgs8CrwH+GBbvhH4DHA5sKpNA1wL/E2StPo1VfUM8MMk48CJbdx4VT0AkOQaYFWSe2foIR3w5vu2ZPDWZO17cz4yqap/AP4KeIheiDwJ3A78pKqea8MmgEVtehHwcFv3uTb+8P76lHWmqx8+Q48XSLI2yViSsZ07d871o0qSZtHlNNeh9I4qlgG/BryO3impqWpylWmW7av6i4tVV1TVaFWNjoyMDBoiSdoHulyA/33gh1W1s6p+DnwV+F3gkCSTp88WA4+06QlgCUBb/kZgV399yjrT1R+foYckaQi6hMlDwIokr23XPk4Gvg98CzijjVkNXNemN7d52vIbq6pa/ax2t9cyYDlwG7AdWN7u3DqI3kX6zW2d6XpIkoagyzWTW+ldSP8ucHfb1hXAp4Dz2oX0w4H1bZX1wOGtfh6wrm1nB7CJXhB9Ezi3qp5v10Q+CmwF7gU2tbHM0EOSNASd7uaqqguAC6aUH+CXd2P1j/0ZcOY027kIuGhAfQuwZUB9YA9J0nD4DXhJUmeGiSSpM8NEktSZYSJJ6swnLUr6Bf+Ui+bKIxNJUmeGiSSpM09zSXpJ8BTbgc0jE0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM46hUmSQ5Jcm+QHSe5N8o4khyXZluT+9n5oG5sklyUZT3JXkuP7trO6jb8/yeq++glJ7m7rXNYeD8x0PSRJw9H1yOSvgW9W1W8D/4Le43XXATdU1XLghjYPcBq957svB9YCl0MvGOg9rfHt9J6eeEFfOFzexk6ut7LVp+shSRqCOf85lSRvAN4N/BuAqnoWeDbJKuCkNmwj8G16z2xfBVxVVQXc0o5qjmpjt1XVrrbdbcDKJN8G3lBVN7f6VcDpwPVtW4N6SNJe80+5dNflyOTNwE7gvye5I8kXk7wOOLKqHgVo729q4xcBD/etP9FqM9UnBtSZoccLJFmbZCzJ2M6dO+f+SSVJM+oSJguB44HLq+qtwD8y8+mmDKjVHOp7rKquqKrRqhodGRnZm1UlSXuhS5hMABNVdWubv5ZeuPy4nb6ivT/WN35J3/qLgUdmqS8eUGeGHpKkIZhzmFTVj4CHk/xWK50MfB/YDEzekbUauK5NbwbOaXd1rQCebKeotgKnJDm0XXg/Bdjalj2VZEW7i+ucKdsa1EOSNARdn2fyMeDLSQ4CHgA+TC+gNiVZAzwEnNnGbgHeC4wDT7exVNWuJBcC29u4z05ejAc+AlwJHEzvwvv1rX7xND0kSUPQKUyq6k5gdMCikweMLeDcabazAdgwoD4GHDeg/sSgHpKk4fAb8JKkznxsryQN0cvlOy4emUiSOjNMJEmdGSaSpM4ME0lSZ4aJJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM4ME0lSZ4aJJKmzzmGSZEGSO5J8vc0vS3JrkvuTfKU9OIskr2nz42350r5tnN/q9yU5ta++stXGk6zrqw/sIUkajn1xZPIJ4N6++UuAS6tqObAbWNPqa4DdVfUW4NI2jiTHAGcBxwIrgS+0gFoAfB44DTgGOLuNnamHJGkIOoVJksXA+4AvtvkA7wGubUM2Aqe36VVtnrb85DZ+FXBNVT1TVT+k91jfE9trvKoeqKpngWuAVbP0kCQNQdcjk/8CfBL4pzZ/OPCTqnquzU8Ai9r0IuBhgLb8yTb+F/Up60xXn6mHJGkI5hwmSf4QeKyqbu8vDxhasyzbV/VB+7g2yViSsZ07dw4aIknaB7ocmbwTeH+SB+mdgnoPvSOVQ5JMPg54MfBIm54AlgC05W8EdvXXp6wzXf3xGXq8QFVdUVWjVTU6MjIy908qSZrRnMOkqs6vqsVVtZTeBfQbq+pDwLeAM9qw1cB1bXpzm6ctv7GqqtXPand7LQOWA7cB24Hl7c6tg1qPzW2d6XpIkoZgPr5n8ingvCTj9K5vrG/19cDhrX4esA6gqnYAm4DvA98Ezq2q59s1kY8CW+ndLbapjZ2phyRpCBbOPmR2VfVt4Ntt+gF6d2JNHfMz4Mxp1r8IuGhAfQuwZUB9YA9J0nD4DXhJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjozTCRJnRkmkqTODBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjozTCRJnRkmkqTO5hwmSZYk+VaSe5PsSPKJVj8sybYk97f3Q1s9SS5LMp7kriTH921rdRt/f5LVffUTktzd1rksSWbqIUkaji5HJs8Bf15V/xxYAZyb5Bh6j+O9oaqWAze0eYDT6D3ffTmwFrgcesEAXAC8nd7TEy/oC4fL29jJ9Va2+nQ9JElDMOcwqapHq+q7bfopes9pXwSsAja2YRuB09v0KuCq6rkFOCTJUcCpwLaq2lVVu4FtwMq27A1VdXNVFXDVlG0N6iFJGoJ9cs0kyVLgrcCtwJFV9Sj0Agd4Uxu2CHi4b7WJVpupPjGgzgw9JElD0DlMkrwe+B/An1bV/5tp6IBazaG+N/u2NslYkrGdO3fuzaqSpL3QKUySvJpekHy5qr7ayj9up6ho74+1+gSwpG/1xcAjs9QXD6jP1OMFquqKqhqtqtGRkZG5fUhJ0qy63M0VYD1wb1X9575Fm4HJO7JWA9f11c9pd3WtAJ5sp6i2AqckObRdeD8F2NqWPZVkRet1zpRtDeohSRqChR3WfSfwx8DdSe5stf8IXAxsSrIGeAg4sy3bArwXGAeeBj4MUFW7klwIbG/jPltVu9r0R4ArgYOB69uLGXpIkoZgzmFSVf+bwdc1AE4eML6Ac6fZ1gZgw4D6GHDcgPoTg3pIkobDb8BLkjozTCRJnRkmkqTODBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1ZphIkjozTCRJnRkmkqTODBNJUmeGiSSpM8NEktSZYSJJ6swwkSR1dkCHSZKVSe5LMp5k3bD3R5JeqQ7YMEmyAPg8cBpwDHB2kmOGu1eS9Mp0wIYJcCIwXlUPVNWzwDXAqiHvkyS9Ih3IYbIIeLhvfqLVJEn7Wapq2PswJ0nOBE6tqn/b5v8YOLGqPtY3Zi2wts3+FnDfftzFI4DH92M/e9vb3vaeD79eVSOzDVq4P/ZknkwAS/rmFwOP9A+oqiuAK/bnTk1KMlZVo/a2t73t/XLpPZMD+TTXdmB5kmVJDgLOAjYPeZ8k6RXpgD0yqarnknwU2AosADZU1Y4h75YkvSIdsGECUFVbgC3D3o9pDOX0mr3tbW97D8MBewFekvTScSBfM5EkvUQYJtorSQ5J8u/b9ElJvj7sfZqU5KfD3odBXqr7NVdJ/n5IfX/xszcsk589ydIkHxzyvnw8yb1JvjzM/ZhkmGhvHQIM9X9oDVdV/e6QWg/9Z6/vsy8Fhhom9P5bvLeqPjTk/QAME+29i4HfSHIn8Dng9UmuTfKDJF9OEoAkJyT5TpLbk2xNctRQ97qjJF9rn2VH+zIsSX6a5KIk30tyS5IjW31ZkpuTbE9y4f7er/k2eaSV5KgkNyW5M8k9Sd41z61/8bOX5HPtdU+Su5N8YJ57Ay84yrwYeFfblz/bD33Pa5/1niR/muS/Am8GNu+P/nukqnz52uMXvX+R3dOmTwKepPeF0VcBNwO/B7wa+HtgpI37AL1bt+d73346j9s+rL0fDNwDHA4U8K9a/S+B/9SmNwPntOlz9/d+7a//zsCfA59u0wuAX92PP3v/GtjW+h4JPAQctR8/+0nA1+e7X+t1AnA38Drg9cAO4K3Ag8AR+2Mf9uR1QN8arJeE26pqAqAdrSwFfgIcB2xrByoLgEeHtYP7yMeT/FGbXgIsB54FJq8Z3Q78QZt+J71fdgBfAi7Zz/v1xDz267cd2JDk1cDXqurO/dQXev9oubqqngd+nOQ7wNt4eX5x+feAv62qfwRI8lVgvo8C95phoq6e6Zt+nt7PVIAdVfWO4ezSvpXkJOD3gXdU1dNJvg38CvDzav905JeffdK833M/w37tF1V1U5J3A+8DvpTkc1V11X5qn/3U56XggPisXjN5GUpyQ5L5+gvKTwG/OsuY+4CRJO9o+/PqJMfO0/7sD28Edrdf2L8NrJhl/N/R+/M+APN5cXRv92ufSvLrwGNV9d+A9cDx89yy/2fvJuADSRYkGQHeDdw2z/2n25f5dhNwepLXJnkd8EfA/9pPvfeYYTIPkmxJ8mtD6v0q4C3ArvnYflU9AfxdknvoXYAfNOZZ4AzgkiTfA+4EhnUH0L7wTWBhkruAC4FbZhn/CeDcJNvp/cJ/qezXvnYScGeSO+id1vvr+Ww25WfvHcBdwPeAG4FPVtWP5rP/FHcBz7WbL+b1AnhVfRe4kl5Y3gp8sarumM+ec+E34F9mkhwH/ElVnTfsfZH0ymGYSJI68zSXJKkzw0SS1JlhIknqzDCRJHVmmEiSOjNMJEmdGSaSpM7+P+B+P6xEILwKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import operator\n",
    "sorted_words = {k:v for k,v in sorted(Counter(words).items(),key = operator.itemgetter(1),reverse = True)}\n",
    "plt.bar(list(itertools.islice(sorted_words.keys(),10)),list(itertools.islice(sorted_words.values(),10)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected, the most frequent words are punctuations and stop-words. Lets further clean our reviews."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning\n",
    "\n",
    "- There are few cleaning steps that we will perform before moving forward on our reviews:\n",
    "  1. Lower-case the text\n",
    "  2. Remove numbers\n",
    "  3. Remove extra white-spaces(if any)\n",
    "  4. Remove Punctuation\n",
    "  5. Tokenize reviews\n",
    "  6. Remove Stop-words\n",
    "  7. Lemmatize all words (I prefer lemmatizing instead of stemming - in my final topics, I need coherent words, and not just random words)\n",
    "  \n",
    "  \n",
    "* For stop-words, we will be using an iterative list, which we will begin with an extra long list of stop-words from rank.nl (around 600 words), and then keep on adding domain specific terms as and when we counter through building initial topic models.\n",
    "* We will also create and consider bi-grams and tri-grams in our model to get the best possible set of topics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading spacy -en\n",
    "nlp = spacy.load(r'C:\\Users\\pares\\AppData\\Local\\Continuum\\anaconda3\\Lib\\site-packages\\en_core_web_sm\\en_core_web_sm-2.1.0', disable=['parser', 'ner'])\n",
    "\n",
    "#function to clean the reviews\n",
    "def clean_text(text):\n",
    "    text = text.lower() #lower-casing\n",
    "    text = [i for i in word_tokenize(text) if i not in stop_words] #remvoving stop-words\n",
    "    doc = nlp(' '.join(text))\n",
    "    text = [token.lemma_ for token in doc] #lemmatizing the reviews\n",
    "    text = ' '.join(text)    \n",
    "    text = re.sub(r'\\d+','',text) #removing numbers\n",
    "    text = text.translate(str.maketrans('','',string.punctuation)) #removing punctuation\n",
    "    text = text.strip() #removing white-spaces\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Initial list of stop-words (from rank.nl)\n",
    "stop_words = [\"a\",\"able\",\"about\",\"above\",\"abst\",\"accordance\",\"according\",\"accordingly\",\"across\",\"act\",\"actually\",\"added\",\"adj\",\"affected\",\"affecting\",\"affects\",\"after\",\"afterwards\",\"again\",\"against\",\"ah\",\"all\",\"almost\",\"alone\",\"along\",\"already\",\"also\",\"although\",\"always\",\"am\",\"among\",\"amongst\",\"an\",\"and\",\"announce\",\"another\",\"any\",\"anybody\",\"anyhow\",\"anymore\",\"anyone\",\"anything\",\"anyway\",\"anyways\",\"anywhere\",\"apparently\",\"approximately\",\"are\",\"aren\",\"arent\",\"arise\",\"around\",\"as\",\"aside\",\"ask\",\"asking\",\"at\",\"auth\",\"available\",\"away\",\"awfully\",\"b\",\"back\",\"be\",\"became\",\"because\",\"become\",\"becomes\",\"becoming\",\"been\",\"before\",\"beforehand\",\"begin\",\"beginning\",\"beginnings\",\"begins\",\"behind\",\"being\",\"believe\",\"below\",\"beside\",\"besides\",\"between\",\"beyond\",\"biol\",\"both\",\"brief\",\"briefly\",\"but\",\"by\",\"c\",\"ca\",\"came\",\"can\",\"cannot\",\"cause\",\"causes\",\"certain\",\"certainly\",\"co\",\"com\",\"come\",\"comes\",\"contain\",\"containing\",\"contains\",\"could\",\"couldnt\",\"d\",\"date\",\"did\",\"different\",\"do\",\"does\",\"doing\",\"done\",\"down\",\"downwards\",\"due\",\"during\",\"e\",\"each\",\"ed\",\"edu\",\"effect\",\"eg\",\"eight\",\"eighty\",\"either\",\"else\",\"elsewhere\",\"end\",\"ending\",\"enough\",\"especially\",\"et\",\"etc\",\"even\",\"ever\",\"every\",\"everybody\",\"everyone\",\"everything\",\"everywhere\",\"ex\",\"except\",\"f\",\"far\",\"few\",\"ff\",\"fifth\",\"first\",\"five\",\"fix\",\"followed\",\"following\",\"follows\",\"for\",\"former\",\"formerly\",\"forth\",\"found\",\"four\",\"from\",\"further\",\"furthermore\",\"g\",\"gave\",\"get\",\"gets\",\"getting\",\"give\",\"given\",\"gives\",\"giving\",\"go\",\"goes\",\"gone\",\"got\",\"gotten\",\"h\",\"had\",\"happens\",\"hardly\",\"has\",\"have\",\"having\",\"he\",\"hed\",\"hence\",\"her\",\"here\",\"hereafter\",\"hereby\",\"herein\",\"heres\",\"hereupon\",\"hers\",\"herself\",\"hes\",\"hi\",\"hid\",\"him\",\"himself\",\"his\",\"hither\",\"home\",\"how\",\"howbeit\",\"however\",\"hundred\",\"i\",\"id\",\"ie\",\"if\",\"im\",\"immediate\",\"immediately\",\"importance\",\"important\",\"in\",\"inc\",\"indeed\",\"index\",\"information\",\"instead\",\"into\",\"invention\",\"inward\",\"is\",\"it\",\"itd\",\"its\",\"itself\",\"j\",\"just\",\"k\",\"keep\",\"keeps\",\"kept\",\"kg\",\"km\",\"know\",\"known\",\"knows\",\"l\",\"largely\",\"last\",\"lately\",\"later\",\"latter\",\"latterly\",\"least\",\"less\",\"lest\",\"let\",\"lets\",\"like\",\"liked\",\"likely\",\"line\",\"little\",\"look\",\"looking\",\"looks\",\"ltd\",\"m\",\"made\",\"mainly\",\"make\",\"makes\",\"many\",\"may\",\"maybe\",\"me\",\"mean\",\"means\",\"meantime\",\"meanwhile\",\"merely\",\"mg\",\"might\",\"million\",\"miss\",\"ml\",\"more\",\"moreover\",\"most\",\"mostly\",\"mr\",\"mrs\",\"much\",\"mug\",\"must\",\"my\",\"myself\",\"n\",\"na\",\"name\",\"namely\",\"nay\",\"nd\",\"near\",\"nearly\",\"necessarily\",\"necessary\",\"need\",\"needs\",\"neither\",\"never\",\"nevertheless\",\"new\",\"next\",\"nine\",\"ninety\",\"no\",\"nobody\",\"non\",\"none\",\"nonetheless\",\"noone\",\"nor\",\"normally\",\"nos\",\"not\",\"noted\",\"nothing\",\"now\",\"nowhere\",\"o\",\"obtain\",\"obtained\",\"obviously\",\"of\",\"off\",\"often\",\"oh\",\"ok\",\"okay\",\"old\",\"omitted\",\"on\",\"once\",\"one\",\"ones\",\"only\",\"onto\",\"or\",\"ord\",\"other\",\"others\",\"otherwise\",\"ought\",\"our\",\"ours\",\"ourselves\",\"out\",\"outside\",\"over\",\"overall\",\"owing\",\"own\",\"p\",\"page\",\"pages\",\"part\",\"particular\",\"particularly\",\"past\",\"per\",\"perhaps\",\"placed\",\"please\",\"plus\",\"poorly\",\"possible\",\"possibly\",\"potentially\",\"pp\",\"predominantly\",\"present\",\"previously\",\"primarily\",\"probably\",\"promptly\",\"proud\",\"provides\",\"put\",\"q\",\"que\",\"quickly\",\"quite\",\"qv\",\"r\",\"ran\",\"rather\",\"rd\",\"re\",\"readily\",\"really\",\"recent\",\"recently\",\"ref\",\"refs\",\"regarding\",\"regardless\",\"regards\",\"related\",\"relatively\",\"research\",\"respectively\",\"resulted\",\"resulting\",\"results\",\"right\",\"run\",\"s\",\"said\",\"same\",\"saw\",\"say\",\"saying\",\"says\",\"sec\",\"section\",\"see\",\"seeing\",\"seem\",\"seemed\",\"seeming\",\"seems\",\"seen\",\"self\",\"selves\",\"sent\",\"seven\",\"several\",\"shall\",\"she\",\"shed\",\"shes\",\"should\",\"show\",\"showed\",\"shown\",\"showns\",\"shows\",\"significant\",\"significantly\",\n",
    "               \"similar\",\"similarly\",\"since\",\"six\",\"slightly\",\"so\",\"some\",\"somebody\",\"somehow\",\"someone\",\"somethan\",\"something\",\"sometime\",\"sometimes\",\"somewhat\",\"somewhere\",\"soon\",\"sorry\",\"specifically\",\"specified\",\"specify\",\"specifying\",\"still\",\"stop\",\"strongly\",\"sub\",\"substantially\",\"successfully\",\"such\",\"sufficiently\",\"suggest\",\"sup\",\"sure\",\"t\",\"take\",\"taken\",\"taking\",\"tell\",\"tends\",\"th\",\"than\",\"thank\",\"thanks\",\"thanx\",\"that\",\"thats\",\"the\",\"their\",\"theirs\",\"them\",\"themselves\",\"then\",\"thence\",\"there\",\"thereafter\",\"thereby\",\"thered\",\"therefore\",\"therein\",\"thereof\",\"therere\",\"theres\",\"thereto\",\"thereupon\",\"these\",\"they\",\"theyd\",\"theyre\",\"think\",\"this\",\"those\",\"thou\",\"though\",\"thoughh\",\"thousand\",\"throug\",\"through\",\"throughout\",\"thru\",\"thus\",\"til\",\"tip\",\"to\",\"together\",\"too\",\"took\",\"toward\",\"towards\",\"tried\",\"tries\",\"truly\",\"try\",\"trying\",\"ts\",\"twice\",\"two\",\"u\",\"un\",\"under\",\"unfortunately\",\"unless\",\"unlike\",\"unlikely\",\"until\",\"unto\",\"up\",\"upon\",\"ups\",\"us\",\"use\",\"used\",\"useful\",\"usefully\",\"usefulness\",\"uses\",\"using\",\"usually\",\"v\",\"value\",\"various\",\"very\",\"via\",\"viz\",\"vol\",\"vols\",\"vs\",\"w\",\"want\",\"wants\",\"was\",\"wasnt\",\"way\",\"we\",\"wed\",\"welcome\",\"went\",\"were\",\"werent\",\"we've\",\"what\",\"whatever\",\"whats\",\"when\",\"whence\",\"whenever\",\"where\",\"whereafter\",\"whereas\",\"whereby\",\"wherein\",\"wheres\",\"whereupon\",\"wherever\",\"whether\",\"which\",\"while\",\"whim\",\"whither\",\"who\",\"whod\",\"whoever\",\"whole\",\"whom\",\"whomever\",\"whos\",\"whose\",\"why\",\"widely\",\"willing\",\"wish\",\"with\",\"within\",\"without\",\"wont\",\"words\",\"world\",\"would\",\"wouldnt\",\"www\",\"x\",\"y\",\"yes\",\"yet\",\"you\",\"youd\",\"your\",\"youre\",\"yours\",\"yourself\",\"yourselves\",\"z\",\"zero\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": [
    "#before moving forward, I want to keep only the columns that I will be using from here on \n",
    "clean_data.drop([\"UserId\",\"ProfileName\",\"Time\"],inplace = True,axis = 1)\n",
    "clean_data.reset_index(inplace = True)\n",
    "clean_data.columns = [\"id\",\"title\",\"review\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Good Quality Dog Food</td>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Not as Advertised</td>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>\"Delight\" says it all</td>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Cough Medicine</td>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Great taffy</td>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                  title  \\\n",
       "0   0  Good Quality Dog Food   \n",
       "1   1      Not as Advertised   \n",
       "2   2  \"Delight\" says it all   \n",
       "3   3         Cough Medicine   \n",
       "4   4            Great taffy   \n",
       "\n",
       "                                              review  \n",
       "0  I have bought several of the Vitality canned d...  \n",
       "1  Product arrived labeled as Jumbo Salted Peanut...  \n",
       "2  This is a confection that has been around a fe...  \n",
       "3  If you are looking for the secret ingredient i...  \n",
       "4  Great taffy at a great price.  There was a wid...  "
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sanity check\n",
    "clean_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the cleaned reviews\n",
    "clean_data.to_pickle('clean_data_original.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to input the cleaning function to multi-core processing\n",
    "def clean_apply(df):\n",
    "    from nltk.tokenize import word_tokenize\n",
    "    import re\n",
    "    import spacy\n",
    "    import string\n",
    "\n",
    "    stop_words = [\"a\",\"able\",\"about\",\"above\",\"abst\",\"accordance\",\"according\",\"accordingly\",\"across\",\"act\",\"actually\",\"added\",\"adj\",\"affected\",\"affecting\",\"affects\",\"after\",\"afterwards\",\"again\",\"against\",\"ah\",\"all\",\"almost\",\"alone\",\"along\",\"already\",\"also\",\"although\",\"always\",\"am\",\"among\",\"amongst\",\"an\",\"and\",\"announce\",\"another\",\"any\",\"anybody\",\"anyhow\",\"anymore\",\"anyone\",\"anything\",\"anyway\",\"anyways\",\"anywhere\",\"apparently\",\"approximately\",\"are\",\"aren\",\"arent\",\"arise\",\"around\",\"as\",\"aside\",\"ask\",\"asking\",\"at\",\"auth\",\"available\",\"away\",\"awfully\",\"b\",\"back\",\"be\",\"became\",\"because\",\"become\",\"becomes\",\"becoming\",\"been\",\"before\",\"beforehand\",\"begin\",\"beginning\",\"beginnings\",\"begins\",\"behind\",\"being\",\"believe\",\"below\",\"beside\",\"besides\",\"between\",\"beyond\",\"biol\",\"both\",\"brief\",\"briefly\",\"but\",\"by\",\"c\",\"ca\",\"came\",\"can\",\"cannot\",\"can't\",\"cause\",\"causes\",\"certain\",\"certainly\",\"co\",\"com\",\"come\",\"comes\",\"contain\",\"containing\",\"contains\",\"could\",\"couldnt\",\"d\",\"date\",\"did\",\"didn't\",\"different\",\"do\",\"does\",\"doesn't\",\"doing\",\"done\",\"don't\",\"down\",\"downwards\",\"due\",\"during\",\"e\",\"each\",\"ed\",\"edu\",\"effect\",\"eg\",\"eight\",\"eighty\",\"either\",\"else\",\"elsewhere\",\"end\",\"ending\",\"enough\",\"especially\",\"et\",\"et-al\",\"etc\",\"even\",\"ever\",\"every\",\"everybody\",\"everyone\",\"everything\",\"everywhere\",\"ex\",\"except\",\"f\",\"far\",\"few\",\"ff\",\"fifth\",\"first\",\"five\",\"fix\",\"followed\",\"following\",\"follows\",\"for\",\"former\",\"formerly\",\"forth\",\"found\",\"four\",\"from\",\"further\",\"furthermore\",\"g\",\"gave\",\"get\",\"gets\",\"getting\",\"give\",\"given\",\"gives\",\"giving\",\"go\",\"goes\",\"gone\",\"got\",\"gotten\",\"h\",\"had\",\"happens\",\"hardly\",\"has\",\"hasn't\",\"have\",\"haven't\",\"having\",\"he\",\"hed\",\"hence\",\"her\",\"here\",\"hereafter\",\"hereby\",\"herein\",\"heres\",\"hereupon\",\"hers\",\"herself\",\"hes\",\"hi\",\"hid\",\"him\",\"himself\",\"his\",\"hither\",\"home\",\"how\",\"howbeit\",\"however\",\"hundred\",\"i\",\"id\",\"ie\",\"if\",\"i'll\",\"im\",\"immediate\",\"immediately\",\"importance\",\"important\",\"in\",\"inc\",\"indeed\",\"index\",\"information\",\"instead\",\"into\",\"invention\",\"inward\",\"is\",\"isn't\",\"it\",\"itd\",\"it'll\",\"its\",\"itself\",\"i've\",\"j\",\"just\",\"k\",\"keep\",\t\"keeps\",\"kept\",\"kg\",\"km\",\"know\",\"known\",\"knows\",\"l\",\"largely\",\"last\",\"lately\",\"later\",\"latter\",\"latterly\",\"least\",\"less\",\"lest\",\"let\",\"lets\",\"like\",\"liked\",\"likely\",\"line\",\"little\",\"'ll\",\"look\",\"looking\",\"looks\",\"ltd\",\"m\",\"made\",\"mainly\",\"make\",\"makes\",\"many\",\"may\",\"maybe\",\"me\",\"mean\",\"means\",\"meantime\",\"meanwhile\",\"merely\",\"mg\",\"might\",\"million\",\"miss\",\"ml\",\"more\",\"moreover\",\"most\",\"mostly\",\"mr\",\"mrs\",\"much\",\"mug\",\"must\",\"my\",\"myself\",\"n\",\"na\",\"name\",\"namely\",\"nay\",\"nd\",\"near\",\"nearly\",\"necessarily\",\"necessary\",\"need\",\"needs\",\"neither\",\"never\",\"nevertheless\",\"new\",\"next\",\"nine\",\"ninety\",\"no\",\"nobody\",\"non\",\"none\",\"nonetheless\",\"noone\",\"nor\",\"normally\",\"nos\",\"not\",\"noted\",\"nothing\",\"now\",\"nowhere\",\"o\",\"obtain\",\"obtained\",\"obviously\",\"of\",\"off\",\"often\",\"oh\",\"ok\",\"okay\",\"old\",\"omitted\",\"on\",\"once\",\"one\",\"ones\",\"only\",\"onto\",\"or\",\"ord\",\"other\",\"others\",\"otherwise\",\"ought\",\"our\",\"ours\",\"ourselves\",\"out\",\"outside\",\"over\",\"overall\",\"owing\",\"own\",\"p\",\"page\",\"pages\",\"part\",\"particular\",\"particularly\",\"past\",\"per\",\"perhaps\",\"placed\",\"please\",\"plus\",\"poorly\",\"possible\",\"possibly\",\"potentially\",\"pp\",\"predominantly\",\"present\",\"previously\",\"primarily\",\"probably\",\"promptly\",\"proud\",\"provides\",\"put\",\"q\",\"que\",\"quickly\",\"quite\",\"qv\",\"r\",\"ran\",\"rather\",\"rd\",\"re\",\"readily\",\"really\",\"recent\",\"recently\",\"ref\",\"refs\",\"regarding\",\"regardless\",\"regards\",\"related\",\"relatively\",\"research\",\"respectively\",\"resulted\",\"resulting\",\"results\",\"right\",\"run\",\"s\",\"said\",\"same\",\"saw\",\"say\",\"saying\",\"says\",\"sec\",\"section\",\"see\",\"seeing\",\"seem\",\"seemed\",\"seeming\",\"seems\",\"seen\",\"self\",\"selves\",\"sent\",\"seven\",\"several\",\"shall\",\"she\",\"shed\",\"she'll\",\"shes\",\"should\",\"shouldn't\",\"show\",\"showed\",\"shown\",\"showns\",\"shows\",\"significant\",\"significantly\",\n",
    "               \"similar\",\"similarly\",\"since\",\"six\",\"slightly\",\"so\",\"some\",\"somebody\",\"somehow\",\"someone\",\"somethan\",\"something\",\"sometime\",\"sometimes\",\"somewhat\",\"somewhere\",\"soon\",\"sorry\",\"specifically\",\"specified\",\"specify\",\"specifying\",\"still\",\"stop\",\"strongly\",\"sub\",\"substantially\",\"successfully\",\"such\",\"sufficiently\",\"suggest\",\"sup\",\"sure\",\t\"t\",\"take\",\"taken\",\"taking\",\"tell\",\"tends\",\"th\",\"than\",\"thank\",\"thanks\",\"thanx\",\"that\",\"that'll\",\"thats\",\"that've\",\"the\",\"their\",\"theirs\",\"them\",\"themselves\",\"then\",\"thence\",\"there\",\"thereafter\",\"thereby\",\"thered\",\"therefore\",\"therein\",\"there'll\",\"thereof\",\"therere\",\"theres\",\"thereto\",\"thereupon\",\"there've\",\"these\",\"they\",\"theyd\",\"they'll\",\"theyre\",\"they've\",\"think\",\"this\",\"those\",\"thou\",\"though\",\"thoughh\",\"thousand\",\"throug\",\"through\",\"throughout\",\"thru\",\"thus\",\"til\",\"tip\",\"to\",\"together\",\"too\",\"took\",\"toward\",\"towards\",\"tried\",\"tries\",\"truly\",\"try\",\"trying\",\"ts\",\"twice\",\"two\",\"u\",\"un\",\"under\",\"unfortunately\",\"unless\",\"unlike\",\"unlikely\",\"until\",\"unto\",\"up\",\"upon\",\"ups\",\"us\",\"use\",\"used\",\"useful\",\"usefully\",\"usefulness\",\"uses\",\"using\",\"usually\",\"v\",\"value\",\"various\",\"'ve\",\"very\",\"via\",\"viz\",\"vol\",\"vols\",\"vs\",\"w\",\"want\",\"wants\",\"was\",\"wasnt\",\"way\",\"we\",\"wed\",\"welcome\",\"we'll\",\"went\",\"were\",\"werent\",\"we've\",\"what\",\"whatever\",\"what'll\",\"whats\",\"when\",\"whence\",\"whenever\",\"where\",\"whereafter\",\"whereas\",\"whereby\",\"wherein\",\"wheres\",\"whereupon\",\"wherever\",\"whether\",\"which\",\"while\",\"whim\",\"whither\",\"who\",\"whod\",\"whoever\",\"whole\",\"who'll\",\"whom\",\"whomever\",\"whos\",\"whose\",\"why\",\"widely\",\"willing\",\"wish\",\"with\",\"within\",\"without\",\"wont\",\"words\",\"world\",\"would\",\"wouldnt\",\"www\",\"x\",\"y\",\"yes\",\"yet\",\"you\",\"youd\",\"you'll\",\"your\",\"youre\",\"yours\",\"yourself\",\"yourselves\",\"you've\",\"z\",\"zero\"]\n",
    "    nlp = spacy.load(r'C:\\Users\\pares\\AppData\\Local\\Continuum\\anaconda3\\Lib\\site-packages\\en_core_web_sm\\en_core_web_sm-2.1.0', disable=['parser', 'ner'])\n",
    "    \n",
    "    def clean_text(text):\n",
    "        text = text.lower() #lower-casing\n",
    "        text = [i for i in word_tokenize(text) if i not in stop_words] #remvoving stop-words\n",
    "        doc = nlp(' '.join(text))\n",
    "        text = [token.lemma_ for token in doc] #lemmatizing the reviews\n",
    "        text = ' '.join(text)    \n",
    "        text = re.sub(r'\\d+','',text) #removing numbers\n",
    "        text = text.translate(str.maketrans('','',string.punctuation)) #removing punctuation\n",
    "        text = text.strip() #removing white-spaces\n",
    "        return text\n",
    "    df['review'] = df.review.apply(clean_text)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time taken to clean 00:16:37.05\n"
     ]
    }
   ],
   "source": [
    "#function to perform data cleaning using parallel processing\n",
    "start = time.time()\n",
    "\n",
    "def parallelize(dataframe,func):\n",
    "    dataframe_split = np.array_split(dataframe,partitions)\n",
    "    pool = mp.Pool(cores)\n",
    "    dataframe_return = pd.concat(pool.map(func,dataframe_split),ignore_index = True)\n",
    "    pool.close()\n",
    "    \n",
    "    return dataframe_return\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    #mp.set_start_method('spawn')\n",
    "    cores = mp.cpu_count()\n",
    "    partitions = cores\n",
    "    \n",
    "    clean_data = parallelize(clean_data,clean_apply)\n",
    "    \n",
    "    \n",
    "    end = time.time()\n",
    "    \n",
    "    #time taken to remove non-frequent words\n",
    "    hours, rem = divmod(end-start, 3600)\n",
    "    minutes, seconds = divmod(rem, 60)\n",
    "    print(\"Total time taken to clean\",\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 395172 entries, 0 to 395171\n",
      "Data columns (total 3 columns):\n",
      "id        395172 non-null int64\n",
      "title     395172 non-null object\n",
      "review    395172 non-null object\n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 9.0+ MB\n"
     ]
    }
   ],
   "source": [
    "#sanity check\n",
    "clean_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the cleaned reviews\n",
    "clean_data.to_pickle('clean_data_lemmatized.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "197401"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = list(itertools.chain(*list(clean_data.review.apply(word_tokenize))))\n",
    "len(set(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have 197,401 unique tokens that we will use for building our topic model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}