code/07-natural-language-processing/05-pipeline-with-spam-data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[back](./04-tweak-nlp-model.ipynb)\n",
    "\n",
    "---\n",
    "## `Pipeline with Spam data`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Imports`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/goutham/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /Users/goutham/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spam</th>\n",
       "      <th>msg</th>\n",
       "      <th>msg_cleaned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>go jurong point, crazy.. available bugis n gre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>ok lar... joking wif u oni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  spam                                                msg  \\\n",
       "0  ham  Go until jurong point, crazy.. Available only ...   \n",
       "1  ham                      Ok lar... Joking wif u oni...   \n",
       "\n",
       "                                         msg_cleaned  \n",
       "0  go jurong point, crazy.. available bugis n gre...  \n",
       "1                      ok lar... joking wif u oni...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import string\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "df = pd.read_table('../../assets/SMSSpamCollection', header=None)\n",
    "df.columns = ['spam', 'msg']\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "stopwords = set(nltk.corpus.stopwords.words('english'))\n",
    "punctuation_set = set(string.punctuation)\n",
    "df['msg_cleaned'] = df.msg.apply(lambda x: ' '.join([word for word in x.split() if word not in stopwords\n",
    "                                                     and word not in punctuation_set]))\n",
    "df['msg_cleaned'] = df.msg_cleaned.str.lower()\n",
    "df.head(2)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Pipeline`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the previous few sections, we have seen a lot of repeated code and **Pipeline** is useful in a way that we can streamline our code and then reuse it, without rewriting it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([('countvect', CountVectorizer(stop_words=stopwords)), \\\n",
    "                    # ('tfidf', TfidfVectorizer(stop_words=stopwords)), \\\n",
    "                        ('rf', RandomForestClassifier())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9770279971284996\n",
      "[[1198    0]\n",
      " [  32  163]]\n"
     ]
    }
   ],
   "source": [
    "X = df.msg_cleaned\n",
    "y = df.spam\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "y_pred = pipeline.predict(X_test)\n",
    "\n",
    "print(pipeline.score(X_test, y_test))\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `Consider we want to do TFIDF`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_tfidf = Pipeline([# ('countvect', CountVectorizer(stop_words=stopwords)), \\\n",
    "                    ('tfidf', TfidfVectorizer(stop_words=stopwords)), \\\n",
    "                     ('rf', RandomForestClassifier())])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9748743718592965\n",
      "[[1188    2]\n",
      " [  33  170]]\n"
     ]
    }
   ],
   "source": [
    "X = df.msg_cleaned\n",
    "y = df.spam\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "pipeline_tfidf.fit(X_train, y_train)\n",
    "y_pred = pipeline_tfidf.predict(X_test)\n",
    "\n",
    "print(pipeline_tfidf.score(X_test, y_test))\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### `What if we want to do Count Vectorizer with Logistic Regression`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_cv = Pipeline([('countvect', CountVectorizer(stop_words=stopwords)), \\\n",
    "    ('lg', LogisticRegression())])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9777458722182341\n",
      "[[1203    3]\n",
      " [  28  159]]\n"
     ]
    }
   ],
   "source": [
    "X = df.msg_cleaned\n",
    "y = df.spam\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "pipeline_cv.fit(X_train, y_train)\n",
    "y_pred = pipeline_cv.predict(X_test)\n",
    "\n",
    "print(pipeline_cv.score(X_test, y_test))\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Conclusion`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can have multiple **pipelines** created and give it different names and go through the same steps to know which gives us the best score and best type1, type2 error combinations."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "---\n",
    "[next](../08-pyspark-df-sql/00-index.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0b4484ae063bf9ef0d5115063857dd0e382a2c73ed49a11a000ca56af03302b6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}