6 - Stacked LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacked LSTM Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# data analysis pkg\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Word vectorization and splitting dataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Building sequential model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, SpatialDropout1D\n",
    "from keras.layers import Embedding\n",
    "\n",
    "# Convolutional Layers\n",
    "from keras.layers import Bidirectional, LSTM\n",
    "\n",
    "# Cofigurations\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "# import metrics\n",
    "from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, recall_score\n",
    "\n",
    "\n",
    "# Avoiding warnings\n",
    "import warnings\n",
    "import os\n",
    "########### Prevent Warnings ###########\n",
    "warnings.filterwarnings(action='ignore')\n",
    "########### Prevent Warnings ###########"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/cleaned/data.csv')\n",
    "df.title = df.title.astype('str')\n",
    "df.text = df.text.astype('str')\n",
    "df.comment = df.comment.astype('str')\n",
    "df.rate = df.rate.astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting features and Splitting dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# features = df.columns.tolist()[:-1]\n",
    "features = ['text']\n",
    "label = ['verification_status']\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(df[features], df[label], test_size=0.3)\n",
    "\n",
    "X_train = [text for text in X_train.text]\n",
    "Y_train = [label for label in Y_train.verification_status]\n",
    "\n",
    "X_test = [text for text in X_test.text]\n",
    "Y_test = [label for label in Y_test.verification_status]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set hypermeter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir='model_output/Stacked_lstm/'\n",
    "\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "n_dim = 64\n",
    "n_unique_words = 5000\n",
    "max_text_length = 500\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2\n",
    "\n",
    "n_lstm_1 = 256\n",
    "n_lstm_2 = 256\n",
    "drop_lstm = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Option 1 - Tokenizing and convert to sequences using keras tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = Tokenizer(num_words=n_unique_words, lower=False)\n",
    "tok.fit_on_texts(X_train)\n",
    "\n",
    "train_sequences = tok.texts_to_sequences(X_train)\n",
    "train_sequences_matrix = pad_sequences(train_sequences, maxlen=max_text_length)\n",
    "\n",
    "test_sequences = tok.texts_to_sequences(X_test)\n",
    "test_sequences_matrix = pad_sequences(test_sequences, maxlen=max_text_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Option 2 - Tokenizing and convert to sequences using sklearn tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf = TfidfVectorizer(use_idf=True, max_features=n_unique_words, tokenizer=word_tokenize)\n",
    "# train_sequences_matrix = tfidf.fit_transform(X_train)\n",
    "# test_sequences_matrix = tfidf.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125369, 500)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sequences_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Designing Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LSTM_NN():\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(n_unique_words, n_dim, input_length=max_text_length))\n",
    "    model.add(SpatialDropout1D(drop_embed))\n",
    "    model.add(Bidirectional(LSTM(n_lstm_1, dropout=drop_lstm, return_sequences=True)))\n",
    "    model.add(Bidirectional(LSTM(n_lstm_2, dropout=drop_lstm)))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 500, 64)           320000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 500, 64)           0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 256)               328704    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 648,961\n",
      "Trainable params: 648,961\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = LSTM_NN()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configure Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "model_ckpt = ModelCheckpoint(filepath=output_dir+'weights.{epoch:02d}.hdf5')\n",
    "\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 125369 samples, validate on 53730 samples\n",
      "Epoch 1/4\n",
      "117888/125369 [===========================>..] - ETA: 2:17 - loss: 0.3083 - accuracy: 0.8859"
     ]
    }
   ],
   "source": [
    "model.fit(train_sequences_matrix, Y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(test_sequences_matrix, Y_test), callbacks=[model_ckpt])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+'weights.02.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict_proba(test_sequences_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc = roc_auc_score(Y_test, y_hat)*100.0\n",
    "print(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}