bert_sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sentiment analysis using BERT",
      "provenance": [],
      "authorship_tag": "ABX9TyM3H7WiG/OYZxNRY/Vywhsb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stephan-strate/sentiment-fine-foods/blob/master/bert_sentiment_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W7cJgzCYmhN0"
      },
      "source": [
        "!pip install -q ktrain"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pw-n0i11G0MI"
      },
      "source": [
        "# ensure images directory exists\n",
        "from pathlib import Path\n",
        "Path(\"images\").mkdir(parents=True, exist_ok=True)"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ACSU7qmmnEM"
      },
      "source": [
        "MODEL_NAME = \"distilbert-base-uncased\"\n",
        "BATCH_SIZE = 32\n",
        "EPOCHS = 2\n",
        "LEARNING_RATE = 5e-5\n",
        "MAX_LEN = 128\n",
        "RANDOM = 42\n",
        "\n",
        "TU_GREEN = \"#639a00\""
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8uc26TUAmsCb"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import urllib.request\n",
        "import zipfile\n",
        "\n",
        "url = \"https://storage.statc.de/datasets/amazon-fine-food/amazon-fine-food.zip\"\n",
        "filehandle, _ = urllib.request.urlretrieve(url)\n",
        "zip_file_object = zipfile.ZipFile(filehandle, \"r\")\n",
        "zip_file_object.extractall()"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lbXoH4cYmyZ5",
        "outputId": "7ee8e86f-2f4e-43c2-ae4c-1c35de06e23f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df = pd.read_csv(\"amazon-fine-food-cleaned.csv\", delimiter=\",\", header=0)\n",
        "print(\"Dataset has {:,} rows\".format(df.shape[0]))"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset has 393,670 rows\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jAER-Bfrm0Qq"
      },
      "source": [
        "import math\n",
        "\n",
        "def calculate_helpfulness(percentage):\n",
        "  if math.isnan(percentage):\n",
        "    return -1\n",
        "  if percentage > 0.5:\n",
        "    return 1\n",
        "  return 0\n",
        "\n",
        "df[\"Helpfulness\"] = (df[\"HelpfulnessNumerator\"] / df[\"HelpfulnessDenominator\"]).apply(calculate_helpfulness)"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJQ-wFVdm3cm",
        "outputId": "d50bf8ec-dfa7-4f49-9bbe-0080f98bca19",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def calculate_sentiment(score):\n",
        "  if score > 3:\n",
        "    return 1\n",
        "  if score < 3:\n",
        "    return 0\n",
        "  return -1\n",
        "\n",
        "df[\"Sentiment\"] = df[\"Score\"].apply(calculate_sentiment)\n",
        "df = df[df[\"Sentiment\"] != -1]\n",
        "\n",
        "print(\"Removed Score 3 entries, {:,} rows left\".format(df_without_neutral.shape[0]))"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Removed Score 3 entries, 363,898 rows left\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SWJpHW9npQUw",
        "outputId": "f1402c38-8b8e-4cc7-a29f-11036ad778dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "g = df.groupby(\"Sentiment\")\n",
        "print(g.size())"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentiment\n",
            "0     57082\n",
            "1    306816\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5m_t5B6GTIO"
      },
      "source": [
        "df_balanced = g.apply(lambda x: x.sample(30000).reset_index(drop=True))"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXLq5Ap2oXlE",
        "outputId": "cf17cff5-afd4-4f62-c383-b7b1ef8216ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(df_balanced.TextClean.values, df_balanced.Sentiment.values, test_size=0.2, random_state=RANDOM)\n",
        "\n",
        "t = text.Transformer(MODEL_NAME, maxlen=MAX_LEN, class_names=[0, 1])\n",
        "trn = t.preprocess_train(X_train, y_train)\n",
        "val = t.preprocess_test(X_test, y_test)\n",
        "\n",
        "model = t.get_classifier()\n",
        "learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=BATCH_SIZE)\n",
        "learner.fit(LEARNING_RATE, EPOCHS)"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 81\n",
            "\t95percentile : 214\n",
            "\t99percentile : 372\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 80\n",
            "\t95percentile : 214\n",
            "\t99percentile : 378\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "1500/1500 [==============================] - 662s 441ms/step - loss: 0.2153 - accuracy: 0.9134 - val_loss: 0.1643 - val_accuracy: 0.9360\n",
            "Epoch 2/2\n",
            "1500/1500 [==============================] - 661s 441ms/step - loss: 0.1210 - accuracy: 0.9553 - val_loss: 0.1686 - val_accuracy: 0.9364\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7ff889dabc50>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a-IeT6OCHj6S",
        "outputId": "289ded88-089d-4dff-d11b-c86dbe12b3a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_except_training = df[~df.Id.isin(df_balanced.Id.values)]\n",
        "print(df_except_training.shape[0])"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "303898\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qxYEVp5yq6UP",
        "outputId": "2da1d3e6-790e-430a-a445-e0f9dd58e65e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "predictor = ktrain.get_predictor(learner.model, preproc=t)\n",
        "predictions = predictor.predict(df_except_training.TextClean.values)\n",
        "\n",
        "comparison = list(map(lambda x: x[0] == x[1], zip(predictions, df_except_training.Sentiment.values)))\n",
        "score = comparison.count(True) / len(df_except_training.Sentiment.values)\n",
        "\n",
        "print(score)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "0.9372486821236072\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1_XIAiErBp4",
        "outputId": "891d2018-6a8a-4187-c49d-17374f6cd1b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        }
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "cm = confusion_matrix(df_except_training.Sentiment.values, predictions, [0, 1], normalize=\"true\")\n",
        "\n",
        "fig = plt.figure(figsize=(7, 5))\n",
        "ax = fig.add_subplot(111)\n",
        "for (i, j), z in np.ndenumerate(cm):\n",
        "    if i == j:\n",
        "      ax.text(j, i, \"{:0.2f}\".format(z), ha=\"center\", va=\"center\", fontsize=12)\n",
        "cax = ax.matshow(cm, cmap=plt.cm.Greens, vmin=0, vmax=1)\n",
        "fig.colorbar(cax)\n",
        "ax.set_xticklabels([''] + [0, 1])\n",
        "ax.set_yticklabels([''] + [0, 1])\n",
        "plt.xlabel(\"Predicted\")\n",
        "plt.ylabel(\"True\")\n",
        "plt.title(\"Sentiment accuracy balanced\")\n",
        "\n",
        "plt.savefig(\"images/confusion_sentiment_balanced.pdf\")\n",
        "plt.show()"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAFJCAYAAABtrh1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd80lEQVR4nO3de/xldV3v8dd77lyGyzCAyHBTMBmhEBG8pHLCDKwjdswEH53ELLKO1SOtc6jTUQ9lp04Xq0eUYhFpKampTUbCMSS8hDICcrgcagKF4SIMMDDCwNw+54+1fs5m//bvNv727N/6zes5j/1gr7W++7u++/f78d7f/V3ftVaqCklSNy0YdQMkSbvOEJekDjPEJanDDHFJ6jBDXJI6zBCXpA4zxDsoyfuS/I9Rt2NPluTqJD+5i6+9NMlvzHabvhNJKsmxo26HZs4QnyVJvjfJl5I8muThJF9M8sJZqPe8JF/oXVdVb62qX/9O696Ftrw7yV/t7v1KmtiiUTdgPkiyH/Bp4GeAjwJLgJcBT42yXXuaJAFSVTtG3RZpd7EnPjueA1BVH6mq7VW1uaqurKqbxgok+YkktyV5JMkVSY7q2VZJ3prk35JsTHJRGscD7wNenORbSTa25b/9dTzJ6UnWJ/mvSR5Icl+S1yZ5dZJ/bb8V/GrPvhYkuSDJvyd5KMlHk6xotx3dtuVNSe5KsiHJf2+3nQn8KvCGti1fG/SD6Kl7U5Jbk/xw3/afan8OY9tPbtcfkeQTSR5s2/XH7fqn9f572rioXb46yXuSfBF4AnhWkjf37OOOJD/d14azk9yY5LG2rWcmeX2Sr/aVe3uSv5vk9/7sJF9p6/m7sZ9j+9qPJbm//WZ2TZLnTfDzOjDJp9v3/Uj7fFXP9quT/Hr7zW5TkiuTrOzZPvYNcGOSu5Oc165fmuR329/jN9MMwe3V87pfbv9W7k3yE5O8R811VeXjO3wA+wEPAX8JnAUc2Lf9bGAdcDzNt59fA77Us71oevIHAEcCDwJnttvOA77QV9+lwG+0z08HtgHvBBYDP9W+/sPAcuB5wGbgmLb8LwDXAquApcD7gY+0245u2/IBYC/ge2i+TRzfbn838FdT/CxeDzyTpoPwBuBx4LCebfcALwQCHAscBSwEvga8F9gHWAZ876B99rRxUbt8NXBX+z4XtT+DHwSe3e7jFTThfnJb/lTgUeD72zYeDjy3/Vk8PPZe27I3AK+b4H1e3b6XE9o2/21fO3+i/fkvBf4AuHGC399BwOuAvdvyHwM+1beff6fpKOzVLv9Wu+0oYBNwbvu+DwJOare9F1gDrGjr/Xvgf7XbzgS+2dP2D7c/02NH/f+Sj13In1E3YL48aAL6UmA9TaiuAQ5tt/0j8JaesgvaYDmqXa6x0GqXPwpc0D4/j6lDfDOwsF1e3tZ3Wk/5rwKvbZ/fBpzRs+0wYGsbgEe3r13Vs/0rwDnt83czRYgP+LncCJzdPr8C+IUBZV5M88GzaMC2p+2TwSF+4RRt+NTYfmk+tN47Qbk/Bd7TPn8e8AiwdIKy3w7Tdnk1sGXs99BX9oC2zfv3//4GlD0JeKRvP7/Ws/yzwGfa578CfHJAHaH58Hx238/4zvb5JX1tfw6GeGcfDqfMkqq6rarOq6pVND2cZ9L0wKDpMf1h+5V3I02PLzS9wDH39zx/Ath3Brt/qKq2t883t//9Zs/2zT31HQV8sqcttwHbgUNnoy1Jfrwdqhir/wRg7Ov/ETS9yn5HAN+oqm3T3U+fu/vacFaSa9uhpI3Aq6fRBmi+Sb0xSYD/DHy0qiY7rtG732/Q9IZXJlmY5LfaoZrHgK+3ZVb2V5Bk7yTvT/KNtuw1wAFJFvYUm+j3MdF7OZimZ//Vnt/DZ9r10Pxt9rddHWWID0FV/T+a3tYJ7aq7gZ+uqgN6HntV1ZemU90sN+9u4Ky+tiyrqnu+07akGef/APA24KCqOgC4meYDa2zfz56gTUeOjXP3eZwmkMY8Y7J2JVlKM7TxuzTfhA4ALp9GG6iqa2l60y8D3gh8aFC5Hkf0PD+S5hvNhva1ZwOvBPan+fZATxt6vQP4LppvTvsBL5+kbL+J3ssGmg/u5/X8jvevqrHwv29A29VRhvgsSPLcJO8YOyCV5Aiaccpr2yLvA35l7OBWkv2TvH6a1X8TWJVkySw1933Ae9rAJcnBSc6eQVuOTjLR380+NIH6YFv3m9n5QQbwZ8AvJXlBGse27fgKTbD8VpJ9kixL8tL2NTcCL09yZJL9aYYQJrOEZhz6QWBbkrOAV/Vs/3PgzUnOSHOQ9/Akz+3Z/kHgj4GtVfW0qZ0D/FiS1Un2Bi4EPt5+I1pOcyzhIZoPoN+cpI7lNIG7sT0w+q4p9tnrr4FXJvnRJIuSHJTkpGpm53wAeG+SQwDa9/kD7es+CpzX0/aZ7FNzjCE+OzYBpwFfTvI4TXjfTNPLoqo+Cfw2cFn7lflmmgOg03EVcAtwf5INs9DWP6QZr78yyaa2radN87Ufa//7UJLr+zdW1a3A7wH/QhP4JwJf7Nn+MeA9NAfSNtGMVa9og+8/0hzovIvmuMIb2tf8H+BvgJtoxvY/PVkDq2oT8PM0QfUITa94Tc/2rwBvpjnw9yjwzzRDTGM+RPPBM5358B+i+cZ1P83B2J9v13+QZojiHuBWdn6YD/IHNAcsN7TlPjON/QJQVXfRDBW9g2aI7kaag9EA/43mYPq17d/cZ2l6/FTVP7b7vaotc9V096m5J1XeFEIa007De4BmNsu/jbo90lTsiUtP9zPAdQa4usIzNqVWkq/THFB87YibIk2bPfE5qD2D8PYk65JcMOr27Cmq6uiqOqqqbhh1WyaS5JI0Z+bePOq2aG4wxOeYdn7wRTQHPlcD5yZZPdpWaQ65lOaMSwkwxOeiU4F1VXVHVW0BLqOZcyxRVdfQzESRAEN8Ljqcp59Nt56nn9kpSd9miEtShxnic889PP2U6FXtOkkaxxCfe64DjktyTHuq/Tn0nHEoSb0M8TmmvZLf22gu23obzZX0bhltqzRXJPkIzWUNvivNzUDeMuo2abQ87V6SOsyeuCR1mCEuSR1miEtShxniktRhhvgcluT8UbdBc5N/G9001QXM2jte/VF78bubkpw8VZ2G+Nzm/6iaiH8b3XQpk1/A7CzguPZxPvCnU1VoiEvSbjKNC5idDXywGtcCByQ5bLI659RNIbJ0YbHv4lE3Y+7YexE5aJkT+Vvfc/Rzpy60h1h1xCpOesFJ/m207v7G3Ty04aHMVn1ZuazYsmNmL9q09RbgyZ41F1fVxTPc9UQXwLtvohfMqRBn38XwA0dMXU57pKv+4spRN0Fz1Pe95FWzW+GWHXDaITN7zWfvebKqTpndhkxtboW4JM0VmbWO/UzM+AJ4jolLUr/QpONMHrNjDfDj7SyVFwGPVtWEQylgT1ySBhtCT7y9gNnpwMok64F3AYsBqup9wOXAq4F1wBPAm6eq0xCXpEGGMJpSVedOsb2A/zKTOg1xSRonoxoTnzFDXJL6jY2Jd4AhLkmD2BOXpA7rRoYb4pI0ToAF3UhxQ1ySBulGhhvikjSQY+KS1GHdyHBDXJLGcUxckjquGxluiEvSeJ6xKUnd1aHhlI6cWCpJGsSeuCQN0o2OuCEuSQM5Ji5JHdaNDDfEJWmcDh3YNMQlaZBuZLghLkkDOSYuSR3WkQnYhrgk9YtnbEpSt3Ujww1xSRrInrgkdZhj4pLUUcGeuCR1Wjcy3BCXpIE8Y1OSOszhFEnqqNCZ4ZSOHH+VJA1iT1ySxgmZ4XBKDaklUzHEJWkAQ1ySOqwjxzUNcUnq19wTYmYpvn04TZmSIS5J/TLz4ZRRMcQlaQBDXJI6a+azU0bFEJekATqS4Ya4JPVrLmLYjRQ3xCWpnwc2Janb0pGLpxjikjSAPXFJ6rCOZLghLkn9QmZ8xuaoGOKSNEBXhlO8nrgk9Wtnp8zkMa1qkzOT3J5kXZILBmw/MsnnktyQ5KYkr56qTkNcknaDJAuBi4CzgNXAuUlW9xX7NeCjVfV84BzgT6aq1xCXpAGSmT2m4VRgXVXdUVVbgMuAs/vKFLBf+3x/4N6pKnVMXJL67OIZmyuTrO1ZvriqLu5ZPhy4u2d5PXBaXx3vBq5M8nPAPsArp9qpIS5JA+xCiG+oqlO+w92eC1xaVb+X5MXAh5KcUFU7JnqBIS5J4wzlKob3AEf0LK9q1/V6C3AmQFX9S5JlwErggYkqdUxckvoNZ3bKdcBxSY5JsoTmwOWavjJ3AWcAJDkeWAY8OFml9sQlaYDZ7ohX1bYkbwOuABYCl1TVLUkuBNZW1RrgHcAHkvwizUHO86pq0nswG+KS1GdYl6KtqsuBy/vWvbPn+a3AS2dSpyEuSQN05YxNQ1ySBvDaKZLUVdM/gWfkDPHd6ant8JUH4L4nYOlC+J6D4Ojl48tt2Q5f3dCUAzhuPzjxoJ3b/+keePQp2F6w72I4cQWs2nf3vAdpDxBvlKyB1j4ICwI/fAxsfAr++T44cAnsv/Tp5a7fANt3wGuOgie3w+fugX0Ww7Pas3FPXgn7L2nq2vBks/2HlsFe/jql2dKVO/sMdZ74VFfs2qNs2wHrv9X0mhcvgIP3gsP3gTs3jS97z+Nw/IGwaEHT037WfnDHYzu3H7i0CXBoDqPvAJ7YtjvehbTHGMZVDIdhaF23nit2fT/NNQKuS7KmnUKz53lsazPItt+SnesOWAIPbJ7e6zduefryP98L92+GHQXP2BtWLB38Okm7xOGUnit2ASQZu2LXnhni23Y0PfBeixc06/sdtjfc+gi86FB4clvTC9/eV+4Vz2wC/P4n4LEt3TkKI3VEV/6XGuZwyqArdh3eXyjJ+UnWJlnLk9uH2JwRW7QAtvYF8dYdzfp+LzgYFgY+/Q245n44ajnsPeDzdkHgmfvAfZth/ePDabe0B8qQbgoxDCM/EtZeqvFigBy0bNLTSzttv8VQBZu2wPJ2SGXjluYAZb+lC+Elz9i5/LWHYMWyieuugm9tnd32SuqEYfbEp3PFrj3HogXNNMCbHm6GUB7c3BzAPGbAFMNNW5vpiDsK7n0c1j0KJxzYbHtsS7Nu245m+52bmroOmSTkJc3QzHrh87Un/u0rdtGE9znAG4e4v7nvlIPhyw/AJ+5setunHNxML3xgc3Og8vXPbso98mQzzXDLDli+GF5y6M5piAXc/DA82o6DL1/c9Non66lLmrE9/sDmRFfsGtb+OmHpQnj5YePXH7LXzgAHOHJ58xhk/yXwqiMGb5M0azqS4cMdEx90xS5J6oI9vicuSV01NjulCwxxSRrAEJekDutIhhvikjSeVzGUpE4zxCWpozywKUkd15EMN8QlaRB74pLUZYa4JHWVs1Mkqbu8270kdVdwTFySOq0rIT7Uu91LkobLnrgkDdCVnrghLkkDdCTDDXFJGmfE982cCUNckvo4O0WSOs4Ql6QOM8Qlqas8Y1OSus2euCR1VLwAliR1myEuSR3WkQw3xCVpHO+xKUkdZ4hLUnfZE5ekjgqwoBsZbohL0njdmWLoTSEkqcPsiUtSv8ACe+KS1E1jl6KdyWNa9SZnJrk9ybokF0xQ5keT3JrkliQfnqpOe+KSNMBs93CTLAQuAr4fWA9cl2RNVd3aU+Y44FeAl1bVI0kOmapeQ1ySBhjCcMqpwLqqugMgyWXA2cCtPWV+Crioqh4BqKoHpmznbLdSkrpuF4dTViZZ2/M4v6/aw4G7e5bXt+t6PQd4TpIvJrk2yZlTtdWeuCSNk13piW+oqlO+wx0vAo4DTgdWAdckObGqNk72AklSr+FcO+Ue4Iie5VXtul7rgS9X1VbgziT/ShPq101UqcMpktQnNOE4k8c0XAccl+SYJEuAc4A1fWU+RdMLJ8lKmuGVOyar1J64JA0w2wc2q2pbkrcBVwALgUuq6pYkFwJrq2pNu+1VSW4FtgO/XFUPTVavIS5JAwzjtPuquhy4vG/dO3ueF/D29jEthrgk9WkugNWNMzYNcUkaoBsRbohL0gC7NMVwJAxxSeqTDl0AyxCXpAG6cj1xQ1ySBuhKT9yTfSSpw+yJS1Kf4OwUSeq0rgynGOKSNI5TDCWpszKcqxgOhSEuSQN0pSc+5eyUNH4syTvb5SOTnDr8pknS6GSGj1GZzhTDPwFeDJzbLm+iudmnJM1LYxfAmsljVKYznHJaVZ2c5AaA9g7MS4bcLkkaqa4Mp0wnxLcmWQgUQJKDgR1DbZUkjVQ6c2BzOsMpfwR8EjgkyXuALwC/OdRWSdIIDen2bEMxZU+8qv46yVeBM2je22ur6raht0ySRmU+TTFMciTwBPD3veuq6q5hNkySRmk+jYn/A814eIBlwDHA7cDzhtguSRqZeXV7tqo6sXc5ycnAzw6tRZI0B8yb4ZR+VXV9ktOG0ZiTjn4uV/3FZ4dRteaBFW960aiboLnqzvWzXGFY0JHrGE5nTPztPYsLgJOBe4fWIkmaA+ZTT3x5z/NtNGPkfzuc5kiSZmLSEG9P8lleVb+0m9ojSSM3L26UnGRRVW1L8tLd2SBJmgsyD8bEv0Iz/n1jkjXAx4DHxzZW1SeG3DZJGpn5NCa+DHgI+D52zhcvwBCXNC9lntzZ55B2ZsrN7AzvMTXUVknSiGWkV0SZvslCfCGwL4Ovd26IS5rX5kNP/L6qunC3tUSS5pD5MCbejXcgSbMs7b8umCzEz9htrZCkuWQ+zBOvqod3Z0MkaS6ZD8MpkrRHau7s0/3ZKZK0h+rOPTYNcUkawBCXpA6bN9cTl6Q9TbAnLknd1aEpht04/CpJGsieuCSNMz/O2JSkPVKABenGQIUhLkkDeGBTkjqsK8Mp3fi+IEm7VXNnn5k8plVrcmaS25OsS3LBJOVel6SSnDJVnYa4JPUJOy9HO91/U9aZLAQuAs4CVgPnJlk9oNxy4BeAL0+nrYa4JA0whJ74qcC6qrqjqrYAlwFnDyj368BvA09Oq53TfUOStMcIJAtm9ABWJlnb8zi/r9bDgbt7lte363buNjkZOKKq/mG6TfXApiSNs0vzxDdU1ZRj2BPusfkk+H3gvJm8zhCXpD7NPPFZn51yD3BEz/Kqdt2Y5cAJwNXt9MZnAGuSvKaq1k5UqSEuSQMMYZ74dcBxSY6hCe9zgDeObayqR4GVPfu/GvilyQIcDHFJGmi2L0VbVduSvA24AlgIXFJVtyS5EFhbVWt2pV5DXJL6DOtStFV1OXB537p3TlD29OnUaYhL0jgZm3Ey5xnikjSAd/aRpI5KvACWJHWaF8CSJA2dPXFJGicOp0hSl3lgU5I6qpkn3o3RZkNcksbxRsmS1GmOiUtSh9kTl6QOsycuSR0VnJ0iSd0V54lLUqelIye0G+KSNIA9cUnqqODsFEnqsAzjRslDYYhL0gD2xCWpwxwTl6SOasbEuzE7pRutlCQNZE9cksbxZB9J6jRPu5ekrvJu95LUXZ7sI0kdZ09ckjornZliaIhL0gCedi9JHeWYuKSZe2o7XPcg3P8ELF0I370Cjlo+vtyW7XDDBrjviWb52P3hhBXjyz2wGT53L6w+AE48aLhtn4ccE5c0M9dvaM6hPvto2PgUfP5+OGAp7L/k6eVueAi2FfzQUU3wf+5e2HsRPGu/nWV2VBP0K5buzncwj6QzPfGhjdwnuSTJA0luHtY+pHlj2w5Y/y04cQUsXgAH7wXP3Bu+vml82Xsfh+ceAIsWwD6Lm/C+s6/c7RvhGXvDfkvGv17TkvYWbdN9jMowD79eCpw5xPql+WPTVkhgeU/oHrAUHtsyvdc/2lPu8a1wxyZYfeDstnEP0twoeWb/RmVoe66qa4CHh1W/NK9s29H0wHstXgBbd4wve9jecNvGZtumrXDHY7C9p9z1G3b26LVrYk982pKcn2RtkrUbHnxo1M2RRmPRgMDeOiDYAZ6/EhYGLr8LvnAfHLUv7NUe3rrn8eYD4ch9h9/meS0z/jcqIz+wWVUXAxcDPP8FJ9WImyONxvLFUAWbtuwcUtn41OAx7aUL4cWH7ly+6SE4aFnz/Jub4eGn4O++3ixv3dGMDWzcAi87bJjvYN5xdoqk6Vu0AA7fF25+BF54MDzyFNz7BJxx+Piy39ra9NAXL4BvPgH//hh8X1vuxBVw/AE7y96woemlOz4+Y12ZnWKIS3PFC1bCdQ/Ap77e9LZfsLKZXvjgZrjmPnjds5pyDz/VhPPWHU0P/kWH7pyGOBbuYxYugEVp6tO8NLQQT/IR4HRgZZL1wLuq6s+HtT+p85YuhO8dMORx8F47Axya8e7pjnmfdsjstG0P4xmbQFWdO6y6JWnoHBOXpK7qzhmbhrgkDeDsFEnqMHviktRhXQnxkZ+xKUlzTRjOafdJzkxye5J1SS4YsP3tSW5NclOSf0py1FR1GuKSNM7sn3afZCFwEXAWsBo4N8nqvmI3AKdU1XcDHwf+91T1GuKSNMAQrp1yKrCuqu6oqi3AZcDZvQWq6nNV1d7tg2uBVVNV6pi4JPXLLs1OWZlkbc/yxe21ocYcDtzds7weOG2S+t4C/ONUOzXEJWmAXTiwuaGqTpmVfSc/BpwCvGKqsoa4JPUZO7A5y+4BjuhZXtWue/q+k1cC/x14RVU9NVWljolL0jhDuZ74dcBxSY5JsgQ4B1jztL0mzwfeD7ymqh6YTqX2xCVpgNmeJ15V25K8DbgCWAhcUlW3JLkQWFtVa4DfAfYFPtZ+E7irql4zWb2GuCQNMIzT7qvqcuDyvnXv7Hn+ypnWaYhL0gCesSlJGjp74pLUx5tCSFKnTf96KKNmiEvSQIa4JHXTrp12PxKGuCQN4Ji4JHWYIS5JHRUPbEpSt9kTl6QOM8QlqcMcTpGkDrMnLkkd5YFNSeo4e+KS1GmGuCR1Vjci3BCXpIG6MibuTSEkqcPsiUvSQN3oiRvikjRANyLcEJekAUJXYtwQl6Q+6dBNITywKUkdZk9ckgbwjE1J6rCuhLjDKZLUYfbEJWkAD2xKkobOnrgkjZPOjIkb4pI0kCEuSZ3UnfM1DXFJGqgrBzYNcUkayBCXpM7qRoQb4pI0gW7EuPPEJanD7IlL0jjpzIFNe+KS1GH2xCWpTzNPvBs9cUNckgYyxCWps7oR4Ya4JA3UlQObhrgkjdOdq6cY4pI0QDci3BCXpAl0I8adJy5J/dKMic/kMa1qkzOT3J5kXZILBmxfmuRv2u1fTnL0VHUa4pK0GyRZCFwEnAWsBs5Nsrqv2FuAR6rqWOC9wG9PVa8hLkl9xk72mcm/aTgVWFdVd1TVFuAy4Oy+MmcDf9k+/zhwRqbo5s+pMfEbr//ahhXLDv7GqNsxh6wENoy6EZqT/Nt4uqNms7Lrv3rDFXst2mflDF+2LMnanuWLq+rinuXDgbt7ltcDp/XV8e0yVbUtyaPAQUzyu55TIV5VB4+6DXNJkrVVdcqo26G5x7+N4aqqM0fdhulyOEWSdo97gCN6lle16waWSbII2B94aLJKDXFJ2j2uA45LckySJcA5wJq+MmuAN7XPfwS4qqpqskrn1HCKxrl46iLaQ/m30THtGPfbgCuAhcAlVXVLkguBtVW1Bvhz4ENJ1gEP0wT9pDJFyEszkmQ78H9pOgi3AW+qqid2sa5LgU9X1ceT/Bnw+1V16wRlTwe2VNWXZriPrwOnVJUHCdVJDqdotm2uqpOq6gRgC/DW3o3tON+MVdVPThTgrdOBl+xK3VKXGeIaps8DxyY5Pcnnk6wBbk2yMMnvJLkuyU1JfhogjT9uz2j7LHDIWEVJrk5ySvv8zCTXJ/lakn9qz2p7K/CLSW5M8rIkByf523Yf1yV5afvag5JcmeSWtnffjXOrpQk4Jq6haHvcZwGfaVedDJxQVXcmOR94tKpemGQp8MUkVwLPB76L5my2Q4FbgUv66j0Y+ADw8rauFVX1cJL3Ad+qqt9ty30YeG9VfSHJkTTjkMcD7wK+UFUXJvlBmjPkpM4yxDXb9kpyY/v88zQHal4CfKWq7mzXvwr47iQ/0i7vDxwHvBz4SFVtB+5NctWA+l8EXDNWV1U9PEE7Xgms7jnZbb8k+7b7+E/ta/8hySO7+D6lOcEQ12zbXFUn9a5og/Tx3lXAz1XVFX3lXj2L7VgAvKiqnhzQFmnecExco3AF8DNJFgMkeU6SfYBrgDe0Y+aHAf9hwGuvBV6e5Jj2tSva9ZuA5T3lrgR+bmwhydgHyzXAG9t1ZwEHztq7kkbAENco/BnNePf1SW4G3k/zrfCTwL+12z4I/Ev/C6vqQeB84BNJvgb8Tbvp74EfHjuwCfw8cEp74PRWds6S+Z80HwK30Ayr3DWk9yjtFs4Tl6QOsycuSR1miEtShxniktRhhrgkdZghLkkdZohLUocZ4pLUYf8fTcvTji+9gJ8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dVTvYjcBhBD",
        "outputId": "5fa62f5c-92ae-465c-e3c1-9cc356963a8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_test = df[df[\"TextClean\"].isin(df_except_training.TextClean.values)]\n",
        "print(df_test.shape[0])"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "303918\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAQPhttECaux",
        "outputId": "bb8e912f-99fd-4c16-da7a-b293617ea448",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "gh = df_test.groupby(\"Helpfulness\")\n",
        "print(gh.size())"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Helpfulness\n",
            "-1    147488\n",
            " 0     28996\n",
            " 1    127434\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRoNcWHvCld2",
        "outputId": "c2036aec-ede3-4cb6-ecf5-0f09a6a7cca2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "scores = []\n",
        "for group_name, group in gh:\n",
        "  predictions_group = predictor.predict(group.TextClean.values)\n",
        "\n",
        "  comparison_group = list(map(lambda x: x[0] == x[1] or x[1] == -1, zip(predictions_group, group.Sentiment.values)))\n",
        "  score_group = comparison_group.count(True) / len(group.Sentiment.values)\n",
        "  scores.append(score_group)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adpRyo_TERtt",
        "outputId": "7d5dfe2f-b531-4415-ac67-337841bcb6c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(scores)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.948090692124105, 0.9018140433163195, 0.9327651961015113]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}