classification/classification_model_dc_fire.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "import sentencepiece as spm\n",
    "import re\n",
    "import pdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('1.0.57', '1.0.0')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import fastai, torch\n",
    "fastai.__version__ , torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_seed(seed_value, use_cuda):\n",
    "    np.random.seed(seed_value)  \n",
    "    torch.manual_seed(seed_value)  \n",
    "    random.seed(seed_value)\n",
    "    if use_cuda:\n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)  \n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed(42, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/home/ubuntu/gaurav/in/fire/code-mixed-enma/classification_task_1\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hoo mammokka police vesham aaha anthas</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Oru rekshayum illa...kidilam kannu nananjupoyi</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ikka     waiting.........</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Raju Ettante Oro Shorttum Ijathi ppwli</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ettan fansil netti poya aarenkilum undo?    #...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text   category\n",
       "0             hoo mammokka police vesham aaha anthas  Positive \n",
       "1     Oru rekshayum illa...kidilam kannu nananjupoyi  Positive \n",
       "2                          Ikka     waiting.........  Positive \n",
       "3             Raju Ettante Oro Shorttum Ijathi ppwli  Positive \n",
       "4   Ettan fansil netti poya aarenkilum undo?    #...  Positive "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(path/'../dc_fire/malayalam_train.tsv', sep='\\t')\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>speechless 🤐.   ikkaaa</td>\n",
       "      <td>not-malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Raja sollunathu mattuthaam seyyvaa seyyunnath...</td>\n",
       "      <td>not-malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Im Prithiviraj fan from tamilnadu... Love it</td>\n",
       "      <td>not-malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mohanlal sir - look ..... kiddo...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kandathil vech mungiya pdam  Rating 1.1/5</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text        category\n",
       "0                             speechless 🤐.   ikkaaa  not-malayalam \n",
       "1   Raja sollunathu mattuthaam seyyvaa seyyunnath...  not-malayalam \n",
       "2       Im Prithiviraj fan from tamilnadu... Love it  not-malayalam \n",
       "3                 mohanlal sir - look ..... kiddo...       Positive \n",
       "4          Kandathil vech mungiya pdam  Rating 1.1/5       Negative "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid = pd.read_csv(path/'../dc_fire/malayalam_dev.tsv', sep='\\t')\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ml_sen_1</td>\n",
       "      <td>Bollywood film Newton inte remake aano?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ml_sen_2</td>\n",
       "      <td>endukond viewrs koodunnilla ?? ippozhum 2.8m a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ml_sen_3</td>\n",
       "      <td>Mara paazhu mega mairananil ninnum ethil koodu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ml_sen_4</td>\n",
       "      <td>Video nay cang xem cang thit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ml_sen_5</td>\n",
       "      <td>Sunny chechiye kaanan vannathu njan maathram aano</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                               text\n",
       "0  ml_sen_1            Bollywood film Newton inte remake aano?\n",
       "1  ml_sen_2  endukond viewrs koodunnilla ?? ippozhum 2.8m a...\n",
       "2  ml_sen_3  Mara paazhu mega mairananil ninnum ethil koodu...\n",
       "3  ml_sen_4                       Video nay cang xem cang thit\n",
       "4  ml_sen_5  Sunny chechiye kaanan vannathu njan maathram aano"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv(path/'../dc_fire/malayalam_test.tsv', sep='\\t')\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4851, 2), (540, 2), (1348, 2))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape, df_valid.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Mixed_feelings ': 289,\n",
       "         'Negative ': 549,\n",
       "         'Positive ': 2022,\n",
       "         'not-malayalam ': 647,\n",
       "         'unknown_state ': 1344})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(df_train['category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Mixed_feelings ': 44,\n",
       "         'Negative ': 51,\n",
       "         'Positive ': 224,\n",
       "         'not-malayalam ': 60,\n",
       "         'unknown_state ': 161})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(df_valid['category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5391, 2)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.concat([df_train, df_valid])\n",
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_cols = ['category']\n",
    "text_cols = ['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_all_caps(t: str) -> str:\n",
    "    tokens = t.split()\n",
    "    tokens = replace_all_caps(tokens)\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "def handle_upper_case_first_letter(t: str) -> str:\n",
    "    tokens = t.split()\n",
    "    tokens = deal_caps(tokens)\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "def lower_case_everything(t: str) -> str:\n",
    "    return t.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CodeMixedMalayalamTokenizer(BaseTokenizer):\n",
    "    def __init__(self, lang:str):\n",
    "        self.lang = lang\n",
    "        self.sp = spm.SentencePieceProcessor()\n",
    "        self.sp.Load(str(path/\"../tokenizer/mlen_spm.model\"))\n",
    "        \n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        return self.sp.EncodeAsPieces(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(str(path/\"../tokenizer/mlen_spm.model\"))\n",
    "itos = [sp.IdToPiece(int(i)) for i in range(25000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xxunk',\n",
       " 'xxbos',\n",
       " 'xxeos',\n",
       " 'xxpad',\n",
       " 'xxfld',\n",
       " 'xxmaj',\n",
       " 'xxup',\n",
       " 'xxrep',\n",
       " 'xxwrep',\n",
       " '.',\n",
       " ',',\n",
       " 'ൽ',\n",
       " '▁the',\n",
       " 'ർ',\n",
       " '▁',\n",
       " 'ൻ',\n",
       " 's',\n",
       " '▁•',\n",
       " '▁of',\n",
       " 'ൾ']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "itos[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 25,000 is the vocab size that we chose in sentencepiece\n",
    "mlen_vocab = Vocab(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(lang='mlen', tok_func=CodeMixedMalayalamTokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pre_rules.append(lower_case_everything)\n",
    "tokenizer.pre_rules.append(handle_all_caps)\n",
    "tokenizer.pre_rules.append(handle_upper_case_first_letter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['xxunk',\n",
       "  'xxpad',\n",
       "  'xxbos',\n",
       "  'xxeos',\n",
       "  'xxfld',\n",
       "  'xxmaj',\n",
       "  'xxup',\n",
       "  'xxrep',\n",
       "  'xxwrep'],\n",
       " [<function fastai.text.transform.fix_html>,\n",
       "  <function fastai.text.transform.replace_rep>,\n",
       "  <function fastai.text.transform.replace_wrep>,\n",
       "  <function fastai.text.transform.spec_add_spaces>,\n",
       "  <function fastai.text.transform.rm_useless_spaces>,\n",
       "  <function __main__.lower_case_everything>,\n",
       "  <function __main__.handle_all_caps>,\n",
       "  <function __main__.handle_upper_case_first_letter>],\n",
       " [<function fastai.text.transform.replace_all_caps>,\n",
       "  <function fastai.text.transform.deal_caps>])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.special_cases, tokenizer.pre_rules, tokenizer.post_rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'▁tell▁me▁about▁tour▁self,▁mujhe▁jaanna▁hai'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = tokenizer.process_all(['Tell me about TOUR self, mujhe jaanna hai'])\n",
    "''.join(tokens[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm = TextLMDataBunch.from_df(path=path, train_df=df_train, valid_df=df_valid, test_df=df_test, tokenizer=tokenizer, vocab=mlen_vocab, label_cols=label_cols, text_cols=text_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk ▁x x bo s ▁wait ing ▁to ▁see ▁mam mu kaa s ▁unda a ▁ xxunk ▁x x bo s ▁last ▁aa ▁cha di ▁adi ▁ xxrep ▁4 ▁. ▁u ff . ▁fr m ▁an ▁et tan ▁fan ▁x x bo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>▁chol lan ▁a van ▁varu nu u . . . ▁ xxunk ▁lu ci fer ▁x x bo s ▁positive ▁ki ttiya l ▁pinne ▁para ya nda llo ▁ xxrep ▁4 ▁. ▁et tan ▁fan ▁aa ya ▁ njan ▁than me ▁kannu ▁tha lli ▁poi ▁book ing s ▁kand u ▁x x bo s ▁annu m ▁ennu m ▁ennu m ▁i kka ▁mass ▁ xxunk ▁x x bo s ▁ver</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>▁6000 ▁su scrib e ▁ ulla dhi l ▁it t ▁22 k ▁su scrib e 4 14 k view ▁ 91 k ▁like ▁ xxrep ▁4 ▁a ▁ki yya ▁i kka ▁mass ▁x x bo s ▁ki du ▁b g m ▁for ▁i kka ▁ki du ▁movie ▁super ▁x x bo s ▁mam mo o kka aa ▁ xxrep ▁4 ▁. ▁ni gha l ▁mu tha anu ▁ xxrep ▁4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>x bo s ▁la le ttan ▁marana ▁mass . . in tha ▁movie ▁ba yang ara ▁hit ▁adi kku m . . . lo ve ▁from ▁tamilnadu ▁x x bo s ▁oru ▁mass ▁political - ▁family ▁block bu ster ▁hit ▁aa ka tte yenn ▁aa sham si kkunnu ▁x x bo s ▁tra il or ▁kand ▁8 ▁nila yil ▁pot tum ▁enn ▁tho n unnu ▁mu mba the ▁pole ▁tho</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>▁chari th ram ▁aka nam enkil ▁mara kkar ▁vara nam ▁x x bo s ▁prithviraj . ▁ni gha l ▁oru ▁marana ▁mass ▁samba vam ▁x x bo s ▁kerala ▁die ▁hard ▁mohanlal ▁fan s ▁like ▁here ▁x x bo s ▁da ▁ mw one ▁madhura ja ed at re um ▁vari lla ▁ke tto ▁. nte ▁va ka ▁ella r kum ▁chu ka va nda nam ▁x x bo s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm, arch=AWD_LSTM, drop_mult=0.3, pretrained=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(25000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=25000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(25000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=25000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=25000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=25000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('../../dataset_preparation/models/best_model', with_opt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.645034</td>\n",
       "      <td>5.009266</td>\n",
       "      <td>0.243973</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fit_head', with_opt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fit_head', with_opt=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.950925</td>\n",
       "      <td>4.676664</td>\n",
       "      <td>0.272991</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.659377</td>\n",
       "      <td>4.209945</td>\n",
       "      <td>0.324107</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.402082</td>\n",
       "      <td>3.995071</td>\n",
       "      <td>0.346577</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.225903</td>\n",
       "      <td>3.907428</td>\n",
       "      <td>0.355580</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.105953</td>\n",
       "      <td>3.893269</td>\n",
       "      <td>0.356399</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fine_tuned', with_opt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fine_tuned', with_opt=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Evideo oru Hollywood story ho p ▁e nu kku ▁aa thu ▁nigeria i .'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict('Evideo oru Hollywood story',n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas = TextClasDataBunch.from_df(path=path, train_df=df_train, valid_df=df_valid, test_df=df_test, tokenizer=tokenizer, vocab=mlen_vocab, bs=128, label_cols=label_cols, text_cols=text_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁cast ▁ xxunk ▁r ▁mammootty ▁ xxunk ▁r ▁unni ▁muk und an ▁ xxunk ▁r ▁pra chi ▁te h lan ▁ xxunk ▁r ▁siddique ▁ xxunk ▁r ▁achutha n ▁ xxunk ▁r ▁anu si tara ▁ xxunk ▁r ▁in iya ▁ xxunk ▁r ▁kani ha ▁ xxunk ▁r ▁tar un ▁ar ora ▁ xxunk ▁r ▁suresh ▁krishna ▁ xxunk ▁r ▁mani ku ttan ▁ xxunk ▁r</td>\n",
       "      <td>unknown_state</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁sh ▁ xxrep ▁5 ▁o ▁ xxrep ▁4 ▁. ▁9 ▁mani kku ▁kaa nan ▁tho da ng iya dha a ▁ xxrep ▁4 ▁. ▁12 : 30 ▁a ayi ▁ xxrep ▁4 ▁. ▁e thra ▁ vattam ▁kand u ▁enna dh inum ▁kana kki laa ▁ xxrep ▁4 ▁. ▁nir tha an ▁pattu nila lo ▁ xxrep ▁5 ▁. ▁oru ▁katta ▁mohanlal ▁prithviraj ▁aaradhak an de</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁all a ▁ xxrep ▁4 ▁. ▁mana s ila ly ▁ xxrep ▁4 ▁o ▁ xxrep ▁7 ▁. ▁vi swa s ichu ▁ko o de varunn a vare ▁je e e vana an ▁ xxrep ▁5 ▁. ▁e etu ▁ xxrep ▁4 ▁. ▁jeevan ▁ xxrep ▁5 ▁. ▁kodu tha yaa alu m ▁ xxrep ▁6 ▁. ▁sam raksh ich ir kku m ▁ xxrep ▁7</td>\n",
       "      <td>unknown_state</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁ull ath ▁para ya llo oo ▁si dhi q ▁o zhi ch ▁bha a kki ▁el lla aam ▁chali ▁actor ors ▁ xxrep ▁4 ▁. ▁e ntha a avu mo oo ▁en th ▁ xxrep ▁4 ▁o ▁ xxrep ▁4 ▁. ▁com ment ▁no o ki ii . . . ▁fay ankara m ▁ennu ▁para ya an ▁e e e ▁tra il or il ▁onnu m</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁kuru pu uti varu nav r de ▁an ak il ▁kod tha ▁ke e chan ▁trai ler ▁i kka aa ▁u ff ▁in gal ▁e e e ja thi ▁mass ▁f d f s ▁ura ppa chu u ▁sing a sa ▁tha li va a ▁ xxrep ▁4 ▁. ▁mad ura raja ▁ xxunk ▁raja ef fect ▁ xxunk ▁ri p ▁you tu be ▁ thu</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas.sanity_check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, arch=AWD_LSTM, drop_mult=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CrossEntropyLoss()"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.loss_func.func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcc = MatthewsCorreff()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.metrics = [mcc, accuracy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.220221</td>\n",
       "      <td>1.039873</td>\n",
       "      <td>0.433369</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('first-full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('first-full');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.044585</td>\n",
       "      <td>0.812918</td>\n",
       "      <td>0.618710</td>\n",
       "      <td>0.731481</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('second-full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('second-full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.870312</td>\n",
       "      <td>0.520581</td>\n",
       "      <td>0.739501</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.847661</td>\n",
       "      <td>0.503152</td>\n",
       "      <td>0.759501</td>\n",
       "      <td>0.827778</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.689732</td>\n",
       "      <td>0.224443</td>\n",
       "      <td>0.903708</td>\n",
       "      <td>0.931481</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.513973</td>\n",
       "      <td>0.090372</td>\n",
       "      <td>0.971418</td>\n",
       "      <td>0.979630</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.350574</td>\n",
       "      <td>0.076525</td>\n",
       "      <td>0.979178</td>\n",
       "      <td>0.985185</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(5, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "learn.save('final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (5391 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁ho o ▁mam mo kka ▁police ▁vesha m ▁aa ha ▁ant has,▁x x bo s ▁oru ▁re k shay um ▁i lla . . . ki di lam ▁kannu ▁na na nju poy i,▁x x bo s ▁i kka ▁wait ing ▁ xxrep ▁9 ▁.,▁x x bo s ▁raj u ▁etta nte ▁oro ▁short tum ▁i ja thi ▁ pp w li,▁x x bo s ▁et tan ▁fan sil ▁ne tti ▁po ya ▁a ar enkil um ▁und o xxunk ▁ xxunk ▁mad ura raja ▁ xxunk ▁wait ing ▁ xxunk\n",
       "y: CategoryList\n",
       "Positive ,Positive ,Positive ,Positive ,Positive \n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (540 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁speech less ▁ xxunk . ▁i kka aa,▁x x bo s ▁raja ▁so llu nath u ▁mat tu tha am ▁se y y va a ▁se y yu n nath ▁mat tum ▁tha a ▁so l va a,▁x x bo s ▁im ▁pri thi vi raj ▁fan ▁from ▁tamilnadu . . . ▁love ▁it,▁x x bo s ▁mohanlal ▁sir ▁- ▁look ▁ xxrep ▁5 ▁. ▁ki d do . . .,▁x x bo s ▁kanda thil ▁ve ch ▁mu ng iya ▁p dam ▁ra ting ▁1 .1 ▁/ ▁5\n",
       "y: CategoryList\n",
       "not-malayalam ,not-malayalam ,not-malayalam ,Positive ,Negative \n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1348 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁bollywood ▁film ▁new ton ▁inte ▁re ma ke ▁aan o xxunk,▁x x bo s ▁end ukond ▁view rs ▁ko od unnill a ▁ xxunk ▁ippo zhu m ▁2 .8 m ▁a ayi to llu,▁x x bo s ▁mara ▁pa a zhu ▁mega ▁mai rana nil ▁ninnu m ▁ethi l ▁ko od u tal ▁pra the e shi karu thu ▁1980 ▁ kalile ▁raja ni kanth inu ▁pa di kkunnu ▁veru m ▁cha varu ▁mai ran ▁ xxrep ▁4 ▁.,▁x x bo s ▁video ▁nay ▁can g ▁x em ▁can g ▁thi t,▁x x bo s ▁sun ny ▁che chi ye ▁kaa nan ▁vannat hu ▁ njan ▁maa th ram ▁aan o\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(25000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(25000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7fcb80fffd08>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(25000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(25000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>actual_label</th>\n",
       "      <th>predicted_label</th>\n",
       "      <th>unknown_state</th>\n",
       "      <th>not-malayalam</th>\n",
       "      <th>Mixed_feelings</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>speechless 🤐.   ikkaaa</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>0.00321969</td>\n",
       "      <td>0.983089</td>\n",
       "      <td>0.000970477</td>\n",
       "      <td>0.000786809</td>\n",
       "      <td>0.0119337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Raja sollunathu mattuthaam seyyvaa seyyunnath...</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>4.22216e-07</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>1.17635e-07</td>\n",
       "      <td>3.19529e-08</td>\n",
       "      <td>4.0478e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Im Prithiviraj fan from tamilnadu... Love it</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>not-malayalam</td>\n",
       "      <td>3.41896e-05</td>\n",
       "      <td>0.997794</td>\n",
       "      <td>0.000224901</td>\n",
       "      <td>7.11995e-05</td>\n",
       "      <td>0.00187623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mohanlal sir - look ..... kiddo...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.0148137</td>\n",
       "      <td>0.131026</td>\n",
       "      <td>0.00122068</td>\n",
       "      <td>0.00103087</td>\n",
       "      <td>0.851909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kandathil vech mungiya pdam  Rating 1.1/5</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>0.0996198</td>\n",
       "      <td>0.00818416</td>\n",
       "      <td>0.00108131</td>\n",
       "      <td>0.856447</td>\n",
       "      <td>0.0346682</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               query    actual_label  \\\n",
       "0                             speechless 🤐.   ikkaaa  not-malayalam    \n",
       "1   Raja sollunathu mattuthaam seyyvaa seyyunnath...  not-malayalam    \n",
       "2       Im Prithiviraj fan from tamilnadu... Love it  not-malayalam    \n",
       "3                 mohanlal sir - look ..... kiddo...       Positive    \n",
       "4          Kandathil vech mungiya pdam  Rating 1.1/5       Negative    \n",
       "\n",
       "  predicted_label unknown_state  not-malayalam  Mixed_feelings     Negative   \\\n",
       "0  not-malayalam      0.00321969       0.983089     0.000970477  0.000786809   \n",
       "1  not-malayalam     4.22216e-07       0.999999     1.17635e-07  3.19529e-08   \n",
       "2  not-malayalam     3.41896e-05       0.997794     0.000224901  7.11995e-05   \n",
       "3       Positive       0.0148137       0.131026      0.00122068   0.00103087   \n",
       "4       Negative       0.0996198     0.00818416      0.00108131     0.856447   \n",
       "\n",
       "    Positive   \n",
       "0   0.0119337  \n",
       "1  4.0478e-08  \n",
       "2  0.00187623  \n",
       "3    0.851909  \n",
       "4   0.0346682  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = df_valid.copy()\n",
    "from sklearn.metrics import accuracy_score, matthews_corrcoef, f1_score\n",
    "df_dict = {'query': list(df_test['text']), 'actual_label': list(df_test['category']), 'predicted_label': ['']*df_test.shape[0]}\n",
    "all_nodes = list(set(df_train['category']))\n",
    "for node in all_nodes:\n",
    "    df_dict[node] = ['']*df_test.shape[0]\n",
    "    \n",
    "i2c = {}\n",
    "for key, value in learn.data.c2i.items():\n",
    "    i2c[value] = key\n",
    "    \n",
    "df_result = pd.DataFrame(df_dict)\n",
    "preds = learn.get_preds(ds_type=DatasetType.Valid, ordered=True)\n",
    "for index, row in df_result.iterrows():\n",
    "    for node in all_nodes:\n",
    "        row[node] = preds[0][index][learn.data.c2i[node]].item()\n",
    "    row['predicted_label'] = i2c[np.argmax(preds[0][index]).data.item()]\n",
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9851851851851852"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(df_result['actual_label'], df_result['predicted_label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9791775467684853"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matthews_corrcoef(df_result['actual_label'], df_result['predicted_label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9851470733984172"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(df_result['actual_label'], df_result['predicted_label'], average='weighted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>unknown_state</th>\n",
       "      <th>not-malayalam</th>\n",
       "      <th>Mixed_feelings</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ml_sen_1</td>\n",
       "      <td>Bollywood film Newton inte remake aano?</td>\n",
       "      <td>Mixed_feelings</td>\n",
       "      <td>0.0315095</td>\n",
       "      <td>0.0116891</td>\n",
       "      <td>0.943262</td>\n",
       "      <td>0.00713557</td>\n",
       "      <td>0.00640415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ml_sen_2</td>\n",
       "      <td>endukond viewrs koodunnilla ?? ippozhum 2.8m a...</td>\n",
       "      <td>unknown_state</td>\n",
       "      <td>0.937249</td>\n",
       "      <td>0.00180892</td>\n",
       "      <td>0.00215666</td>\n",
       "      <td>0.00473083</td>\n",
       "      <td>0.054055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ml_sen_3</td>\n",
       "      <td>Mara paazhu mega mairananil ninnum ethil koodu...</td>\n",
       "      <td>unknown_state</td>\n",
       "      <td>0.581574</td>\n",
       "      <td>0.0235574</td>\n",
       "      <td>0.157017</td>\n",
       "      <td>0.11001</td>\n",
       "      <td>0.127841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ml_sen_4</td>\n",
       "      <td>Video nay cang xem cang thit</td>\n",
       "      <td>unknown_state</td>\n",
       "      <td>0.709561</td>\n",
       "      <td>0.207912</td>\n",
       "      <td>0.00976879</td>\n",
       "      <td>0.0402459</td>\n",
       "      <td>0.032512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ml_sen_5</td>\n",
       "      <td>Sunny chechiye kaanan vannathu njan maathram aano</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.451647</td>\n",
       "      <td>0.00063324</td>\n",
       "      <td>0.00140113</td>\n",
       "      <td>0.00090685</td>\n",
       "      <td>0.545412</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                               text  \\\n",
       "0  ml_sen_1            Bollywood film Newton inte remake aano?   \n",
       "1  ml_sen_2  endukond viewrs koodunnilla ?? ippozhum 2.8m a...   \n",
       "2  ml_sen_3  Mara paazhu mega mairananil ninnum ethil koodu...   \n",
       "3  ml_sen_4                       Video nay cang xem cang thit   \n",
       "4  ml_sen_5  Sunny chechiye kaanan vannathu njan maathram aano   \n",
       "\n",
       "          category unknown_state  not-malayalam  Mixed_feelings    Negative   \\\n",
       "0  Mixed_feelings       0.0315095      0.0116891        0.943262  0.00713557   \n",
       "1   unknown_state        0.937249     0.00180892      0.00215666  0.00473083   \n",
       "2   unknown_state        0.581574      0.0235574        0.157017     0.11001   \n",
       "3   unknown_state        0.709561       0.207912      0.00976879   0.0402459   \n",
       "4        Positive        0.451647     0.00063324      0.00140113  0.00090685   \n",
       "\n",
       "    Positive   \n",
       "0  0.00640415  \n",
       "1    0.054055  \n",
       "2    0.127841  \n",
       "3    0.032512  \n",
       "4    0.545412  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv(path/'../dc_fire/malayalam_test.tsv', sep='\\t')\n",
    "from sklearn.metrics import accuracy_score, matthews_corrcoef, f1_score\n",
    "df_dict = {'id': list(df_test['id']), 'text': list(df_test['text']), 'category': ['']*df_test.shape[0]}\n",
    "all_nodes = list(set(df_train['category']))\n",
    "for node in all_nodes:\n",
    "    df_dict[node] = ['']*df_test.shape[0]\n",
    "    \n",
    "i2c = {}\n",
    "for key, value in learn.data.c2i.items():\n",
    "    i2c[value] = key\n",
    "    \n",
    "df_result = pd.DataFrame(df_dict)\n",
    "preds = learn.get_preds(ds_type=DatasetType.Test, ordered=True)\n",
    "for index, row in df_result.iterrows():\n",
    "    for node in all_nodes:\n",
    "        row[node] = preds[0][index][learn.data.c2i[node]].item()\n",
    "    row['category'] = i2c[np.argmax(preds[0][index]).data.item()]\n",
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Mixed_feelings ': 55,\n",
       "         'Negative ': 146,\n",
       "         'Positive ': 588,\n",
       "         'not-malayalam ': 188,\n",
       "         'unknown_state ': 371})"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(df_result['category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv('dc_fire_full.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1348, 8)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(371, 8)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result[df_result['category']=='unknown_state '].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}