classification/Marathi_Classification_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "import sentencepiece as spm\n",
    "import re\n",
    "import pdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('1.0.57', '1.0.0')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import fastai, torch\n",
    "fastai.__version__ , torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_seed(seed_value, use_cuda):\n",
    "    np.random.seed(seed_value)  \n",
    "    torch.manual_seed(seed_value)  \n",
    "    random.seed(seed_value)\n",
    "    if use_cuda:\n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)  \n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed(42, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/home/ubuntu/gaurav/in/nlp-for-marathi/classification\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>…म्हणून सानिया मिर्झाची ड्यू डेट आहे खास</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>Video : दीपिका-रणवीरच्या लग्नाचा मेन्यू झाला लीक!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sports</td>\n",
       "      <td>सचिनच्या रणजी कारकीर्दीचा शेवट गोड, मुंबईचा विजय</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>state</td>\n",
       "      <td>पुरंदरेंना महाराष्ट्र भूषण पुरस्काराविरोधात सं...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sports</td>\n",
       "      <td>जयपूर पोलिसांच्या जाहिरातीवर बुमराह संतापला, स...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0                                                  1\n",
       "0  entertainment           …म्हणून सानिया मिर्झाची ड्यू डेट आहे खास\n",
       "1  entertainment  Video : दीपिका-रणवीरच्या लग्नाचा मेन्यू झाला लीक!\n",
       "2         sports   सचिनच्या रणजी कारकीर्दीचा शेवट गोड, मुंबईचा विजय\n",
       "3          state  पुरंदरेंना महाराष्ट्र भूषण पुरस्काराविरोधात सं...\n",
       "4         sports  जयपूर पोलिसांच्या जाहिरातीवर बुमराह संतापला, स..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(path/'../../classification_public_datasets/inltk-headlines/mr/mr-train.csv', header=None)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sports</td>\n",
       "      <td>CWG 2018 : संजिता चानूची सुवर्णभरारी, भारताच्य...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>state</td>\n",
       "      <td>कर्मचार्‍यांच्या हलगर्जीपणामुळे होणार होते जिव...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sports</td>\n",
       "      <td>श्रीलंकेचा भारतावर 7 गडी राखून विजय</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>state</td>\n",
       "      <td>औरंगाबाद महापौरपदाचा तिढा अखेर सुटला</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>state</td>\n",
       "      <td>औरंगाबाद विद्यापीठाचा भोंगळ कारभार; गोदामात दि...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        0                                                  1\n",
       "0  sports  CWG 2018 : संजिता चानूची सुवर्णभरारी, भारताच्य...\n",
       "1   state  कर्मचार्‍यांच्या हलगर्जीपणामुळे होणार होते जिव...\n",
       "2  sports                श्रीलंकेचा भारतावर 7 गडी राखून विजय\n",
       "3   state               औरंगाबाद महापौरपदाचा तिढा अखेर सुटला\n",
       "4   state  औरंगाबाद विद्यापीठाचा भोंगळ कारभार; गोदामात दि..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid = pd.read_csv(path/'../../classification_public_datasets/inltk-headlines/mr/mr-valid.csv', header=None)\n",
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>काजोल पुन्हा माझ्या आयुष्यात येणार नाही - करण ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>विराटला चीअर करण्यासाठी अनुष्का सिडनीत</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>संतोषचा अड्डा!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>state</td>\n",
       "      <td>मुरूड समुद्रात बुडालेल्या विद्यार्थ्याचा मृतदे...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>state</td>\n",
       "      <td>गोव्यात शिवसेना वेलिंगकरांसोबत,लवकरच युतीची घोषणा</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0                                                  1\n",
       "0  entertainment  काजोल पुन्हा माझ्या आयुष्यात येणार नाही - करण ...\n",
       "1  entertainment             विराटला चीअर करण्यासाठी अनुष्का सिडनीत\n",
       "2  entertainment                                     संतोषचा अड्डा!\n",
       "3          state  मुरूड समुद्रात बुडालेल्या विद्यार्थ्याचा मृतदे...\n",
       "4          state  गोव्यात शिवसेना वेलिंगकरांसोबत,लवकरच युतीची घोषणा"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv(path/'../../classification_public_datasets/inltk-headlines/mr/mr-test.csv', header=None)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((9672, 2), (1210, 2), (1210, 2))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape, df_valid.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((0, 2), (0, 2), (0, 2))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[df_train[0].isnull()].shape, df_valid[df_valid[0].isnull()].shape, df_test[df_test[0].isnull()].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_cols = [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MarathiTokenizer(BaseTokenizer):\n",
    "    def __init__(self, lang:str):\n",
    "        self.lang = lang\n",
    "        self.sp = spm.SentencePieceProcessor()\n",
    "        self.sp.Load(str('./../../models/marathi/tokenizer/marathi_lm.model'))\n",
    "        \n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        return self.sp.EncodeAsPieces(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(str('./../../models/marathi/tokenizer/marathi_lm.model'))\n",
    "itos = [sp.IdToPiece(int(i)) for i in range(30000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 30,000 is the vocab size that we chose in sentencepiece\n",
    "marathi_vocab = Vocab(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(tok_func=MarathiTokenizer, lang='mr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xxunk',\n",
       " 'xxpad',\n",
       " 'xxbos',\n",
       " 'xxeos',\n",
       " 'xxfld',\n",
       " 'xxmaj',\n",
       " 'xxup',\n",
       " 'xxrep',\n",
       " 'xxwrep']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.special_cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm = TextLMDataBunch.from_df(path=path, train_df=df_train, valid_df=df_valid, test_df=df_test, tokenizer=tokenizer, vocab=marathi_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल ▁x x bo s ▁संतप्त ▁ग्रामस्थ ांनी ▁राष्ट्रवादी ▁आमदार ाच्या ▁दिशे नं ▁भिरकाव ल्या ▁पाण्याने ▁भरलेल्या ▁घागरी ▁x x bo s &lt;unk&gt; &lt;unk&gt; p e c i &lt;unk&gt; al &lt;unk&gt; ▁r e p &lt;unk&gt; or t ▁: ▁लातूर मध्ये ▁का ▁करावा ▁लागतो य ▁विद्यार्थ्यांना ▁एसटी ने ▁लटक ून ▁प्रवास ? ▁x x bo s ▁यश</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>अजूनही ▁मो का ट च ▁x x bo s ▁रि पा इ ंमध्ये ▁मतभेद ▁चव्हा ट ्यावर , ▁डांग ळे ंची ▁हकालपट्टी ▁आणि ▁मन धरण ी ही ▁! ▁x x bo s ▁मतदार या द्य ांचा ▁घोळ , ▁अनेक ▁जण ▁मतदान ाला ▁मु कले ▁x x bo s ▁ सई ▁आणि ▁ प्रिया ▁झाल्या ▁' व जन दार ' ▁x x bo s ▁न्यूझीलंड ची ▁दक्षिण ▁आफ्रिके वर ▁मात ,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>वर ▁गुन्हा ▁दाखल ▁x x bo s ▁भुजबळ - ▁राज ▁ठाकरे ▁यांच्यात ▁चर्चा , ▁पाहा ▁भेटी चे &lt;unk&gt; ▁e &lt;unk&gt; &lt;unk&gt; l u &lt;unk&gt; si v e &lt;unk&gt; ▁p h o t &lt;unk&gt; os ▁x x bo s ▁दुरुस्ती च्या ▁काम ामुळे ▁एक्स्प्रेस - वे ची ▁वाहतूक ▁जुन्या ▁मार्गाने ▁x x bo s ▁गप्पा ▁नौदल ▁अभ्यासक ▁डॉ . ▁दत्तात्रय ▁केतकर ांशी ▁: ▁भाग ▁5 ▁x x bo s ▁दुष्काळ ग्रस्त</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>, येव ल्या त ▁जल्लोष ! ▁x x bo s ▁पुण्याचे ▁हे ▁पाच ▁खेळाडू ▁होऊ ▁शकतात ▁मुंबई वर ▁भारी ▁x x bo s ▁शिव बंधन ाचा ▁धागा ▁तुट ला , ▁वाक चौ रे ▁काँग्रेस मध्ये ▁! ▁x x bo s ▁अण्णां ची ▁मन धरण ी ▁करण्यासाठी ▁गिरीश ▁महाजन ▁रा ळे गण सिद्धी त ▁! ▁x x bo s ▁ हेल्म ेट स क्त ी ▁ही ▁संपूर्ण ▁महाराष्ट्र ासाठी - ▁दिवाकर</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>bo s ▁औरंगाबाद ▁आणि ▁परभणी मध्ये ▁डे ंग ्यू च्या ▁ताप ाची ▁साथ ? ▁x x bo s ▁प्रेम विवाह ▁केलेल्या ▁पती ने च ▁केला ▁पत्नी चा ▁गळा ▁कापून ▁खून ▁x x bo s ▁सचिन &lt;unk&gt; ▁un i c e f ▁ चा ▁ &lt;unk&gt; म्ब ॉस े डर ▁x x bo s ▁मुंबई चा ▁खरा ▁वाली ▁कोण , ▁केव्हा ▁संपणार ▁नागरिकां ची ▁द शा ? ▁x x bo s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "awd_lstm_config = awd_lstm_lm_config.copy()\n",
    "awd_lstm_config['n_hid'] = 1150\n",
    "learn = language_model_learner(data_lm, arch=AWD_LSTM, drop_mult=0.3, config=awd_lstm_config, pretrained=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: LMTextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(30000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1150, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1150, 1150, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1150, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=30000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: LanguageLearner(data=TextLMDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: LMTextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: LMLabelList\n",
       ",,,,\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: LMTextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(30000, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1150, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1150, 1150, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1150, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=30000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=30000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "  (2): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=30000, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loading the pretrained language model on kannada wikipedia\n",
    "learn.load('../../../models/marathi/lm/ULMFiT/fourth_mr_lm', with_opt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.539380</td>\n",
       "      <td>5.002946</td>\n",
       "      <td>0.317455</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.744837</td>\n",
       "      <td>4.623729</td>\n",
       "      <td>0.342277</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.473907</td>\n",
       "      <td>4.359137</td>\n",
       "      <td>0.368214</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.233382</td>\n",
       "      <td>4.248090</td>\n",
       "      <td>0.379509</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.037029</td>\n",
       "      <td>4.213844</td>\n",
       "      <td>0.385000</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.919879</td>\n",
       "      <td>4.206117</td>\n",
       "      <td>0.385893</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'सीमा भागातील मराठी ▁घर च ▁सोन ी चं ▁वेग ▁24 00 4 ▁खुले'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict('सीमा भागातील मराठी',n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas = TextClasDataBunch.from_df(path=path, train_df=df_train, valid_df=df_valid, test_df=df_test, tokenizer=tokenizer, vocab=marathi_vocab, label_cols=label_cols, bs=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>▁x x bo s &lt;unk&gt; ▁l ive ▁cricket &lt;unk&gt; &lt;unk&gt; co re , &lt;unk&gt; ▁v s &lt;unk&gt; ▁2 nd ▁test , ▁4 th ▁ &lt;unk&gt; day - ▁ल ंच नंतर ▁ऑस्ट्रेलिया ला ▁दोन ▁धक्के , ▁शमी ने च ▁घेतल्या ▁दोन्ही ▁विकेट , ▁पेन - ▁फिंच ▁बाद</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s &lt;unk&gt; ▁p ul wa ma &lt;unk&gt; ▁t er r or &lt;unk&gt; ▁a tt ack - ▁सर ण ावर ची ▁आग ▁ अजूनही ▁व िझ ली ▁नाही , ▁मुले ▁पौ र की ▁शाही द ांची ▁हो ▁निज ली ▁नाही - ▁जितेंद्र ▁जोशी</td>\n",
       "      <td>entertainment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s ▁ &lt;unk&gt; &lt;unk&gt; ▁i n d v e n g ▁ &lt;unk&gt; &lt;unk&gt; ▁women ' s ▁world &lt;unk&gt; ▁t 20 ▁: ▁महिला ▁टीम चं ▁बॅ डल क , ▁सगळे ▁सामने ▁जिंकून ▁सेमी फाय न ल मध्ये ▁पराभूत</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s &lt;unk&gt; ▁l ive ▁cricket &lt;unk&gt; &lt;unk&gt; co re , &lt;unk&gt; ▁v s &lt;unk&gt; ▁4 th &lt;unk&gt; &lt;unk&gt; - ▁ए श्ट र ▁टर्नर ने ▁भारताकडून ▁हिसका वला ▁सामना , ▁मालिकेत ▁२ - ▁२ ▁ ची ▁बरोबर</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁x x bo s &lt;unk&gt; &lt;unk&gt; p e c i &lt;unk&gt; al &lt;unk&gt; ▁r e p &lt;unk&gt; or t ▁: ▁व र्दी शी ▁ग द्द ारी , ▁खा की वाले ▁झाले ▁ड्र ग् ज ▁माफिया चे ▁खबर ी !</td>\n",
       "      <td>state</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "del awd_lstm_config['tie_weights']\n",
    "del awd_lstm_config['out_bias']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, arch=AWD_LSTM, drop_mult=0.5, config=awd_lstm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: CategoryList\n",
       "entertainment,entertainment,sports,state,sports\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: CategoryList\n",
       "sports,state,sports,state,state\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(30000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: CategoryList\n",
       "entertainment,entertainment,sports,state,sports\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: CategoryList\n",
       "sports,state,sports,state,state\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(30000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CrossEntropyLoss()"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.loss_func.func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcc = MatthewsCorreff()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.metrics = [mcc, accuracy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.403414</td>\n",
       "      <td>0.328715</td>\n",
       "      <td>0.752030</td>\n",
       "      <td>0.872727</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.304916</td>\n",
       "      <td>0.264528</td>\n",
       "      <td>0.823941</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('second-full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>matthews_correff</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.203628</td>\n",
       "      <td>0.237077</td>\n",
       "      <td>0.847671</td>\n",
       "      <td>0.921488</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.163873</td>\n",
       "      <td>0.222175</td>\n",
       "      <td>0.858373</td>\n",
       "      <td>0.926446</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.070179</td>\n",
       "      <td>0.219261</td>\n",
       "      <td>0.862041</td>\n",
       "      <td>0.928926</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.037321</td>\n",
       "      <td>0.227653</td>\n",
       "      <td>0.868663</td>\n",
       "      <td>0.932231</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.033704</td>\n",
       "      <td>0.228375</td>\n",
       "      <td>0.868562</td>\n",
       "      <td>0.932231</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.9214876294136047.\n",
      "Better model found at epoch 1 with accuracy value: 0.926446259021759.\n",
      "Better model found at epoch 2 with accuracy value: 0.928925633430481.\n",
      "Better model found at epoch 3 with accuracy value: 0.9322314262390137.\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(5, 1e-3, callbacks=[callbacks.SaveModelCallback(learn, every='improvement', monitor='accuracy', name='final')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: CategoryList\n",
       "entertainment,entertainment,sports,state,sports\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: CategoryList\n",
       "sports,state,sports,state,state\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(30000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (9672 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁... म्हणून ▁सान िया ▁मिर्झा ची ▁ड्यू ▁डे ट ▁आहे ▁खास,▁x x bo s <unk> ▁v ide o ▁: ▁दीपिका - रण वीर च्या ▁लग्न ाचा ▁मे न्यू ▁झाला ▁ली क !,▁x x bo s ▁सचिन च्या ▁रणजी ▁कारकीर्दी चा ▁शेवट ▁गोड , ▁मुंबई चा ▁विजय,▁x x bo s ▁पुरंदरे ंना ▁महाराष्ट्र ▁भूषण ▁पुरस्कार ाविरोधात ▁संभाजी ▁ब्रिगेड ▁रस्त्यावर,▁x x bo s ▁जयपूर ▁पोलिस ांच्या ▁जाहिराती वर ▁बु म राह ▁संताप ला , ▁सु ना वले ▁खडे बोल\n",
       "y: CategoryList\n",
       "entertainment,entertainment,sports,state,sports\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s <unk> ▁c w g ▁2018 ▁: ▁सं जित ा ▁ चा नू ची ▁सुवर्ण भर ारी , ▁भारताच्या ▁खात्यात ▁दुसरे ▁सुवर्ण ▁!,▁x x bo s ▁कर्मचार् ▁यांच्या ▁हल ग र्ज ी पणामुळे ▁होणार ▁होते ▁जिवंत ▁बाळ ावर ▁अंत्यसंस्कार,▁x x bo s ▁श्रीलंकेचा ▁भारतावर ▁7 ▁गडी ▁राखून ▁विजय,▁x x bo s ▁औरंगाबाद ▁महापौर पदाचा ▁ति ढा ▁अखेर ▁सुट ला,▁x x bo s ▁औरंगाबाद ▁विद्यापीठाचा ▁भो ंग ळ ▁कारभार ; ▁गोदाम ात ▁दिले ▁विद्यार्थ्यांनी ▁पेपर\n",
       "y: CategoryList\n",
       "sports,state,sports,state,state\n",
       "Path: .;\n",
       "\n",
       "Test: LabelList (1210 items)\n",
       "x: TextList\n",
       "▁x x bo s ▁काजोल ▁पुन्हा ▁माझ्या ▁आयुष्यात ▁येणार ▁नाही ▁- ▁करण ▁जोहर,▁x x bo s ▁विराट ला ▁ ची अर ▁करण्यासाठी ▁अ नुष्का ▁सिडनी त,▁x x bo s ▁संतोष चा ▁अ ड्डा !,▁x x bo s ▁मुरूड ▁समुद्रात ▁बुडाले ल्या ▁विद्यार्थ्या चा ▁मृतदेह ▁सापडला,▁x x bo s ▁गोव्यात ▁शिवसेना ▁वे लिंग कर ांसोबत , ल व कर च ▁युती ची ▁घोषणा\n",
       "y: EmptyLabelList\n",
       ",,,,\n",
       "Path: ., model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(30000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(30000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[MatthewsCorreff(), <function accuracy at 0x7f2bcdbe8c80>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(30000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(30000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.05)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False, cb_fns_registered=True)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>actual_label</th>\n",
       "      <th>predicted_label</th>\n",
       "      <th>entertainment</th>\n",
       "      <th>state</th>\n",
       "      <th>sports</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>काजोल पुन्हा माझ्या आयुष्यात येणार नाही - करण ...</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>0.999083</td>\n",
       "      <td>5.68735e-05</td>\n",
       "      <td>0.000860293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>विराटला चीअर करण्यासाठी अनुष्का सिडनीत</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>sports</td>\n",
       "      <td>0.282717</td>\n",
       "      <td>0.000120957</td>\n",
       "      <td>0.717162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>संतोषचा अड्डा!</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>0.920608</td>\n",
       "      <td>0.0321119</td>\n",
       "      <td>0.0472798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>मुरूड समुद्रात बुडालेल्या विद्यार्थ्याचा मृतदे...</td>\n",
       "      <td>state</td>\n",
       "      <td>state</td>\n",
       "      <td>0.00119353</td>\n",
       "      <td>0.996408</td>\n",
       "      <td>0.00239817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>गोव्यात शिवसेना वेलिंगकरांसोबत,लवकरच युतीची घोषणा</td>\n",
       "      <td>state</td>\n",
       "      <td>state</td>\n",
       "      <td>0.000146009</td>\n",
       "      <td>0.999794</td>\n",
       "      <td>5.95854e-05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               query   actual_label  \\\n",
       "0  काजोल पुन्हा माझ्या आयुष्यात येणार नाही - करण ...  entertainment   \n",
       "1             विराटला चीअर करण्यासाठी अनुष्का सिडनीत  entertainment   \n",
       "2                                     संतोषचा अड्डा!  entertainment   \n",
       "3  मुरूड समुद्रात बुडालेल्या विद्यार्थ्याचा मृतदे...          state   \n",
       "4  गोव्यात शिवसेना वेलिंगकरांसोबत,लवकरच युतीची घोषणा          state   \n",
       "\n",
       "  predicted_label entertainment        state       sports  \n",
       "0   entertainment      0.999083  5.68735e-05  0.000860293  \n",
       "1          sports      0.282717  0.000120957     0.717162  \n",
       "2   entertainment      0.920608    0.0321119    0.0472798  \n",
       "3           state    0.00119353     0.996408   0.00239817  \n",
       "4           state   0.000146009     0.999794  5.95854e-05  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, matthews_corrcoef\n",
    "df_dict = {'query': list(df_test[1]), 'actual_label': list(df_test[0]), 'predicted_label': ['']*df_test.shape[0]}\n",
    "all_nodes = list(set(df_train[0]))\n",
    "for node in all_nodes:\n",
    "    df_dict[node] = ['']*df_test.shape[0]\n",
    "    \n",
    "i2c = {}\n",
    "for key, value in learn.data.c2i.items():\n",
    "    i2c[value] = key\n",
    "    \n",
    "df_result = pd.DataFrame(df_dict)\n",
    "preds = learn.get_preds(ds_type=DatasetType.Test, ordered=True)\n",
    "for index, row in df_result.iterrows():\n",
    "    for node in all_nodes:\n",
    "        row[node] = preds[0][index][learn.data.c2i[node]].item()\n",
    "    row['predicted_label'] = i2c[np.argmax(preds[0][index]).data.item()]\n",
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9239669421487603"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(df_result['actual_label'], df_result['predicted_label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8522918155778538"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matthews_corrcoef(df_result['actual_label'], df_result['predicted_label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv('inltk_headlines_mr.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "in",
   "language": "python",
   "name": "in"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}