cuda_bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8c206d3-696c-4ca4-864c-549b21e00cdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as snsbert_data\n",
    "from pylab import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "\n",
    "from stance_utils import *\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "# sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "# HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
    "\n",
    "# sns.set_palette(sns.color_palette(HAPPY_COLORbert_dataS_PALETTE))\n",
    "# rcParams['figure.figsize'] = 12, 8\n",
    "\n",
    "RANDOM_SEED = 23\n",
    "np.random.seed(RANDOM_SEED)\n",
    "\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8abdf725",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e68af5eb-e441-4ff9-ba37-292d9c64f4a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cdc86008-d2c3-4a74-a1c4-19a113421633",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NVIDIA A40'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.get_device_name(device=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4fe14356",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a5e07682",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    x_train = []\n",
    "    y_train = []\n",
    "\n",
    "    \n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        for line in trainfile:\n",
    "            \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            \n",
    "            if line[0].strip() != 'ID' and target not in line[1].strip():\n",
    "            #if line[0].strip() != 'ID':  #Uncomment this line if training on wholedataset and comment the line above.\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_train.append(tweet)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "                \n",
    "#     with open('rich_data_feminism_athesim_abortion.txt','r') as newfile:\n",
    "#         for line in newfile:\n",
    "#             line = line.replace('#SemST', '').strip()\n",
    "#             line = line.split('\\t')\n",
    "#             tweet = line[0]\n",
    "#             tweet = process_tweet(tweet)\n",
    "#             if len(tweet) > sentence_maxlen:\n",
    "#                 sentence_maxlen = len(tweet)\n",
    "#             x_train.append(tweet)\n",
    "#             y_train.append(classes[line[1].strip()])\n",
    "\n",
    "    pp = len(x_train)\n",
    "    x_test = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            #if line[0] != 'ID' and line[1] == t:\n",
    "            if line[0] != 'ID' and target not in line[1].strip():\n",
    "            #if line[0].strip() != 'ID': #Uncomment this line if testing on wholedataset and comment the line above.\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_test.append(tweet)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_train, y_train, x_test, y_test, sentence_maxlen,pp\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b64fc9a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target: Donald Trump\n"
     ]
    }
   ],
   "source": [
    "train_data_file = '/data/parush/stance_mohammed/new_train.txt'\n",
    "test_data_file = '/data/parush/stance_mohammed/new_test.txt'\n",
    "TARGETS = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion', 'Donald Trump']\n",
    "\n",
    "\n",
    "# train_data_file = '/data/parush/Data_MPCHI/train.txt'\n",
    "# test_data_file = '/data/parush/Data_MPCHI/test.txt'\n",
    "# TARGETS = ['Are E-Cigarettes safe?','Does MMR Vaccine lead to autism in children?',\n",
    "#       'Does Sunlight exposure lead to skin cancer?','Does Vitamin C prevent common cold?',\n",
    "#       'Should women take HRT post-menopause?']\n",
    "\n",
    "\n",
    "target = TARGETS[5]\n",
    "print(\"Target:\", target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "13696437",
   "metadata": {},
   "outputs": [],
   "source": [
    "# In[273]:\n",
    "\n",
    "\n",
    "train_texts, train_labels, test_texts, test_labels, sen_maxlen, pp = train_and_test(train_data_file, test_data_file, target)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b377ce52",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame([])\n",
    "df2['TITLE'] = train_texts\n",
    "df2['target_list'] = train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e1732066-f056-4de7-88b6-de62fbba50c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.DataFrame([])\n",
    "df_test['TITLE'] = test_texts\n",
    "df_test['target_list'] = test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ed968052-5a3f-4d54-8708-fa3ca0baff60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TITLE</th>\n",
       "      <th>target_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@tedcruz And, #HandOverTheServer she wiped cle...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hillary is our best choice if we truly want to...</td>\n",
       "      <td>[1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@TheView I think our country is ready for a fe...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I just gave an unhealthy amount of my hard-ear...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@PortiaABoulger Thank you for adding me to you...</td>\n",
       "      <td>[0, 0, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2909</th>\n",
       "      <td>There's a law protecting unborn eagles, but no...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2910</th>\n",
       "      <td>I am 1 in 3... I have had an abortion #Abortio...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2911</th>\n",
       "      <td>How dare you say my sexual preference is a cho...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2912</th>\n",
       "      <td>Equal rights for those 'born that way', no rig...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2913</th>\n",
       "      <td>#POTUS seals his legacy w/ 1/2 doz wins. The #...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2914 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  TITLE target_list\n",
       "0     @tedcruz And, #HandOverTheServer she wiped cle...   [0, 1, 0]\n",
       "1     Hillary is our best choice if we truly want to...   [1, 0, 0]\n",
       "2     @TheView I think our country is ready for a fe...   [0, 1, 0]\n",
       "3     I just gave an unhealthy amount of my hard-ear...   [0, 1, 0]\n",
       "4     @PortiaABoulger Thank you for adding me to you...   [0, 0, 1]\n",
       "...                                                 ...         ...\n",
       "2909  There's a law protecting unborn eagles, but no...   [0, 1, 0]\n",
       "2910  I am 1 in 3... I have had an abortion #Abortio...   [0, 1, 0]\n",
       "2911  How dare you say my sexual preference is a cho...   [0, 1, 0]\n",
       "2912  Equal rights for those 'born that way', no rig...   [0, 1, 0]\n",
       "2913  #POTUS seals his legacy w/ 1/2 doz wins. The #...   [0, 1, 0]\n",
       "\n",
       "[2914 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6387baeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TITLE</th>\n",
       "      <th>target_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>He who exalts himself shall      be humbled; a...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @prayerbullets: I remove Nehushtan -previou...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@Brainman365 @heidtjj @BenjaminLives I have so...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#God is utterly powerless without Human interv...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@David_Cameron   Miracles of #Multiculturalism...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1244</th>\n",
       "      <td>@MetalheadMonty @tom_six I followed him before...</td>\n",
       "      <td>[0, 0, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1245</th>\n",
       "      <td>For he who avenges blood remembers, he does no...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1246</th>\n",
       "      <td>Life is sacred on all levels. Abortion does no...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247</th>\n",
       "      <td>\"@ravensymone U refer to \"\"WE\"\" which =\"\"YOU\"\"...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1248</th>\n",
       "      <td>Al Robertson's mom #DuckDynasty  chose life as...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1249 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  TITLE target_list\n",
       "0     He who exalts himself shall      be humbled; a...   [0, 1, 0]\n",
       "1     RT @prayerbullets: I remove Nehushtan -previou...   [0, 1, 0]\n",
       "2     @Brainman365 @heidtjj @BenjaminLives I have so...   [0, 1, 0]\n",
       "3     #God is utterly powerless without Human interv...   [0, 1, 0]\n",
       "4     @David_Cameron   Miracles of #Multiculturalism...   [0, 1, 0]\n",
       "...                                                 ...         ...\n",
       "1244  @MetalheadMonty @tom_six I followed him before...   [0, 0, 1]\n",
       "1245  For he who avenges blood remembers, he does no...   [0, 1, 0]\n",
       "1246  Life is sacred on all levels. Abortion does no...   [0, 1, 0]\n",
       "1247  \"@ravensymone U refer to \"\"WE\"\" which =\"\"YOU\"\"...   [0, 1, 0]\n",
       "1248  Al Robertson's mom #DuckDynasty  chose life as...   [0, 1, 0]\n",
       "\n",
       "[1249 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d415c695",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 128\n",
    "TRAIN_BATCH_SIZE = 32\n",
    "VALID_BATCH_SIZE = 32\n",
    "EPOCHS = 50\n",
    "LEARNING_RATE = 1e-05\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2572333c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8010ac26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c312e63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.title = dataframe['TITLE']\n",
    "        self.targets = self.data.target_list\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.title)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        title = str(self.title[index])\n",
    "        title = \" \".join(title.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            title,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            padding='max_length',\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c8835f33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL Dataset: (2914, 2)\n",
      "TRAIN Dataset: (2331, 2)\n",
      "TEST Dataset: (583, 2)\n"
     ]
    }
   ],
   "source": [
    "train_size = 0.8\n",
    "train_dataset = df2.sample(frac=train_size,random_state=200)\n",
    "valid_dataset = df2.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "\n",
    "print(\"FULL Dataset: {}\".format(df2.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(valid_dataset.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e53f2f99-2038-4860-ad33-6292e1b12c6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "validation_set = CustomDataset(valid_dataset,  tokenizer, MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d7b2fda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "64a6bd3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': False,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "validation_loader = DataLoader(validation_set, **test_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0a2c231c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BERTClass(\n",
       "  (l1): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (l2): Dropout(p=0.3, inplace=False)\n",
       "  (l3): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.l2 = torch.nn.Dropout(0.3)\n",
    "        self.l3 = torch.nn.Linear(768, 3)\n",
    "    \n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids,return_dict=False)\n",
    "        output_2 = self.l2(output_1)\n",
    "        output = self.l3(output_2)\n",
    "        return output\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "79ae560b-82f7-4632-a2d0-c0ff037b2655",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n",
    "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "31ee07de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ckp(checkpoint_fpath, model, optimizer):\n",
    "    \"\"\"\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    model: model that we want to load checkpoint parameters into       \n",
    "    optimizer: optimizer we defined in previous training\n",
    "    \"\"\"\n",
    "    # load check point\n",
    "    checkpoint = torch.load(checkpoint_fpath)\n",
    "    # initialize state_dict from checkpoint to model\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "    # initialize optimizer from checkpoint to optimizer\n",
    "    optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "    # initialize valid_loss_min from checkpoint to valid_loss_min\n",
    "    valid_loss_min = checkpoint['valid_loss_min']\n",
    "    # return model, optimizer, epoch value, min validation loss \n",
    "    return model, optimizer, checkpoint['epoch'], valid_loss_min.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f2b6d1a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil, sys   \n",
    "def save_ckp(state, is_best, checkpoint_path, best_model_path):\n",
    "    \"\"\"\n",
    "    state: checkpoint we want to save\n",
    "    is_best: is this the best checkpoint; min validation loss\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    best_model_path: path to save best model\n",
    "    \"\"\"\n",
    "    f_path = checkpoint_path\n",
    "    # save checkpoint data to the path given, checkpoint_path\n",
    "    torch.save(state, f_path)\n",
    "    # if it is a best model, min validation loss\n",
    "    if is_best:\n",
    "        best_fpath = best_model_path\n",
    "        # copy that checkpoint file to best path given, best_model_path\n",
    "        shutil.copyfile(f_path, best_fpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6f1b0726",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(start_epochs,  n_epochs, valid_loss_min_input, \n",
    "                training_loader, validation_loader, model, \n",
    "                optimizer, checkpoint_path, best_model_path):\n",
    "   \n",
    "  # initialize tracker for minimum validation loss\n",
    "    valid_loss_min = valid_loss_min_input \n",
    "    for epoch in range(start_epochs, n_epochs+1):\n",
    "        train_loss = 0\n",
    "        valid_loss = 0\n",
    "\n",
    "        model.train()\n",
    "        print('############# Epoch {}: Training Start   #############'.format(epoch))\n",
    "        for batch_idx, data in enumerate(training_loader):\n",
    "            #print('yyy epoch', batch_idx)\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss = loss_fn(outputs, targets)\n",
    "            #if batch_idx%5000==0:\n",
    "             #   print(f'Epoch: {epoch}, Training Loss:  {loss.item()}')\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            #print('before loss data in training', loss.item(), train_loss)\n",
    "            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.item() - train_loss))\n",
    "            #print('after loss data in training', loss.item(), train_loss)\n",
    "\n",
    "        print('############# Epoch {}: Training End     #############'.format(epoch))\n",
    "\n",
    "        print('############# Epoch {}: Validation Start   #############'.format(epoch))\n",
    "        ######################    \n",
    "        # validate the model #\n",
    "        ######################\n",
    "\n",
    "        model.eval()\n",
    "        val_targets, val_outputs = [] , []\n",
    "        with torch.no_grad():\n",
    "            for batch_idx, data in enumerate(validation_loader, 0):\n",
    "                ids = data['ids'].to(device, dtype = torch.long)\n",
    "                mask = data['mask'].to(device, dtype = torch.long)\n",
    "                token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "                targets = data['targets'].to(device, dtype = torch.float)\n",
    "                outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "                loss = loss_fn(outputs, targets)\n",
    "                valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))                \n",
    "                \n",
    "                val_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "                val_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "            print('############# Epoch {}: Validation End     #############'.format(epoch))\n",
    "            # calculate average losses\n",
    "            #print('before cal avg train loss', train_loss)\n",
    "            train_loss = train_loss/len(training_loader)\n",
    "            valid_loss = valid_loss/len(validation_loader)\n",
    "            # print training/validation statistics \n",
    "            print('Epoch: {} \\tAvgerage Training Loss: {:.6f} \\tAverage Validation Loss: {:.6f}'.format(\n",
    "                epoch, \n",
    "                train_loss,\n",
    "                valid_loss\n",
    "                ))\n",
    "            # create checkpoint variable and add important data\n",
    "            checkpoint = {\n",
    "                'epoch': epoch + 1,\n",
    "                'valid_loss_min': valid_loss,\n",
    "                'state_dict': model.state_dict(),\n",
    "                'optimizer': optimizer.state_dict()\n",
    "            }\n",
    "\n",
    "            # save checkpoint\n",
    "            save_ckp(checkpoint, False, checkpoint_path, best_model_path)\n",
    "            ## TODO: save the model if validation loss has decreased\n",
    "            if valid_loss <= valid_loss_min:\n",
    "                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))\n",
    "                # save checkpoint as best model\n",
    "                save_ckp(checkpoint, True, checkpoint_path, best_model_path)\n",
    "                valid_loss_min = valid_loss\n",
    "\n",
    "        print('############# Epoch {}  Done   #############\\n'.format(epoch))\n",
    "    \n",
    "    return model,val_targets, val_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e49dcc84-7f6f-464d-a7b0-653877ff4053",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "id": "ba37a6c6-a36a-413e-9605-f77edf85ac39",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "85494eb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############# Epoch 1: Training Start   #############\n",
      "############# Epoch 1: Training End     #############\n",
      "############# Epoch 1: Validation Start   #############\n",
      "############# Epoch 1: Validation End     #############\n",
      "Epoch: 1 \tAvgerage Training Loss: 0.008476 \tAverage Validation Loss: 0.031578\n",
      "Validation loss decreased (inf --> 0.031578).  Saving model ...\n",
      "############# Epoch 1  Done   #############\n",
      "\n",
      "############# Epoch 2: Training Start   #############\n",
      "############# Epoch 2: Training End     #############\n",
      "############# Epoch 2: Validation Start   #############\n",
      "############# Epoch 2: Validation End     #############\n",
      "Epoch: 2 \tAvgerage Training Loss: 0.007814 \tAverage Validation Loss: 0.029465\n",
      "Validation loss decreased (0.031578 --> 0.029465).  Saving model ...\n",
      "############# Epoch 2  Done   #############\n",
      "\n",
      "############# Epoch 3: Training Start   #############\n",
      "############# Epoch 3: Training End     #############\n",
      "############# Epoch 3: Validation Start   #############\n",
      "############# Epoch 3: Validation End     #############\n",
      "Epoch: 3 \tAvgerage Training Loss: 0.006916 \tAverage Validation Loss: 0.026801\n",
      "Validation loss decreased (0.029465 --> 0.026801).  Saving model ...\n",
      "############# Epoch 3  Done   #############\n",
      "\n",
      "############# Epoch 4: Training Start   #############\n",
      "############# Epoch 4: Training End     #############\n",
      "############# Epoch 4: Validation Start   #############\n",
      "############# Epoch 4: Validation End     #############\n",
      "Epoch: 4 \tAvgerage Training Loss: 0.005679 \tAverage Validation Loss: 0.027397\n",
      "############# Epoch 4  Done   #############\n",
      "\n",
      "############# Epoch 5: Training Start   #############\n",
      "############# Epoch 5: Training End     #############\n",
      "############# Epoch 5: Validation Start   #############\n",
      "############# Epoch 5: Validation End     #############\n",
      "Epoch: 5 \tAvgerage Training Loss: 0.004446 \tAverage Validation Loss: 0.026036\n",
      "Validation loss decreased (0.026801 --> 0.026036).  Saving model ...\n",
      "############# Epoch 5  Done   #############\n",
      "\n",
      "############# Epoch 6: Training Start   #############\n",
      "############# Epoch 6: Training End     #############\n",
      "############# Epoch 6: Validation Start   #############\n",
      "############# Epoch 6: Validation End     #############\n",
      "Epoch: 6 \tAvgerage Training Loss: 0.003249 \tAverage Validation Loss: 0.027464\n",
      "############# Epoch 6  Done   #############\n",
      "\n",
      "############# Epoch 7: Training Start   #############\n",
      "############# Epoch 7: Training End     #############\n",
      "############# Epoch 7: Validation Start   #############\n",
      "############# Epoch 7: Validation End     #############\n",
      "Epoch: 7 \tAvgerage Training Loss: 0.002299 \tAverage Validation Loss: 0.030525\n",
      "############# Epoch 7  Done   #############\n",
      "\n",
      "############# Epoch 8: Training Start   #############\n",
      "############# Epoch 8: Training End     #############\n",
      "############# Epoch 8: Validation Start   #############\n",
      "############# Epoch 8: Validation End     #############\n",
      "Epoch: 8 \tAvgerage Training Loss: 0.001501 \tAverage Validation Loss: 0.030317\n",
      "############# Epoch 8  Done   #############\n",
      "\n",
      "############# Epoch 9: Training Start   #############\n",
      "############# Epoch 9: Training End     #############\n",
      "############# Epoch 9: Validation Start   #############\n",
      "############# Epoch 9: Validation End     #############\n",
      "Epoch: 9 \tAvgerage Training Loss: 0.001025 \tAverage Validation Loss: 0.031739\n",
      "############# Epoch 9  Done   #############\n",
      "\n",
      "############# Epoch 10: Training Start   #############\n",
      "############# Epoch 10: Training End     #############\n",
      "############# Epoch 10: Validation Start   #############\n",
      "############# Epoch 10: Validation End     #############\n",
      "Epoch: 10 \tAvgerage Training Loss: 0.000659 \tAverage Validation Loss: 0.034797\n",
      "############# Epoch 10  Done   #############\n",
      "\n",
      "############# Epoch 11: Training Start   #############\n",
      "############# Epoch 11: Training End     #############\n",
      "############# Epoch 11: Validation Start   #############\n",
      "############# Epoch 11: Validation End     #############\n",
      "Epoch: 11 \tAvgerage Training Loss: 0.000431 \tAverage Validation Loss: 0.036987\n",
      "############# Epoch 11  Done   #############\n",
      "\n",
      "############# Epoch 12: Training Start   #############\n",
      "############# Epoch 12: Training End     #############\n",
      "############# Epoch 12: Validation Start   #############\n",
      "############# Epoch 12: Validation End     #############\n",
      "Epoch: 12 \tAvgerage Training Loss: 0.000318 \tAverage Validation Loss: 0.040311\n",
      "############# Epoch 12  Done   #############\n",
      "\n",
      "############# Epoch 13: Training Start   #############\n",
      "############# Epoch 13: Training End     #############\n",
      "############# Epoch 13: Validation Start   #############\n",
      "############# Epoch 13: Validation End     #############\n",
      "Epoch: 13 \tAvgerage Training Loss: 0.000253 \tAverage Validation Loss: 0.041637\n",
      "############# Epoch 13  Done   #############\n",
      "\n",
      "############# Epoch 14: Training Start   #############\n",
      "############# Epoch 14: Training End     #############\n",
      "############# Epoch 14: Validation Start   #############\n",
      "############# Epoch 14: Validation End     #############\n",
      "Epoch: 14 \tAvgerage Training Loss: 0.000218 \tAverage Validation Loss: 0.042717\n",
      "############# Epoch 14  Done   #############\n",
      "\n",
      "############# Epoch 15: Training Start   #############\n",
      "############# Epoch 15: Training End     #############\n",
      "############# Epoch 15: Validation Start   #############\n",
      "############# Epoch 15: Validation End     #############\n",
      "Epoch: 15 \tAvgerage Training Loss: 0.000169 \tAverage Validation Loss: 0.046076\n",
      "############# Epoch 15  Done   #############\n",
      "\n",
      "############# Epoch 16: Training Start   #############\n",
      "############# Epoch 16: Training End     #############\n",
      "############# Epoch 16: Validation Start   #############\n",
      "############# Epoch 16: Validation End     #############\n",
      "Epoch: 16 \tAvgerage Training Loss: 0.000142 \tAverage Validation Loss: 0.046784\n",
      "############# Epoch 16  Done   #############\n",
      "\n",
      "############# Epoch 17: Training Start   #############\n",
      "############# Epoch 17: Training End     #############\n",
      "############# Epoch 17: Validation Start   #############\n",
      "############# Epoch 17: Validation End     #############\n",
      "Epoch: 17 \tAvgerage Training Loss: 0.000270 \tAverage Validation Loss: 0.050126\n",
      "############# Epoch 17  Done   #############\n",
      "\n",
      "############# Epoch 18: Training Start   #############\n",
      "############# Epoch 18: Training End     #############\n",
      "############# Epoch 18: Validation Start   #############\n",
      "############# Epoch 18: Validation End     #############\n",
      "Epoch: 18 \tAvgerage Training Loss: 0.000411 \tAverage Validation Loss: 0.044376\n",
      "############# Epoch 18  Done   #############\n",
      "\n",
      "############# Epoch 19: Training Start   #############\n",
      "############# Epoch 19: Training End     #############\n",
      "############# Epoch 19: Validation Start   #############\n",
      "############# Epoch 19: Validation End     #############\n",
      "Epoch: 19 \tAvgerage Training Loss: 0.000167 \tAverage Validation Loss: 0.044954\n",
      "############# Epoch 19  Done   #############\n",
      "\n",
      "############# Epoch 20: Training Start   #############\n",
      "############# Epoch 20: Training End     #############\n",
      "############# Epoch 20: Validation Start   #############\n",
      "############# Epoch 20: Validation End     #############\n",
      "Epoch: 20 \tAvgerage Training Loss: 0.000124 \tAverage Validation Loss: 0.046709\n",
      "############# Epoch 20  Done   #############\n",
      "\n",
      "############# Epoch 21: Training Start   #############\n",
      "############# Epoch 21: Training End     #############\n",
      "############# Epoch 21: Validation Start   #############\n",
      "############# Epoch 21: Validation End     #############\n",
      "Epoch: 21 \tAvgerage Training Loss: 0.000101 \tAverage Validation Loss: 0.051833\n",
      "############# Epoch 21  Done   #############\n",
      "\n",
      "############# Epoch 22: Training Start   #############\n",
      "############# Epoch 22: Training End     #############\n",
      "############# Epoch 22: Validation Start   #############\n",
      "############# Epoch 22: Validation End     #############\n",
      "Epoch: 22 \tAvgerage Training Loss: 0.000164 \tAverage Validation Loss: 0.051572\n",
      "############# Epoch 22  Done   #############\n",
      "\n",
      "############# Epoch 23: Training Start   #############\n",
      "############# Epoch 23: Training End     #############\n",
      "############# Epoch 23: Validation Start   #############\n",
      "############# Epoch 23: Validation End     #############\n",
      "Epoch: 23 \tAvgerage Training Loss: 0.000180 \tAverage Validation Loss: 0.050504\n",
      "############# Epoch 23  Done   #############\n",
      "\n",
      "############# Epoch 24: Training Start   #############\n",
      "############# Epoch 24: Training End     #############\n",
      "############# Epoch 24: Validation Start   #############\n",
      "############# Epoch 24: Validation End     #############\n",
      "Epoch: 24 \tAvgerage Training Loss: 0.000119 \tAverage Validation Loss: 0.055286\n",
      "############# Epoch 24  Done   #############\n",
      "\n",
      "############# Epoch 25: Training Start   #############\n",
      "############# Epoch 25: Training End     #############\n",
      "############# Epoch 25: Validation Start   #############\n",
      "############# Epoch 25: Validation End     #############\n",
      "Epoch: 25 \tAvgerage Training Loss: 0.000138 \tAverage Validation Loss: 0.053318\n",
      "############# Epoch 25  Done   #############\n",
      "\n",
      "############# Epoch 26: Training Start   #############\n",
      "############# Epoch 26: Training End     #############\n",
      "############# Epoch 26: Validation Start   #############\n",
      "############# Epoch 26: Validation End     #############\n",
      "Epoch: 26 \tAvgerage Training Loss: 0.000123 \tAverage Validation Loss: 0.052631\n",
      "############# Epoch 26  Done   #############\n",
      "\n",
      "############# Epoch 27: Training Start   #############\n",
      "############# Epoch 27: Training End     #############\n",
      "############# Epoch 27: Validation Start   #############\n",
      "############# Epoch 27: Validation End     #############\n",
      "Epoch: 27 \tAvgerage Training Loss: 0.000100 \tAverage Validation Loss: 0.053279\n",
      "############# Epoch 27  Done   #############\n",
      "\n",
      "############# Epoch 28: Training Start   #############\n",
      "############# Epoch 28: Training End     #############\n",
      "############# Epoch 28: Validation Start   #############\n",
      "############# Epoch 28: Validation End     #############\n",
      "Epoch: 28 \tAvgerage Training Loss: 0.000084 \tAverage Validation Loss: 0.049366\n",
      "############# Epoch 28  Done   #############\n",
      "\n",
      "############# Epoch 29: Training Start   #############\n",
      "############# Epoch 29: Training End     #############\n",
      "############# Epoch 29: Validation Start   #############\n",
      "############# Epoch 29: Validation End     #############\n",
      "Epoch: 29 \tAvgerage Training Loss: 0.000087 \tAverage Validation Loss: 0.054992\n",
      "############# Epoch 29  Done   #############\n",
      "\n",
      "############# Epoch 30: Training Start   #############\n",
      "############# Epoch 30: Training End     #############\n",
      "############# Epoch 30: Validation Start   #############\n",
      "############# Epoch 30: Validation End     #############\n",
      "Epoch: 30 \tAvgerage Training Loss: 0.000067 \tAverage Validation Loss: 0.052766\n",
      "############# Epoch 30  Done   #############\n",
      "\n",
      "############# Epoch 31: Training Start   #############\n",
      "############# Epoch 31: Training End     #############\n",
      "############# Epoch 31: Validation Start   #############\n",
      "############# Epoch 31: Validation End     #############\n",
      "Epoch: 31 \tAvgerage Training Loss: 0.000043 \tAverage Validation Loss: 0.053639\n",
      "############# Epoch 31  Done   #############\n",
      "\n",
      "############# Epoch 32: Training Start   #############\n",
      "############# Epoch 32: Training End     #############\n",
      "############# Epoch 32: Validation Start   #############\n",
      "############# Epoch 32: Validation End     #############\n",
      "Epoch: 32 \tAvgerage Training Loss: 0.000041 \tAverage Validation Loss: 0.053644\n",
      "############# Epoch 32  Done   #############\n",
      "\n",
      "############# Epoch 33: Training Start   #############\n",
      "############# Epoch 33: Training End     #############\n",
      "############# Epoch 33: Validation Start   #############\n",
      "############# Epoch 33: Validation End     #############\n",
      "Epoch: 33 \tAvgerage Training Loss: 0.000037 \tAverage Validation Loss: 0.054224\n",
      "############# Epoch 33  Done   #############\n",
      "\n",
      "############# Epoch 34: Training Start   #############\n",
      "############# Epoch 34: Training End     #############\n",
      "############# Epoch 34: Validation Start   #############\n",
      "############# Epoch 34: Validation End     #############\n",
      "Epoch: 34 \tAvgerage Training Loss: 0.000034 \tAverage Validation Loss: 0.055026\n",
      "############# Epoch 34  Done   #############\n",
      "\n",
      "############# Epoch 35: Training Start   #############\n",
      "############# Epoch 35: Training End     #############\n",
      "############# Epoch 35: Validation Start   #############\n",
      "############# Epoch 35: Validation End     #############\n",
      "Epoch: 35 \tAvgerage Training Loss: 0.000031 \tAverage Validation Loss: 0.055375\n",
      "############# Epoch 35  Done   #############\n",
      "\n",
      "############# Epoch 36: Training Start   #############\n",
      "############# Epoch 36: Training End     #############\n",
      "############# Epoch 36: Validation Start   #############\n",
      "############# Epoch 36: Validation End     #############\n",
      "Epoch: 36 \tAvgerage Training Loss: 0.000147 \tAverage Validation Loss: 0.052942\n",
      "############# Epoch 36  Done   #############\n",
      "\n",
      "############# Epoch 37: Training Start   #############\n",
      "############# Epoch 37: Training End     #############\n",
      "############# Epoch 37: Validation Start   #############\n",
      "############# Epoch 37: Validation End     #############\n",
      "Epoch: 37 \tAvgerage Training Loss: 0.000072 \tAverage Validation Loss: 0.054894\n",
      "############# Epoch 37  Done   #############\n",
      "\n",
      "############# Epoch 38: Training Start   #############\n",
      "############# Epoch 38: Training End     #############\n",
      "############# Epoch 38: Validation Start   #############\n",
      "############# Epoch 38: Validation End     #############\n",
      "Epoch: 38 \tAvgerage Training Loss: 0.000190 \tAverage Validation Loss: 0.053327\n",
      "############# Epoch 38  Done   #############\n",
      "\n",
      "############# Epoch 39: Training Start   #############\n",
      "############# Epoch 39: Training End     #############\n",
      "############# Epoch 39: Validation Start   #############\n",
      "############# Epoch 39: Validation End     #############\n",
      "Epoch: 39 \tAvgerage Training Loss: 0.000059 \tAverage Validation Loss: 0.053846\n",
      "############# Epoch 39  Done   #############\n",
      "\n",
      "############# Epoch 40: Training Start   #############\n",
      "############# Epoch 40: Training End     #############\n",
      "############# Epoch 40: Validation Start   #############\n",
      "############# Epoch 40: Validation End     #############\n",
      "Epoch: 40 \tAvgerage Training Loss: 0.000034 \tAverage Validation Loss: 0.056379\n",
      "############# Epoch 40  Done   #############\n",
      "\n",
      "############# Epoch 41: Training Start   #############\n",
      "############# Epoch 41: Training End     #############\n",
      "############# Epoch 41: Validation Start   #############\n",
      "############# Epoch 41: Validation End     #############\n",
      "Epoch: 41 \tAvgerage Training Loss: 0.000031 \tAverage Validation Loss: 0.058556\n",
      "############# Epoch 41  Done   #############\n",
      "\n",
      "############# Epoch 42: Training Start   #############\n",
      "############# Epoch 42: Training End     #############\n",
      "############# Epoch 42: Validation Start   #############\n",
      "############# Epoch 42: Validation End     #############\n",
      "Epoch: 42 \tAvgerage Training Loss: 0.000064 \tAverage Validation Loss: 0.062270\n",
      "############# Epoch 42  Done   #############\n",
      "\n",
      "############# Epoch 43: Training Start   #############\n",
      "############# Epoch 43: Training End     #############\n",
      "############# Epoch 43: Validation Start   #############\n",
      "############# Epoch 43: Validation End     #############\n",
      "Epoch: 43 \tAvgerage Training Loss: 0.000121 \tAverage Validation Loss: 0.055837\n",
      "############# Epoch 43  Done   #############\n",
      "\n",
      "############# Epoch 44: Training Start   #############\n",
      "############# Epoch 44: Training End     #############\n",
      "############# Epoch 44: Validation Start   #############\n",
      "############# Epoch 44: Validation End     #############\n",
      "Epoch: 44 \tAvgerage Training Loss: 0.000224 \tAverage Validation Loss: 0.068515\n",
      "############# Epoch 44  Done   #############\n",
      "\n",
      "############# Epoch 45: Training Start   #############\n",
      "############# Epoch 45: Training End     #############\n",
      "############# Epoch 45: Validation Start   #############\n",
      "############# Epoch 45: Validation End     #############\n",
      "Epoch: 45 \tAvgerage Training Loss: 0.000107 \tAverage Validation Loss: 0.055233\n",
      "############# Epoch 45  Done   #############\n",
      "\n",
      "############# Epoch 46: Training Start   #############\n",
      "############# Epoch 46: Training End     #############\n",
      "############# Epoch 46: Validation Start   #############\n",
      "############# Epoch 46: Validation End     #############\n",
      "Epoch: 46 \tAvgerage Training Loss: 0.000042 \tAverage Validation Loss: 0.055007\n",
      "############# Epoch 46  Done   #############\n",
      "\n",
      "############# Epoch 47: Training Start   #############\n",
      "############# Epoch 47: Training End     #############\n",
      "############# Epoch 47: Validation Start   #############\n",
      "############# Epoch 47: Validation End     #############\n",
      "Epoch: 47 \tAvgerage Training Loss: 0.000023 \tAverage Validation Loss: 0.057160\n",
      "############# Epoch 47  Done   #############\n",
      "\n",
      "############# Epoch 48: Training Start   #############\n",
      "############# Epoch 48: Training End     #############\n",
      "############# Epoch 48: Validation Start   #############\n",
      "############# Epoch 48: Validation End     #############\n",
      "Epoch: 48 \tAvgerage Training Loss: 0.000021 \tAverage Validation Loss: 0.058142\n",
      "############# Epoch 48  Done   #############\n",
      "\n",
      "############# Epoch 49: Training Start   #############\n",
      "############# Epoch 49: Training End     #############\n",
      "############# Epoch 49: Validation Start   #############\n",
      "############# Epoch 49: Validation End     #############\n",
      "Epoch: 49 \tAvgerage Training Loss: 0.000021 \tAverage Validation Loss: 0.058933\n",
      "############# Epoch 49  Done   #############\n",
      "\n",
      "############# Epoch 50: Training Start   #############\n",
      "############# Epoch 50: Training End     #############\n",
      "############# Epoch 50: Validation Start   #############\n",
      "############# Epoch 50: Validation End     #############\n",
      "Epoch: 50 \tAvgerage Training Loss: 0.000133 \tAverage Validation Loss: 0.067276\n",
      "############# Epoch 50  Done   #############\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# checkpoint_path = '/home/p/parush/comparison/current_checkpoint_'+target+'.pt'\n",
    "# best_model = '/home/p/parush/comparison/best_model_trained_'+ target+'.pt'\n",
    "# trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "#                       optimizer,checkpoint_path,best_model)\n",
    "\n",
    "checkpoint_path = '/home/p/parush/comparison/checkpoint_overall_mohammed'+'.pt'\n",
    "best_model = '/home/p/parush/comparison/best_model_overall_mohammed'+'.pt'\n",
    "trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "                      optimizer,checkpoint_path,best_model)\n",
    "\n",
    "\n",
    "# checkpoint_path = '/home/p/parush/comparison/checkpoint_overall_mpchi'+'.pt'\n",
    "# best_model = '/home/p/parush/comparison/best_model_overall_mpchi'+'.pt'\n",
    "# trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "#                       optimizer,checkpoint_path,best_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d6d1dd18",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_preds = (np.array(val_outputs) > 0.5).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "89a4e22e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7048    0.4654    0.5606       159\n",
      "           1     0.6235    0.8249    0.7102       257\n",
      "\n",
      "   micro avg     0.6427    0.6875    0.6643       416\n",
      "   macro avg     0.6641    0.6452    0.6354       416\n",
      "weighted avg     0.6546    0.6875    0.6530       416\n",
      " samples avg     0.4880    0.4906    0.4889       416\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(val_targets, val_preds,labels=[0,1],digits = 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43281ffd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff9e2147",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a02122ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "783c0a49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ac80ce4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b80290b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8b08efe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43305844",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1477b03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c78c285",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44db7497",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f10ef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "765e4208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ace94e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4ba7a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139bb283",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ef4fb45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f63d12a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f28ed6a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b26195f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e44f3df",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cf5cbdf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40197fd6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11a2d7a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74266b83",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f856c1d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93224e89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c259bc10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81a03ace",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f220bd05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cdb22a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b27ef23",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41e3b589",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee8bed59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd5cb30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3323d73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "407914f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1577a3be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a6b1724",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56c3f021",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6150724b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78167564",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb61afe7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "791ce674",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3164884d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7fba559",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}