PreTrainedZeroShot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from transformers import AutoTokenizer,AutoModel, BertTokenizer, BertForSequenceClassification\n",
    "from datasets import load_dataset, Dataset\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\kapsu\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb = load_dataset('imdb',split='test')\n",
    "imdb.set_format(\"torch\",columns=['text','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stop_words(example):\n",
    "    wrds = example['text'].split(' ')\n",
    "    flts = [w for w in wrds if w.lower() not in stop_words]\n",
    "    str = \"\"\n",
    "    \n",
    "    for f in flts:\n",
    "        str+= f+\" \"\n",
    "    \n",
    "    new_one = {'text':str[:-1],'label':example['label']}\n",
    "    return new_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb = imdb.train_test_split(0.6,stratify_by_column='label')['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 7500/7500 [00:01<00:00, 6562.85 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 7500\n",
       "})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb.map(remove_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"textattack/bert-base-uncased-yelp-polarity\")\n",
    "model = BertForSequenceClassification.from_pretrained(\"textattack/bert-base-uncased-yelp-polarity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg loss after 0: 0.03904830291867256\n",
      "avg loss after 1: 0.035252440720796585\n",
      "avg loss after 2: 0.2532491981983185\n",
      "avg loss after 3: 0.2171950489282608\n",
      "avg loss after 4: 0.25441059470176697\n",
      "avg loss after 5: 0.21966715157032013\n",
      "avg loss after 6: 0.32000455260276794\n",
      "avg loss after 7: 0.3235739469528198\n",
      "avg loss after 8: 0.37583112716674805\n",
      "avg loss after 9: 0.45144858956336975\n",
      "avg loss after 10: 0.4319971799850464\n",
      "avg loss after 11: 0.4373348653316498\n",
      "avg loss after 12: 0.40742021799087524\n",
      "avg loss after 13: 0.3796795904636383\n",
      "avg loss after 14: 0.397330105304718\n",
      "avg loss after 15: 0.49873602390289307\n",
      "avg loss after 16: 0.5364951491355896\n",
      "avg loss after 17: 0.5574448108673096\n",
      "avg loss after 18: 0.5290433764457703\n",
      "avg loss after 19: 0.5425915718078613\n",
      "avg loss after 20: 0.5584104061126709\n",
      "avg loss after 21: 0.550232470035553\n",
      "avg loss after 22: 0.5502983927726746\n",
      "avg loss after 23: 0.5306203961372375\n",
      "avg loss after 24: 0.5349090099334717\n",
      "avg loss after 25: 0.5355023741722107\n",
      "avg loss after 26: 0.5316321849822998\n",
      "avg loss after 27: 0.5341110825538635\n",
      "avg loss after 28: 0.5295612215995789\n",
      "avg loss after 29: 0.516499400138855\n",
      "avg loss after 30: 0.516453206539154\n",
      "avg loss after 31: 0.513046383857727\n",
      "avg loss after 32: 0.4988677203655243\n",
      "avg loss after 33: 0.4907647967338562\n",
      "avg loss after 34: 0.47920146584510803\n",
      "avg loss after 35: 0.46831178665161133\n",
      "avg loss after 36: 0.4709298312664032\n",
      "avg loss after 37: 0.46095362305641174\n",
      "avg loss after 38: 0.4496613144874573\n",
      "avg loss after 39: 0.43962448835372925\n",
      "avg loss after 40: 0.44093620777130127\n",
      "avg loss after 41: 0.45229607820510864\n",
      "avg loss after 42: 0.45005515217781067\n",
      "avg loss after 43: 0.440298467874527\n",
      "avg loss after 44: 0.45199504494667053\n",
      "avg loss after 45: 0.4435326159000397\n",
      "avg loss after 46: 0.4346795678138733\n",
      "avg loss after 47: 0.4335048198699951\n",
      "avg loss after 48: 0.4353128969669342\n",
      "avg loss after 49: 0.43862202763557434\n",
      "avg loss after 50: 0.44040563702583313\n",
      "avg loss after 51: 0.4393763542175293\n",
      "avg loss after 52: 0.4361225366592407\n",
      "avg loss after 53: 0.42859989404678345\n",
      "avg loss after 54: 0.4449404776096344\n",
      "avg loss after 55: 0.44649407267570496\n",
      "avg loss after 56: 0.4425639808177948\n",
      "avg loss after 57: 0.44403553009033203\n",
      "avg loss after 58: 0.4501040279865265\n",
      "avg loss after 59: 0.44444894790649414\n",
      "avg loss after 60: 0.4451727271080017\n",
      "avg loss after 61: 0.4391346871852875\n",
      "avg loss after 62: 0.43678802251815796\n",
      "avg loss after 63: 0.43086013197898865\n",
      "avg loss after 64: 0.4288921654224396\n",
      "avg loss after 65: 0.42281991243362427\n",
      "avg loss after 66: 0.42021873593330383\n",
      "avg loss after 67: 0.42696061730384827\n",
      "avg loss after 68: 0.42709511518478394\n",
      "avg loss after 69: 0.4301939010620117\n",
      "avg loss after 70: 0.4248807430267334\n",
      "avg loss after 71: 0.4191664159297943\n",
      "avg loss after 72: 0.41363969445228577\n",
      "avg loss after 73: 0.40878722071647644\n",
      "avg loss after 74: 0.41086485981941223\n",
      "avg loss after 75: 0.4125576317310333\n",
      "avg loss after 76: 0.4147503674030304\n",
      "avg loss after 77: 0.41972342133522034\n",
      "avg loss after 78: 0.4149380922317505\n",
      "avg loss after 79: 0.4099748730659485\n",
      "avg loss after 80: 0.40726611018180847\n",
      "avg loss after 81: 0.40377625823020935\n",
      "avg loss after 82: 0.4018314778804779\n",
      "avg loss after 83: 0.3972409963607788\n",
      "avg loss after 84: 0.40553557872772217\n",
      "avg loss after 85: 0.41054481267929077\n",
      "avg loss after 86: 0.409000962972641\n",
      "avg loss after 87: 0.41299402713775635\n",
      "avg loss after 88: 0.41807517409324646\n",
      "avg loss after 89: 0.4198046326637268\n",
      "avg loss after 90: 0.4163477420806885\n",
      "avg loss after 91: 0.4214727580547333\n",
      "avg loss after 92: 0.4171341061592102\n",
      "avg loss after 93: 0.41913315653800964\n",
      "avg loss after 94: 0.4151822030544281\n",
      "avg loss after 95: 0.4181485176086426\n",
      "avg loss after 96: 0.41772857308387756\n",
      "avg loss after 97: 0.41915425658226013\n",
      "avg loss after 98: 0.42652568221092224\n",
      "avg loss after 99: 0.42510440945625305\n",
      "avg loss after 100: 0.4236607253551483\n",
      "avg loss after 101: 0.42635002732276917\n",
      "avg loss after 102: 0.42597803473472595\n",
      "avg loss after 103: 0.42428430914878845\n",
      "avg loss after 104: 0.4209596514701843\n",
      "avg loss after 105: 0.42248761653900146\n",
      "avg loss after 106: 0.4218973219394684\n",
      "avg loss after 107: 0.42699959874153137\n",
      "avg loss after 108: 0.42705482244491577\n",
      "avg loss after 109: 0.43243294954299927\n",
      "avg loss after 110: 0.43172764778137207\n",
      "avg loss after 111: 0.4319137632846832\n",
      "avg loss after 112: 0.42994070053100586\n",
      "avg loss after 113: 0.4306279122829437\n",
      "avg loss after 114: 0.4318525195121765\n",
      "avg loss after 115: 0.43265920877456665\n",
      "avg loss after 116: 0.4297666549682617\n",
      "avg loss after 117: 0.4307304322719574\n",
      "avg loss after 118: 0.4326080083847046\n",
      "avg loss after 119: 0.4311978220939636\n",
      "avg loss after 120: 0.4279012680053711\n",
      "avg loss after 121: 0.4259604811668396\n",
      "avg loss after 122: 0.4230124056339264\n",
      "avg loss after 123: 0.4239945709705353\n",
      "avg loss after 124: 0.42746594548225403\n",
      "avg loss after 125: 0.4255567789077759\n",
      "avg loss after 126: 0.4243148863315582\n",
      "avg loss after 127: 0.42133769392967224\n",
      "avg loss after 128: 0.4182291030883789\n",
      "avg loss after 129: 0.41514045000076294\n",
      "avg loss after 130: 0.41380858421325684\n",
      "avg loss after 131: 0.412311315536499\n",
      "avg loss after 132: 0.40977612137794495\n",
      "avg loss after 133: 0.41029495000839233\n",
      "avg loss after 134: 0.40746524930000305\n",
      "avg loss after 135: 0.408672034740448\n",
      "avg loss after 136: 0.4096885919570923\n",
      "avg loss after 137: 0.41244444251060486\n",
      "avg loss after 138: 0.40972602367401123\n",
      "avg loss after 139: 0.41102874279022217\n",
      "avg loss after 140: 0.4082031548023224\n",
      "avg loss after 141: 0.4055173695087433\n",
      "avg loss after 142: 0.4064590632915497\n",
      "avg loss after 143: 0.40407446026802063\n",
      "avg loss after 144: 0.4013969600200653\n",
      "avg loss after 145: 0.3988582193851471\n",
      "avg loss after 146: 0.39649173617362976\n",
      "avg loss after 147: 0.3977776765823364\n",
      "avg loss after 148: 0.3953421413898468\n",
      "avg loss after 149: 0.396365761756897\n",
      "avg loss after 150: 0.3947184085845947\n",
      "avg loss after 151: 0.39820656180381775\n",
      "avg loss after 152: 0.3981136381626129\n",
      "avg loss after 153: 0.40190616250038147\n",
      "avg loss after 154: 0.40120813250541687\n",
      "avg loss after 155: 0.40304914116859436\n",
      "avg loss after 156: 0.40269193053245544\n",
      "avg loss after 157: 0.4007425308227539\n",
      "avg loss after 158: 0.39830172061920166\n",
      "avg loss after 159: 0.397676020860672\n",
      "avg loss after 160: 0.3989751636981964\n",
      "avg loss after 161: 0.4031605124473572\n",
      "avg loss after 162: 0.4007686376571655\n",
      "avg loss after 163: 0.39887866377830505\n",
      "avg loss after 164: 0.3990257978439331\n",
      "avg loss after 165: 0.39673691987991333\n",
      "avg loss after 166: 0.3949894607067108\n",
      "avg loss after 167: 0.39288026094436646\n",
      "avg loss after 168: 0.3936014175415039\n",
      "avg loss after 169: 0.3946213722229004\n",
      "avg loss after 170: 0.39242416620254517\n",
      "avg loss after 171: 0.39291438460350037\n",
      "avg loss after 172: 0.3914884924888611\n",
      "avg loss after 173: 0.3893106281757355\n",
      "avg loss after 174: 0.3888608515262604\n",
      "avg loss after 175: 0.3867962062358856\n",
      "avg loss after 176: 0.38484278321266174\n",
      "avg loss after 177: 0.3836269974708557\n",
      "avg loss after 178: 0.38182422518730164\n",
      "avg loss after 179: 0.38228416442871094\n",
      "avg loss after 180: 0.38036370277404785\n",
      "avg loss after 181: 0.3817845582962036\n",
      "avg loss after 182: 0.3813413679599762\n",
      "avg loss after 183: 0.38089272379875183\n",
      "avg loss after 184: 0.3844326436519623\n",
      "avg loss after 185: 0.3848837614059448\n",
      "avg loss after 186: 0.38322913646698\n",
      "net loss: 71.66384887695312\n",
      "avg loss: 0.07648222893476486\n"
     ]
    }
   ],
   "source": [
    "n = imdb.shape[0]\n",
    "BATCH_SIZE = 8\n",
    "\n",
    "shards = int(n / BATCH_SIZE)\n",
    "\n",
    "loss = 0\n",
    "\n",
    "loss_rec = {}\n",
    "\n",
    "all_preds = []\n",
    "all_acts = []\n",
    "\n",
    "for i in range(int(shards/5)):\n",
    "    \n",
    "    batch = imdb.shard(shards,i,contiguous=True)\n",
    "    \n",
    "    X = batch['text']\n",
    "    y = batch['label']\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        c1 = torch.tensor([0,1])\n",
    "        c2 = torch.tensor([1,0])\n",
    "    \n",
    "        temp = [c1 if y_t == 1 else c2 for y_t in y]\n",
    "    \n",
    "        ytt = torch.stack(temp).float()\n",
    "        tokenized = tokenizer(X,padding='max_length', max_length=512, truncation=True, return_tensors='pt')\n",
    "    \n",
    "        preds = model(**tokenized)\n",
    "        \n",
    "        all_preds.extend(np.argmax(preds.logits,axis = 1))\n",
    "        all_acts.extend(y)\n",
    "        \n",
    "        smx = torch.nn.Softmax(preds.logits)\n",
    "    \n",
    "        t_ls = loss_fn(preds.logits,ytt)\n",
    "        loss += t_ls\n",
    "    \n",
    "    loss_rec[i] = loss/(i+1)\n",
    "    print(f'avg loss after {i}: {loss/(i+1)}')\n",
    "    \n",
    "print(f'net loss: {loss}')\n",
    "print(f'avg loss: {loss/shards}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(loss_rec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(type(loss_rec[5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_metrics(p,a):\n",
    "    ps = precision_score(a,p)\n",
    "    rs = recall_score(a,p)\n",
    "    f1 = f1_score(a,p)\n",
    "    ac = accuracy_score(a,p)\n",
    "    \n",
    "    return ps,rs,f1,ac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1500\n"
     ]
    }
   ],
   "source": [
    "print(len(all_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1500\n"
     ]
    }
   ],
   "source": [
    "print(len(all_acts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.8810126582278481, 0.8721804511278195, 0.8765743073047859, 0.8693333333333333)\n"
     ]
    }
   ],
   "source": [
    "print(calculate_metrics(all_acts,all_preds))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlprep",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}