Charformer_Twitter_sarcasm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import  f1_score\n",
    "from charformer_pytorch import GBST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install charformer-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import torch \n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_conversation(text):\n",
    "    input_ids = torch.tensor([list(text.encode(\"utf-8\"))]) + 3 # add 3 for special tokens\n",
    "    return input_ids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MetNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.dim = 512\n",
    "        self.tokenizer = GBST(\n",
    "            num_tokens = 257,             # number of tokens, should be 256 for byte encoding (+ 1 special token for padding in this example)\n",
    "            dim = 512,                    # dimension of token and intra-block positional embedding\n",
    "            max_block_size = 4,           # maximum block size\n",
    "            downsample_factor = 4,        # the final downsample factor by which the sequence length will decrease by\n",
    "            score_consensus_attn = True   # whether to do the cheap score consensus (aka attention) as in eq. 5 in the paper\n",
    "        ).to(device)\n",
    "        self.positionEmbeddings = nn.Embedding(512 ,64)\n",
    "        self.transformerLayer = nn.TransformerEncoderLayer(576,8) \n",
    "        self.linear1 = nn.Linear(576,  64)\n",
    "        self.linear2 = nn.Linear(64,  1)\n",
    "        self.linear3 = nn.Linear(512,  16)\n",
    "        self.linear4 = nn.Linear(16,  1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.type(torch.LongTensor).to(device)\n",
    "        tokenize = self.tokenizer(x)\n",
    "        positions = (torch.arange(0,self.dim ).reshape(1,self.dim ) + torch.zeros(x.shape[0],self.dim )).to(device)\n",
    "        sentence = torch.cat((tokenize[0],self.positionEmbeddings(positions.long())),axis=2)\n",
    "        attended = self.transformerLayer(sentence)\n",
    "        linear1 = F.relu(self.linear1(attended))\n",
    "        linear2 = F.relu(self.linear2(linear1))\n",
    "        linear2 = linear2.view(-1,512) # reshaping the layer as the transformer outputs a 2d tensor (or 3d considering the batch size)\n",
    "        linear3 = F.relu(self.linear3(linear2))\n",
    "        out = torch.sigmoid(self.linear4(linear3))\n",
    "        return out\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MetNet().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_json_file = '/scratch/sm9669/sarcasm_detection_shared_task_twitter_training.jsonl'\n",
    "import json \n",
    "\n",
    "with open(path_to_json_file, 'r') as j:\n",
    "     json_data = [json.loads(line) for line in j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [00:00<00:00, 436025.53it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "accurate = 0\n",
    "tot_tweets = 0\n",
    "\n",
    "tweets=[]\n",
    "labels=[]\n",
    "for tweet in tqdm(json_data):\n",
    "    tot_cont = \"\"\n",
    "    for con in tweet['context']:\n",
    "        tot_cont+=con \n",
    "    tot_cont+=tweet['response']\n",
    "    tweets.append(tot_cont)\n",
    "    labels.append(tweet['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "label_list=[]\n",
    "for i in range(0,len(labels)):\n",
    "    if labels[i]==\"SARCASM\":\n",
    "        label_list.append(1)\n",
    "    else:\n",
    "        label_list.append(0)\n",
    "\n",
    "X_train =tweets\n",
    "y_train = label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5000, 1, 2048])\n",
      "torch.Size([5000])\n"
     ]
    }
   ],
   "source": [
    "train_data=[]\n",
    "mi=10000000000000000\n",
    "ma=-1\n",
    "for i in range(0,len(X_train)):\n",
    "    train_data.append(eval_conversation(X_train[i]));\n",
    "    mi=min(train_data[i].shape[1],mi)\n",
    "    ma=max(train_data[i].shape[1],ma)\n",
    "\n",
    "new_train_data=[]\n",
    "for i in range(0,len(train_data)):\n",
    "    if train_data[i].shape[1]<2048:\n",
    "        z = torch.zeros(1, 2048-train_data[i].shape[1])\n",
    "        s = torch.cat((train_data[i],z),1)\n",
    "        new_train_data.append(s)\n",
    "    else:\n",
    "        cur_train_data = torch.reshape(train_data[i][0][:2048], (1, 2048)).type(torch.FloatTensor)\n",
    "        new_train_data.append(torch.Tensor(cur_train_data))\n",
    "\n",
    "new_train_data = torch.stack(new_train_data)\n",
    "print(new_train_data.shape)\n",
    "\n",
    "y_train=torch.Tensor(y_train)\n",
    "print(y_train.shape)\n",
    "new_train_data = torch.squeeze(new_train_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch.optim as  optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch.utils.data as data_utils\n",
    "train=data_utils.TensorDataset(new_train_data ,y_train)\n",
    "\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train, batch_size=32,\n",
    "               shuffle=True, num_workers=0, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_json_file = '/scratch/sm9669/sarcasm_detection_shared_task_twitter_testing.jsonl'\n",
    "import json \n",
    "\n",
    "with open(path_to_json_file, 'r') as j:\n",
    "     json_data = [json.loads(line) for line in j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1800/1800 [00:00<00:00, 421867.86it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "tweets=[]\n",
    "labels=[]\n",
    "for tweet in tqdm(json_data):\n",
    "    tot_cont = \"\"\n",
    "    for con in tweet['context']:\n",
    "        tot_cont+=con \n",
    "    tot_cont+=tweet['response']\n",
    "    tweets.append(tot_cont)\n",
    "    labels.append(tweet['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "test_label_list=[]\n",
    "for i in range(0,len(labels)):\n",
    "    if labels[i]==\"SARCASM\":\n",
    "        test_label_list.append(1)\n",
    "    else:\n",
    "        test_label_list.append(0)\n",
    "\n",
    "X_test =tweets\n",
    "y_test = test_label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117 4140\n",
      "torch.Size([1800, 1, 2048])\n",
      "torch.Size([1800])\n"
     ]
    }
   ],
   "source": [
    "test_data=[]\n",
    "mi=10000000000000000\n",
    "ma=-1\n",
    "for i in range(0,len(X_test)):\n",
    "    test_data.append(eval_conversation(X_test[i]));\n",
    "    mi=min(test_data[i].shape[1],mi)\n",
    "    ma=max(test_data[i].shape[1],ma)\n",
    "\n",
    "print(mi, ma)\n",
    "new_test_data=[]\n",
    "for i in range(0,len(test_data)):\n",
    "    if test_data[i].shape[1]<2048:\n",
    "        z = torch.zeros(1, 2048-test_data[i].shape[1])\n",
    "        s = torch.cat((test_data[i],z),1)\n",
    "        new_test_data.append(s)\n",
    "    else:\n",
    "        cur_test_data = torch.reshape(test_data[i][0][:2048], (1, 2048)).type(torch.FloatTensor)\n",
    "        new_test_data.append(torch.Tensor(cur_test_data))\n",
    "\n",
    "new_test_data = torch.stack(new_test_data)\n",
    "print(new_test_data.shape)\n",
    "\n",
    "y_test=torch.Tensor(y_test)\n",
    "print(y_test.shape)\n",
    "new_test_data = torch.squeeze(new_test_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.utils.data as data_utils\n",
    "test=data_utils.TensorDataset(new_test_data ,y_test)\n",
    "\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32,\n",
    "               shuffle=True, num_workers=0, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "lr =  0.0045\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr,\n",
    "                            momentum=0.9,\n",
    "                            weight_decay=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculateMetrics(ypred,ytrue):\n",
    "    acc  = accuracy_score(ytrue,ypred)\n",
    "    f1  = f1_score(ytrue,ypred)\n",
    "    f1_average  = f1_score(ytrue,ypred,average=\"macro\")\n",
    "    return \" f1 score: \"+str(round(f1,3))+\" f1 average: \"+str(round(f1_average,3))+\" accuracy: \"+str(round(acc,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_loader, model, optimizer, epoch):\n",
    "\n",
    "    model.train()\n",
    "    trainpreds = torch.tensor([])\n",
    "    traintrues = torch.tensor([])\n",
    "    testpreds = torch.tensor([])\n",
    "    testtrues = torch.tensor([])\n",
    "    for i, (input, target) in enumerate(train_loader):\n",
    "        input_var = input.cuda()\n",
    "        target_var = target.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        output = model(input_var)\n",
    "        output = torch.squeeze(output, 1)\n",
    "        trainpreds = torch.cat((trainpreds,output.cpu().detach()))\n",
    "        traintrues = torch.cat((traintrues,target_var.cpu().detach()))\n",
    "        \n",
    "        loss = F.binary_cross_entropy(output,target_var)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "    err = F.binary_cross_entropy(trainpreds,traintrues)    \n",
    "    print(\"train BCE loss: \",err.item(),calculateMetrics(torch.round(trainpreds).numpy(),traintrues.numpy()))\n",
    "    \n",
    "    model.eval()\n",
    "    for i, (input, target) in enumerate(test_loader):\n",
    "        input_var = input.cuda()\n",
    "        target_var = target.cuda()\n",
    "        output = model(input_var)\n",
    "        output = torch.squeeze(output, 1)\n",
    "\n",
    "        testpreds = torch.cat((testpreds,output.cpu().detach()))\n",
    "        testtrues = torch.cat((testtrues,target_var.cpu().detach()))\n",
    "\n",
    "        loss = F.binary_cross_entropy(output,target_var)\n",
    "\n",
    "    print(\"test BCE loss: \",calculateMetrics(torch.round(testpreds).numpy(),testtrues.numpy()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train BCE loss:  0.6935937404632568  f1 score: 0.373 f1 average: 0.485 accuracy: 0.509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 1/30 [00:21<10:22, 21.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.667 f1 average: 0.333 accuracy: 0.5\n",
      "train BCE loss:  0.6913658976554871  f1 score: 0.446 f1 average: 0.518 accuracy: 0.529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 2/30 [00:42<10:01, 21.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.54 f1 average: 0.501 accuracy: 0.504\n",
      "train BCE loss:  0.6866149306297302  f1 score: 0.536 f1 average: 0.552 accuracy: 0.552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 3/30 [01:04<09:40, 21.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.41 f1 average: 0.491 accuracy: 0.503\n",
      "train BCE loss:  0.6839903593063354  f1 score: 0.52 f1 average: 0.56 accuracy: 0.564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 4/30 [01:26<09:20, 21.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.414 f1 average: 0.492 accuracy: 0.504\n",
      "train BCE loss:  0.6842751502990723  f1 score: 0.513 f1 average: 0.559 accuracy: 0.564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 5/30 [01:47<08:59, 21.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.587 f1 average: 0.504 accuracy: 0.518\n",
      "train BCE loss:  0.6843687891960144  f1 score: 0.572 f1 average: 0.559 accuracy: 0.559\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 6/30 [02:09<08:39, 21.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.398 f1 average: 0.486 accuracy: 0.502\n",
      "train BCE loss:  0.6827521920204163  f1 score: 0.523 f1 average: 0.56 accuracy: 0.563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 7/30 [02:31<08:18, 21.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.444 f1 average: 0.5 accuracy: 0.507\n",
      "train BCE loss:  0.6825318336486816  f1 score: 0.523 f1 average: 0.563 accuracy: 0.566\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 8/30 [02:52<07:56, 21.67s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.528 f1 average: 0.531 accuracy: 0.531\n",
      "train BCE loss:  0.6802144646644592  f1 score: 0.548 f1 average: 0.568 accuracy: 0.569\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 9/30 [03:14<07:35, 21.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.315 f1 average: 0.459 accuracy: 0.498\n",
      "train BCE loss:  0.6780626773834229  f1 score: 0.541 f1 average: 0.57 accuracy: 0.572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 10/30 [03:36<07:13, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.394 f1 average: 0.492 accuracy: 0.511\n",
      "train BCE loss:  0.6728733777999878  f1 score: 0.581 f1 average: 0.585 accuracy: 0.585\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 11/30 [03:58<06:52, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.302 f1 average: 0.462 accuracy: 0.509\n",
      "train BCE loss:  0.6709381937980652  f1 score: 0.574 f1 average: 0.586 accuracy: 0.586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 12/30 [04:19<06:30, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.648 f1 average: 0.442 accuracy: 0.518\n",
      "train BCE loss:  0.6674875020980835  f1 score: 0.605 f1 average: 0.601 accuracy: 0.601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 13/30 [04:41<06:08, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.62 f1 average: 0.521 accuracy: 0.542\n",
      "train BCE loss:  0.6512551307678223  f1 score: 0.625 f1 average: 0.62 accuracy: 0.62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 14/30 [05:03<05:47, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.487 f1 average: 0.529 accuracy: 0.532\n",
      "train BCE loss:  0.6414102911949158  f1 score: 0.636 f1 average: 0.635 accuracy: 0.635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 15/30 [05:24<05:25, 21.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.533 f1 average: 0.549 accuracy: 0.55\n",
      "train BCE loss:  0.642330527305603  f1 score: 0.642 f1 average: 0.634 accuracy: 0.635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 16/30 [05:46<05:03, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.478 f1 average: 0.53 accuracy: 0.536\n",
      "train BCE loss:  0.615013837814331  f1 score: 0.671 f1 average: 0.667 accuracy: 0.667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 17/30 [06:08<04:42, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.488 f1 average: 0.529 accuracy: 0.533\n",
      "train BCE loss:  0.5900360345840454  f1 score: 0.696 f1 average: 0.689 accuracy: 0.689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 18/30 [06:29<04:20, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.33 f1 average: 0.483 accuracy: 0.528\n",
      "train BCE loss:  0.5882797837257385  f1 score: 0.689 f1 average: 0.683 accuracy: 0.684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 19/30 [06:51<03:58, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.578 f1 average: 0.569 accuracy: 0.569\n",
      "train BCE loss:  0.5555251836776733  f1 score: 0.718 f1 average: 0.713 accuracy: 0.713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 20/30 [07:13<03:36, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.607 f1 average: 0.57 accuracy: 0.573\n",
      "train BCE loss:  0.5361648201942444  f1 score: 0.735 f1 average: 0.732 accuracy: 0.732\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 21/30 [07:35<03:15, 21.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.568 f1 average: 0.559 accuracy: 0.559\n",
      "train BCE loss:  0.5149388909339905  f1 score: 0.743 f1 average: 0.739 accuracy: 0.739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 22/30 [07:56<02:53, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.663 f1 average: 0.461 accuracy: 0.537\n",
      "train BCE loss:  0.5085058808326721  f1 score: 0.746 f1 average: 0.742 accuracy: 0.742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 23/30 [08:18<02:31, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.646 f1 average: 0.531 accuracy: 0.559\n",
      "train BCE loss:  0.48672983050346375  f1 score: 0.766 f1 average: 0.761 accuracy: 0.761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 24/30 [08:40<02:10, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.578 f1 average: 0.565 accuracy: 0.566\n",
      "train BCE loss:  0.4662865102291107  f1 score: 0.778 f1 average: 0.772 accuracy: 0.772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 25/30 [09:01<01:48, 21.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.562 f1 average: 0.57 accuracy: 0.571\n",
      "train BCE loss:  0.4398786723613739  f1 score: 0.793 f1 average: 0.791 accuracy: 0.791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 26/30 [09:23<01:26, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.623 f1 average: 0.537 accuracy: 0.553\n",
      "train BCE loss:  0.39745813608169556  f1 score: 0.822 f1 average: 0.82 accuracy: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 27/30 [09:45<01:05, 21.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.632 f1 average: 0.556 accuracy: 0.569\n",
      "train BCE loss:  0.4033062756061554  f1 score: 0.813 f1 average: 0.811 accuracy: 0.811\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 28/30 [10:06<00:43, 21.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.539 f1 average: 0.564 accuracy: 0.566\n",
      "train BCE loss:  0.3838340640068054  f1 score: 0.83 f1 average: 0.828 accuracy: 0.828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 29/30 [10:28<00:21, 21.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.589 f1 average: 0.573 accuracy: 0.574\n",
      "train BCE loss:  0.3666486442089081  f1 score: 0.836 f1 average: 0.835 accuracy: 0.835\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [10:50<00:00, 21.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test BCE loss:   f1 score: 0.52 f1 average: 0.564 accuracy: 0.568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "for epoch in tqdm(range(0, 30)):\n",
    "    lr = lr * (0.1 ** (epoch // 30))\n",
    "    train(train_loader, model, optimizer, epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), \"charformer-twitter-30.pt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}