bert_pe_finetune_sentence_pos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2219f782",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 1,
     "id": "031a2430-7ba4-431f-8e42-e31baf9ce991",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting pandas==1.3.4\n",
      "  Downloading pandas-1.3.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 11.5 MB 13.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (1.21.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas==1.3.4) (1.16.0)\n",
      "Installing collected packages: pandas\n",
      "Successfully installed pandas-1.3.4\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting transformers==4.12.5\n",
      "  Downloading transformers-4.12.5-py3-none-any.whl (3.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.1 MB 22.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: sacremoses in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (0.0.46)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (5.4.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (2.26.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (1.21.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (2021.10.8)\n",
      "Collecting tokenizers<0.11,>=0.10.1\n",
      "  Downloading tokenizers-0.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.3 MB 28.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
      "  Downloading huggingface_hub-0.8.1-py3-none-any.whl (101 kB)\n",
      "\u001b[K     |████████████████████████████████| 101 kB 32.4 MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (3.3.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (21.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (4.62.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.12.5) (3.10.0.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.0->transformers==4.12.5) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (2.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (3.1)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (1.16.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (8.0.1)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (1.1.0)\n",
      "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
      "Successfully installed huggingface-hub-0.8.1 tokenizers-0.10.3 transformers-4.12.5\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting datasets==1.15.1\n",
      "  Downloading datasets-1.15.1-py3-none-any.whl (290 kB)\n",
      "\u001b[K     |████████████████████████████████| 290 kB 12.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (2.26.0)\n",
      "Collecting fsspec[http]>=2021.05.0\n",
      "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
      "\u001b[K     |████████████████████████████████| 140 kB 33.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (0.8.1)\n",
      "Collecting aiohttp\n",
      "  Downloading aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 36.6 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (21.0)\n",
      "Collecting pyarrow!=4.0.0,>=1.0.0\n",
      "  Downloading pyarrow-8.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 29.4 MB 26.4 MB/s eta 0:00:01    |███                             | 2.8 MB 31.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (4.62.3)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (1.3.4)\n",
      "Collecting xxhash\n",
      "  Downloading xxhash-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[K     |████████████████████████████████| 212 kB 26.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting multiprocess\n",
      "  Downloading multiprocess-0.70.13-py38-none-any.whl (131 kB)\n",
      "\u001b[K     |████████████████████████████████| 131 kB 26.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (1.21.2)\n",
      "Collecting dill\n",
      "  Downloading dill-0.3.5.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[K     |████████████████████████████████| 95 kB 43.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (3.10.0.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (5.4.1)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (3.3.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->datasets==1.15.1) (2.4.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (3.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (2.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (1.26.7)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.15.1) (21.2.0)\n",
      "Collecting frozenlist>=1.1.1\n",
      "  Downloading frozenlist-1.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
      "\u001b[K     |████████████████████████████████| 158 kB 26.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
      "  Downloading yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (308 kB)\n",
      "\u001b[K     |████████████████████████████████| 308 kB 19.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
      "Collecting multidict<7.0,>=4.5\n",
      "  Downloading multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\n",
      "\u001b[K     |████████████████████████████████| 121 kB 31.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting aiosignal>=1.1.2\n",
      "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.15.1) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.15.1) (2021.3)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->datasets==1.15.1) (1.16.0)\n",
      "Installing collected packages: multidict, frozenlist, yarl, async-timeout, aiosignal, fsspec, dill, aiohttp, xxhash, pyarrow, multiprocess, datasets\n",
      "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 datasets-1.15.1 dill-0.3.5.1 frozenlist-1.3.0 fsspec-2022.5.0 multidict-6.0.2 multiprocess-0.70.13 pyarrow-8.0.0 xxhash-3.0.0 yarl-1.7.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting ipywidgets\n",
      "  Downloading ipywidgets-7.7.1-py2.py3-none-any.whl (123 kB)\n",
      "\u001b[K     |████████████████████████████████| 123 kB 22.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (5.1.0)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (0.2.0)\n",
      "Collecting widgetsnbextension~=3.6.0\n",
      "  Downloading widgetsnbextension-3.6.1-py2.py3-none-any.whl (1.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.6 MB 29.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (7.28.0)\n",
      "Collecting jupyterlab-widgets>=1.0.0\n",
      "  Downloading jupyterlab_widgets-1.1.1-py3-none-any.whl (245 kB)\n",
      "\u001b[K     |████████████████████████████████| 245 kB 26.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (6.4.1)\n",
      "Requirement already satisfied: debugpy<2.0,>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.5.0)\n",
      "Requirement already satisfied: tornado<7.0,>=4.2 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1)\n",
      "Requirement already satisfied: jupyter-client<8.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (7.0.6)\n",
      "Requirement already satisfied: matplotlib-inline<0.2.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.3)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (2.10.0)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.18.0)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: setuptools>=18.5 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (58.2.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.20)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (5.1.0)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.8/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.8.2)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (4.8.1)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (1.5.1)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (0.3)\n",
      "Requirement already satisfied: pyzmq>=13 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (22.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.8/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.1->jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (1.16.0)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.8/site-packages (from widgetsnbextension~=3.6.0->ipywidgets) (6.4.1)\n",
      "Requirement already satisfied: nbconvert in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.2.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.12.1)\n",
      "Requirement already satisfied: nbformat in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.1.3)\n",
      "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.11.0)\n",
      "Requirement already satisfied: Send2Trash>=1.5.0 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.8.0)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.0.1)\n",
      "Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.1.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.14.6)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi>=1.0.0->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.20)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.8/site-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.0.1)\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.0)\n",
      "Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.1.2)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.4)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.1.0)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.4)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.7.1)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.8/site-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.0.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.2.0)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.18.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.0)\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.4.7)\n",
      "Installing collected packages: widgetsnbextension, jupyterlab-widgets, ipywidgets\n",
      "Successfully installed ipywidgets-7.7.1 jupyterlab-widgets-1.1.1 widgetsnbextension-3.6.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install pandas==1.3.4\n",
    "!pip install transformers==4.12.5\n",
    "!pip install datasets==1.15.1\n",
    "!pip install ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aead07a6",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 3,
     "id": "db736e14-03fb-4438-938d-c705d6786666",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "# import pandas as pd\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import transformers\n",
    "from transformers import Trainer\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertForSequenceClassification\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from transformers.data.data_collator import DataCollatorWithPadding\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset\n",
    "from datasets import ClassLabel\n",
    "from datasets import load_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79862709",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e63304e7",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "37286c02-6bf4-41f5-ab70-51515a054e7b",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "source": [
    "## Global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ba2c6605",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 2,
     "id": "0c699094-439f-4dda-85a9-815e7948540c",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "DATA_FOLDER = '/notebooks/Data/bert_sequence_classification'\n",
    "DATA_FILE = '/notebooks/linguistic_features/data/hf_datasets/pe_dataset_linguistic_features.pt'\n",
    "RESULTS_FOLDER = '/notebooks/Results/bert_sequence_classification'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d4e1e15",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 3,
     "id": "35aada91-232a-421e-a28f-94b359c6d65d",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "428c8a28",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 4,
     "id": "dc52e71e-65fa-4946-acdf-e4fffe9d0f79",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b65959b5",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "d352c1cd-abff-4b1e-b5e2-611288e4fddb",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c5c707ce",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "013a6d64-65e7-4189-a468-40ecfd8a6736",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "885e7fa7",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 5,
     "id": "626737f9-ff56-4992-b72b-60750375e455",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset = torch.load(DATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5fe5863e",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 6,
     "id": "4fbfcc7b-55fb-456e-ab02-2ae975803046",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 3770\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 1260\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 943\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9112dc3e",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 7,
     "id": "57c21a2e-83cd-4f4b-a19a-0bae29eb4794",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Topic: Young people should go to university or not, Sentence: Although higher education does not guarantee young students' success, the benefits of learning in universities are the vital factor in creating more possibilities for the development of society through advancing academic fulfillment and the young generation who have experiences and challenges., Para Number: 4, First in Para: No, Last in Para: Yes, Is in Introduction: No, Is in Conclusion: Yes. Part Of Speech tags: SCONJ, ADJ, NOUN, AUX, PART, VERB, ADJ, NOUN, PART, NOUN, PUNCT, DET, NOUN, ADP, VERB, ADP, NOUN, AUX, DET, ADJ, NOUN, ADP, VERB, ADJ, NOUN, ADP, DET, NOUN, ADP, NOUN, ADP, VERB, ADJ, NOUN, CCONJ, DET, ADJ, NOUN, PRON, AUX, NOUN, CCONJ, NOUN, PUNCT\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train']['strct_fts_and_sentence_pos'][230]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aaf8afd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aefd16ec",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 4,
     "id": "492e0415-2634-47bb-88bf-665c130d032c",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d870ee7e0cb241649d1f7669d0ca468a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ae3407971e6471e91700cd07ca09d0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3acf008d446c458fb04b051ba241dcc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37c86a1da5b244d68932f7cfbb25eb5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6ca7920e",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 9,
     "id": "ba2c9132-d013-4b26-be96-146cf024a45e",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "label_names = set(dataset['train']['label_ComponentType'])\n",
    "label_nb = len(label_names)\n",
    "labels = ClassLabel(num_classes=label_nb, names=label_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "599f3dda",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 10,
     "id": "2f1ce51b-18bc-40db-983f-3434b8651e45",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ClassLabel(num_classes=3, names={'Premise', 'Claim', 'MajorClaim'}, names_file=None, id=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fba4c4b0",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 11,
     "id": "5438b78c-249c-48ac-98e8-a87a2cc0c86d",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize(batch):\n",
    "    tokens = tokenizer(batch['strct_fts_and_sentence_pos'], truncation=True, padding=True, max_length=512)\n",
    "    tokens['labels'] = labels.str2int(batch['label_ComponentType'])\n",
    "    return tokens\n",
    "\n",
    "# this is just the text. if the results are nice, check transfer with text + topic "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "728f6f71",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 12,
     "id": "cb8d8c9a-7539-4d02-9dc5-98bade787549",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parameter 'function'=<function tokenize at 0x7febb2772670> of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5905b9e00ec84d1e84c0f82594098f63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2666011398714650981ad542f5d2e3e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1c9575e51df47d39a5bc21a56d0dcb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = dataset.map(tokenize, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e3956901",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 13,
     "id": "84abf830-f842-4e91-8d5c-7fcd9ea2942e",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d0e9ccc3",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 14,
     "id": "c7d70e98-c2e0-4055-9d52-74c67c199c72",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 3770\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 1260\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 943\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d44beba9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "337d52d0",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 15,
     "id": "e825c71a-23f8-4794-b114-729819def9ab",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = dataset['train'].shuffle(seed=42)\n",
    "test_dataset = dataset['test'].shuffle(seed=42)\n",
    "\n",
    "train_val_datasets = dataset['train'].train_test_split(train_size=0.8, seed=42)\n",
    "train_dataset = train_val_datasets['train']\n",
    "val_dataset = train_val_datasets['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f20cc8a3",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 16,
     "id": "e13433b1-343d-417d-bfbd-8bacb4c57d23",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset_d = {}\n",
    "dataset_d['train'] = train_dataset\n",
    "dataset_d['test'] = test_dataset\n",
    "dataset_d['val'] = val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1a481dfc",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 17,
     "id": "a298a871-1a0d-4ed8-9a1c-1ed3795f1d74",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] topic : zoos have no useful purpose?, sentence : in the zoo you can see an animal and their different variations, the male and the female or the baby and the adult, para number : 3, first in para : no, last in para : no, is in introduction : no, is in conclusion : no. part of speech tags : adp, det, noun, pron, verb, verb, det, noun, cconj, det, adj, noun, punct, det, noun, cconj, det, noun, cconj, det, noun, cconj, det, noun [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(dataset['train'][2945]['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "79429f00",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 18,
     "id": "aa1384b3-bd47-41a3-86b1-9cf814d47cdf",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TRAIN'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['train']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "298cf630",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 19,
     "id": "84df366a-6334-4e3a-902b-deb1c99491fb",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TRAIN'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['val']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c42fe849",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 20,
     "id": "406ad783-a570-4c26-8aa6-243e866b6fbf",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TEST'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['test']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "07ca1849",
   "metadata": {},
   "outputs": [],
   "source": [
    "# global variables\n",
    "NUM_LABELS = labels.num_classes\n",
    "BATCH_SIZE = 48\n",
    "NB_EPOCHS = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "de3f9e96",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24637fef0706417a9a08ca28b2f3ea50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=NUM_LABELS)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65407f1f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0315b6d3",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 27,
     "id": "521987db-56d5-4e71-95af-7d1fb4a06e3f",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "# https://huggingface.co/transformers/main_classes/trainer.html\n",
    "class CustomTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get('logits')\n",
    "        loss_fct = nn.CrossEntropyLoss()#(weight=class_weights)\n",
    "        loss = loss_fct(logits, labels)\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2f0bf362",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 28,
     "id": "06697293-0ef4-4d24-95a9-6ca0ed569b51",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b998b82d43cc4059b8b4120e2dde5556",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/2.07k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metric = load_metric('f1')\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    \n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    \n",
    "    return metric.compute(predictions=predictions, references=labels, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cce51b84",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 29,
     "id": "956f8a45-f8bd-42a5-b829-5e2b0e82cf42",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    \n",
    "    # output\n",
    "    output_dir=RESULTS_FOLDER,          \n",
    "    \n",
    "    # params\n",
    "    num_train_epochs=NB_EPOCHS,               # nb of epochs\n",
    "    per_device_train_batch_size=BATCH_SIZE,   # batch size per device during training\n",
    "    per_device_eval_batch_size=BATCH_SIZE,    # cf. paper Sun et al.\n",
    "    learning_rate=1e-5,#2e-5,                 # cf. paper Sun et al.\n",
    "#     warmup_steps=500,                         # number of warmup steps for learning rate scheduler\n",
    "    warmup_ratio=0.1,                         # cf. paper Sun et al.\n",
    "    weight_decay=0.01,                        # strength of weight decay\n",
    "    \n",
    "    # eval\n",
    "    evaluation_strategy=\"steps\",              # cf. paper Sun et al.\n",
    "    eval_steps=20,                            # cf. paper Sun et al.\n",
    "    \n",
    "    # log\n",
    "    logging_dir=\"/notebooks/Results/bert_sequence_classification/tb_logs\",  \n",
    "    logging_strategy='steps',\n",
    "    logging_steps=20,\n",
    "    \n",
    "    # save\n",
    "    save_strategy='steps',\n",
    "    save_total_limit=2,\n",
    "    # save_steps=20, # default 500\n",
    "    load_best_model_at_end=True,              # cf. paper Sun et al.\n",
    "    # metric_for_best_model='eval_loss' \n",
    "    metric_for_best_model='f1'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9043e0b1",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 30,
     "id": "219545f7-5aff-4d0c-87e2-4ca28a6e7acc",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "trainer = CustomTrainer( # Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics\n",
    "    # callbacks=[EarlyStoppingCallback(early_stopping_patience=5)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7258a3e2",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 31,
     "id": "b524a80a-ccf0-41f4-a015-e4ba2913fdc4",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running training *****\n",
      "  Num examples = 3016\n",
      "  Num Epochs = 6\n",
      "  Instantaneous batch size per device = 48\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 48\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 378\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='378' max='378' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [378/378 10:26, Epoch 6/6]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>1.044300</td>\n",
       "      <td>0.951897</td>\n",
       "      <td>0.257662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.905100</td>\n",
       "      <td>0.881562</td>\n",
       "      <td>0.257662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.844700</td>\n",
       "      <td>0.861775</td>\n",
       "      <td>0.257662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.794900</td>\n",
       "      <td>0.780641</td>\n",
       "      <td>0.351450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.727500</td>\n",
       "      <td>0.681806</td>\n",
       "      <td>0.432060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.649100</td>\n",
       "      <td>0.582885</td>\n",
       "      <td>0.730064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.561000</td>\n",
       "      <td>0.547803</td>\n",
       "      <td>0.738310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>0.540963</td>\n",
       "      <td>0.699613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.466300</td>\n",
       "      <td>0.530244</td>\n",
       "      <td>0.725176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.500400</td>\n",
       "      <td>0.495348</td>\n",
       "      <td>0.751404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.470400</td>\n",
       "      <td>0.484368</td>\n",
       "      <td>0.762015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.453100</td>\n",
       "      <td>0.471407</td>\n",
       "      <td>0.768977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.426500</td>\n",
       "      <td>0.477550</td>\n",
       "      <td>0.769112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.435800</td>\n",
       "      <td>0.477051</td>\n",
       "      <td>0.762726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.411600</td>\n",
       "      <td>0.475634</td>\n",
       "      <td>0.764581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>0.419400</td>\n",
       "      <td>0.463278</td>\n",
       "      <td>0.770268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>0.426200</td>\n",
       "      <td>0.473081</td>\n",
       "      <td>0.759984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>0.393200</td>\n",
       "      <td>0.463858</td>\n",
       "      <td>0.773168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1efe63ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save best model\n",
    "trainer.save_model(os.path.join(\"/notebooks/Results/bert_sequence_classification\", 'best-model-for-probs'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "965aa4ce",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "15193ed0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No `TrainingArguments` passed, using `output_dir=tmp_trainer`.\n",
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "The following columns in the test set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: strct_fts_and_sentence_pos, strct_pos_syn_deps_sentence, argument_bound_1, component_POS, nr_following_comps_in_para, nr_preceeding_comps_in_para, relative_position_in_para_char, sentence, succeeding_tokens_in_sentence_count, label_ComponentType, split, preceeding_tokens_in_sentence_count, is_in_conclusion, essay_nr, label_RelationType, component_id, label_LinkedNotLinked, tokens_count_covering_sentence, strct_fts_and_sentence_syn_deps, token_count, token_ratio, is_first_in_para, structural_fts_as_text, is_last_in_para, strct_pos_syn_deps_component, total_paras, is_in_intro, para_nr, paragraph, relative_position_in_para_token, argument_id, label_x, argument_bound_2, label_and_comp_idxs, component_syn_deps, strct_fts_and_component_pos, relation_SupportAttack, structural_fts_as_text_combined, sentence_POS, essay, sentence_syn_deps, strct_fts_and_component_syn_deps, text, token_count_covering_para.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 1260\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='158' max='158' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [158/158 00:11]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_trainer = Trainer(model, data_collator=DataCollatorWithPadding(tokenizer))\n",
    "test_raw_preds, test_labels, _ = test_trainer.predict(test_dataset)\n",
    "test_preds = np.argmax(test_raw_preds, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "17c1f0cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1260"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "877fbe07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Premise      0.930     0.878     0.904       805\n",
      "       Claim      0.667     0.649     0.658       302\n",
      "  MajorClaim      0.723     0.974     0.830       153\n",
      "\n",
      "    accuracy                          0.835      1260\n",
      "   macro avg      0.773     0.834     0.797      1260\n",
      "weighted avg      0.842     0.835     0.836      1260\n",
      "\n"
     ]
    }
   ],
   "source": [
    "target_name = labels.int2str([0,1,2])\n",
    "print(classification_report(test_labels, test_preds, target_names=target_name, digits=3))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "354074e2",
   "metadata": {},
   "source": [
    "precision    recall  f1-score   support\n",
    "\n",
    "     Premise      0.930     0.878     0.904       805\n",
    "       Claim      0.667     0.649     0.658       302\n",
    "  MajorClaim      0.723     0.974     0.830       153\n",
    "\n",
    "    accuracy                          0.835      1260\n",
    "   macro avg      0.773     0.834     0.797      1260\n",
    "weighted avg      0.842     0.835     0.836      1260\n",
    "\n",
    "field used: strct_fts_and_sentence_pos, 1e-5, 48, 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2d0557",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}