bert_pe_finetune_component_pos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f83a370c",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 1,
     "id": "031a2430-7ba4-431f-8e42-e31baf9ce991",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting pandas==1.3.4\n",
      "  Downloading pandas-1.3.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 11.5 MB 13.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2.8.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (1.21.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas==1.3.4) (1.16.0)\n",
      "Installing collected packages: pandas\n",
      "Successfully installed pandas-1.3.4\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting transformers==4.12.5\n",
      "  Downloading transformers-4.12.5-py3-none-any.whl (3.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.1 MB 22.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (4.62.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (5.4.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (1.21.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (2021.10.8)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (3.3.0)\n",
      "Collecting tokenizers<0.11,>=0.10.1\n",
      "  Downloading tokenizers-0.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.3 MB 33.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: sacremoses in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (0.0.46)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (2.26.0)\n",
      "Collecting huggingface-hub<1.0,>=0.1.0\n",
      "  Downloading huggingface_hub-0.8.1-py3-none-any.whl (101 kB)\n",
      "\u001b[K     |████████████████████████████████| 101 kB 31.2 MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from transformers==4.12.5) (21.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.12.5) (3.10.0.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.0->transformers==4.12.5) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (2.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.12.5) (3.1)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (1.16.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (8.0.1)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.12.5) (1.1.0)\n",
      "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
      "Successfully installed huggingface-hub-0.8.1 tokenizers-0.10.3 transformers-4.12.5\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting datasets==1.15.1\n",
      "  Downloading datasets-1.15.1-py3-none-any.whl (290 kB)\n",
      "\u001b[K     |████████████████████████████████| 290 kB 14.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyarrow!=4.0.0,>=1.0.0\n",
      "  Downloading pyarrow-8.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 29.4 MB 22.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (2.26.0)\n",
      "Collecting dill\n",
      "  Downloading dill-0.3.5.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[K     |████████████████████████████████| 95 kB 30.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting multiprocess\n",
      "  Downloading multiprocess-0.70.13-py38-none-any.whl (131 kB)\n",
      "\u001b[K     |████████████████████████████████| 131 kB 31.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (1.21.2)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (4.62.3)\n",
      "Collecting aiohttp\n",
      "  Downloading aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 38.1 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (0.8.1)\n",
      "Collecting fsspec[http]>=2021.05.0\n",
      "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
      "\u001b[K     |████████████████████████████████| 140 kB 28.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (1.3.4)\n",
      "Collecting xxhash\n",
      "  Downloading xxhash-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[K     |████████████████████████████████| 212 kB 12.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from datasets==1.15.1) (21.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (3.3.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets==1.15.1) (3.10.0.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->datasets==1.15.1) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (2.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.15.1) (3.1)\n",
      "Collecting async-timeout<5.0,>=4.0.0a3\n",
      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
      "Collecting multidict<7.0,>=4.5\n",
      "  Downloading multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\n",
      "\u001b[K     |████████████████████████████████| 121 kB 27.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting frozenlist>=1.1.1\n",
      "  Downloading frozenlist-1.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
      "\u001b[K     |████████████████████████████████| 158 kB 29.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting aiosignal>=1.1.2\n",
      "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.15.1) (21.2.0)\n",
      "Collecting yarl<2.0,>=1.0\n",
      "  Downloading yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (308 kB)\n",
      "\u001b[K     |████████████████████████████████| 308 kB 28.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.15.1) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.15.1) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->datasets==1.15.1) (1.16.0)\n",
      "Installing collected packages: multidict, frozenlist, yarl, async-timeout, aiosignal, fsspec, dill, aiohttp, xxhash, pyarrow, multiprocess, datasets\n",
      "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 datasets-1.15.1 dill-0.3.5.1 frozenlist-1.3.0 fsspec-2022.5.0 multidict-6.0.2 multiprocess-0.70.13 pyarrow-8.0.0 xxhash-3.0.0 yarl-1.7.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting ipywidgets\n",
      "  Downloading ipywidgets-7.7.1-py2.py3-none-any.whl (123 kB)\n",
      "\u001b[K     |████████████████████████████████| 123 kB 16.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (6.4.1)\n",
      "Collecting widgetsnbextension~=3.6.0\n",
      "  Downloading widgetsnbextension-3.6.1-py2.py3-none-any.whl (1.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.6 MB 29.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting jupyterlab-widgets>=1.0.0\n",
      "  Downloading jupyterlab_widgets-1.1.1-py3-none-any.whl (245 kB)\n",
      "\u001b[K     |████████████████████████████████| 245 kB 28.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (7.28.0)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (5.1.0)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: debugpy<2.0,>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.5.0)\n",
      "Requirement already satisfied: tornado<7.0,>=4.2 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1)\n",
      "Requirement already satisfied: matplotlib-inline<0.2.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.3)\n",
      "Requirement already satisfied: jupyter-client<8.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (7.0.6)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (2.10.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (5.1.0)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.18.0)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.20)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: setuptools>=18.5 in /opt/conda/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (58.2.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.8/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.8.2)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (0.3)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (1.5.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (4.8.1)\n",
      "Requirement already satisfied: pyzmq>=13 in /opt/conda/lib/python3.8/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (22.3.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.8/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.1->jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets) (1.16.0)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.8/site-packages (from widgetsnbextension~=3.6.0->ipywidgets) (6.4.1)\n",
      "Requirement already satisfied: nbconvert in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.2.0)\n",
      "Requirement already satisfied: nbformat in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.1.3)\n",
      "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.11.0)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.0.1)\n",
      "Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.1.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.12.1)\n",
      "Requirement already satisfied: Send2Trash>=1.5.0 in /opt/conda/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.8.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.14.6)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi>=1.0.0->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.20)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.8/site-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.0.1)\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.0)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.4)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.0)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.4)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.1.0)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.7.1)\n",
      "Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.1.2)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.8/site-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.0.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.2.0)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.18.0)\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.1)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.4.7)\n",
      "Installing collected packages: widgetsnbextension, jupyterlab-widgets, ipywidgets\n",
      "Successfully installed ipywidgets-7.7.1 jupyterlab-widgets-1.1.1 widgetsnbextension-3.6.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install pandas==1.3.4\n",
    "!pip install transformers==4.12.5\n",
    "!pip install datasets==1.15.1\n",
    "!pip install ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a9fab128",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 3,
     "id": "db736e14-03fb-4438-938d-c705d6786666",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "# import pandas as pd\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import transformers\n",
    "from transformers import Trainer\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertForSequenceClassification\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from transformers.data.data_collator import DataCollatorWithPadding\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset\n",
    "from datasets import ClassLabel\n",
    "from datasets import load_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a6d9c14",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9d8f3825",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "37286c02-6bf4-41f5-ab70-51515a054e7b",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "source": [
    "## Global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f5c2a76a",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 2,
     "id": "0c699094-439f-4dda-85a9-815e7948540c",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "DATA_FOLDER = '/notebooks/Data/bert_sequence_classification'\n",
    "DATA_FILE = '/notebooks/linguistic_features/data/hf_datasets/pe_dataset_linguistic_features.pt'\n",
    "RESULTS_FOLDER = '/notebooks/Results/bert_sequence_classification'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "faadad02",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 3,
     "id": "35aada91-232a-421e-a28f-94b359c6d65d",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7b9b9c4b",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 4,
     "id": "dc52e71e-65fa-4946-acdf-e4fffe9d0f79",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cb017e4",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "d352c1cd-abff-4b1e-b5e2-611288e4fddb",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f1461702",
   "metadata": {
    "gradient": {
     "editing": false,
     "id": "013a6d64-65e7-4189-a468-40ecfd8a6736",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f561646b",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 5,
     "id": "626737f9-ff56-4992-b72b-60750375e455",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset = torch.load(DATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "33bae5b0",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 6,
     "id": "4fbfcc7b-55fb-456e-ab02-2ae975803046",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 3770\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 1260\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['essay_nr', 'component_id', 'label_and_comp_idxs', 'text', 'label_x', 'label_ComponentType', 'relation_SupportAttack', 'label_RelationType', 'label_LinkedNotLinked', 'split', 'essay', 'argument_bound_1', 'argument_bound_2', 'argument_id', 'sentence', 'paragraph', 'para_nr', 'total_paras', 'token_count', 'token_count_covering_para', 'tokens_count_covering_sentence', 'preceeding_tokens_in_sentence_count', 'succeeding_tokens_in_sentence_count', 'token_ratio', 'relative_position_in_para_char', 'is_in_intro', 'relative_position_in_para_token', 'is_in_conclusion', 'is_first_in_para', 'is_last_in_para', 'nr_preceeding_comps_in_para', 'nr_following_comps_in_para', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'component_POS', 'strct_fts_and_component_pos', 'sentence_POS', 'strct_fts_and_sentence_pos', 'component_syn_deps', 'strct_fts_and_component_syn_deps', 'sentence_syn_deps', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence'],\n",
       "        num_rows: 943\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb3fe20c",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 7,
     "id": "57c21a2e-83cd-4f4b-a19a-0bae29eb4794",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Topic: Young people should go to university or not, Sentence: Although higher education does not guarantee young students' success, the benefits of learning in universities are the vital factor in creating more possibilities for the development of society through advancing academic fulfillment and the young generation who have experiences and challenges., Para Number: 4, First in Para: No, Last in Para: Yes, Is in Introduction: No, Is in Conclusion: Yes. Part Of Speech tags: DET, NOUN, ADP, VERB, ADP, NOUN, AUX, DET, ADJ, NOUN, ADP, VERB, ADJ, NOUN, ADP, DET, NOUN, ADP, NOUN, ADP, VERB, ADJ, NOUN, CCONJ, DET, ADJ, NOUN, PRON, AUX, NOUN, CCONJ, NOUN\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train']['strct_fts_and_component_pos'][230]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "57831b19",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 4,
     "id": "492e0415-2634-47bb-88bf-665c130d032c",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37a3a472a3134ac385d9f2d1acb9fd07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94084d3d22ef4a9796f9a532e8f76354",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f32b0c8a298e439f88011a109fd1ba1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71c7428eafc34e8a9511e05bc3294e5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "014406b8",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 9,
     "id": "ba2c9132-d013-4b26-be96-146cf024a45e",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "label_names = set(dataset['train']['label_ComponentType'])\n",
    "label_nb = len(label_names)\n",
    "labels = ClassLabel(num_classes=label_nb, names=label_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f34065c1",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 10,
     "id": "2f1ce51b-18bc-40db-983f-3434b8651e45",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ClassLabel(num_classes=3, names={'Premise', 'MajorClaim', 'Claim'}, names_file=None, id=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "13dc644b",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 11,
     "id": "5438b78c-249c-48ac-98e8-a87a2cc0c86d",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize(batch):\n",
    "    tokens = tokenizer(batch['strct_fts_and_component_pos'], truncation=True, padding=True, max_length=512)\n",
    "    tokens['labels'] = labels.str2int(batch['label_ComponentType'])\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "062f8a56",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 12,
     "id": "cb8d8c9a-7539-4d02-9dc5-98bade787549",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parameter 'function'=<function tokenize at 0x7f55c581d430> of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cacb92b05e974d2d865ce266f9ae1e0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28b696b875e84f16b61405ee151f3f0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db6ac160dbaf4b83ad62d65af6ef64a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = dataset.map(tokenize, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e031830d",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 13,
     "id": "84abf830-f842-4e91-8d5c-7fcd9ea2942e",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "90e68039",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 14,
     "id": "c7d70e98-c2e0-4055-9d52-74c67c199c72",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 3770\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 1260\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['argument_bound_1', 'argument_bound_2', 'argument_id', 'attention_mask', 'component_POS', 'component_id', 'component_syn_deps', 'essay', 'essay_nr', 'input_ids', 'is_first_in_para', 'is_in_conclusion', 'is_in_intro', 'is_last_in_para', 'label_ComponentType', 'label_LinkedNotLinked', 'label_RelationType', 'label_and_comp_idxs', 'label_x', 'labels', 'nr_following_comps_in_para', 'nr_preceeding_comps_in_para', 'para_nr', 'paragraph', 'preceeding_tokens_in_sentence_count', 'relation_SupportAttack', 'relative_position_in_para_char', 'relative_position_in_para_token', 'sentence', 'sentence_POS', 'sentence_syn_deps', 'split', 'strct_fts_and_component_pos', 'strct_fts_and_component_syn_deps', 'strct_fts_and_sentence_pos', 'strct_fts_and_sentence_syn_deps', 'strct_pos_syn_deps_component', 'strct_pos_syn_deps_sentence', 'structural_fts_as_text', 'structural_fts_as_text_combined', 'succeeding_tokens_in_sentence_count', 'text', 'token_count', 'token_count_covering_para', 'token_ratio', 'token_type_ids', 'tokens_count_covering_sentence', 'total_paras'],\n",
       "        num_rows: 943\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5c9cb12f",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 15,
     "id": "e825c71a-23f8-4794-b114-729819def9ab",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = dataset['train'].shuffle(seed=42)\n",
    "test_dataset = dataset['test'].shuffle(seed=42)\n",
    "\n",
    "train_val_datasets = dataset['train'].train_test_split(train_size=0.8, seed=42)\n",
    "train_dataset = train_val_datasets['train']\n",
    "val_dataset = train_val_datasets['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4305a7ad",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 16,
     "id": "e13433b1-343d-417d-bfbd-8bacb4c57d23",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "dataset_d = {}\n",
    "dataset_d['train'] = train_dataset\n",
    "dataset_d['test'] = test_dataset\n",
    "dataset_d['val'] = val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7e3d2785",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 17,
     "id": "a298a871-1a0d-4ed8-9a1c-1ed3795f1d74",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] topic : zoos have no useful purpose?, sentence : in the zoo you can see an animal and their different variations, the male and the female or the baby and the adult, para number : 3, first in para : no, last in para : no, is in introduction : no, is in conclusion : no. part of speech tags : adp, det, noun, pron, verb, verb, det, noun, cconj, det, adj, noun, punct, det, noun, cconj, det, noun, cconj, det, noun, cconj, det, noun [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(dataset['train'][2945]['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2a36722a",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 18,
     "id": "aa1384b3-bd47-41a3-86b1-9cf814d47cdf",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TRAIN'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['train']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "124b47d1",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 19,
     "id": "84df366a-6334-4e3a-902b-deb1c99491fb",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TRAIN'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['val']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e867fb86",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 20,
     "id": "406ad783-a570-4c26-8aa6-243e866b6fbf",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TEST'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check\n",
    "set(dataset_d['test']['split'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "61a04b5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# global variables\n",
    "NUM_LABELS = labels.num_classes\n",
    "BATCH_SIZE = 48\n",
    "NB_EPOCHS = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b8854199",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff7c64188fe645d59204805d4c799e4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=NUM_LABELS)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a2cc5f01",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 27,
     "id": "521987db-56d5-4e71-95af-7d1fb4a06e3f",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "# https://huggingface.co/transformers/main_classes/trainer.html\n",
    "class CustomTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get('logits')\n",
    "        loss_fct = nn.CrossEntropyLoss()#(weight=class_weights)\n",
    "        loss = loss_fct(logits, labels)\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "91de6d28",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 28,
     "id": "06697293-0ef4-4d24-95a9-6ca0ed569b51",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d34f8ca6900482087698cdf58d7976a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/2.07k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metric = load_metric('f1')\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    \n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    \n",
    "    return metric.compute(predictions=predictions, references=labels, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dae18384",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 29,
     "id": "956f8a45-f8bd-42a5-b829-5e2b0e82cf42",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    \n",
    "    # output\n",
    "    output_dir=RESULTS_FOLDER,          \n",
    "    \n",
    "    # params\n",
    "    num_train_epochs=NB_EPOCHS,               # nb of epochs\n",
    "    per_device_train_batch_size=BATCH_SIZE,   # batch size per device during training\n",
    "    per_device_eval_batch_size=BATCH_SIZE,    # cf. paper Sun et al.\n",
    "    learning_rate=1e-5,#2e-5,                 # cf. paper Sun et al.\n",
    "#     warmup_steps=500,                         # number of warmup steps for learning rate scheduler\n",
    "    warmup_ratio=0.1,                         # cf. paper Sun et al.\n",
    "    weight_decay=0.01,                        # strength of weight decay\n",
    "    \n",
    "    # eval\n",
    "    evaluation_strategy=\"steps\",              # cf. paper Sun et al.\n",
    "    eval_steps=20,                            # cf. paper Sun et al.\n",
    "    \n",
    "    # log\n",
    "    logging_dir=\"/notebooks/Results/bert_sequence_classification/tb_logs\",  \n",
    "    logging_strategy='steps',\n",
    "    logging_steps=20,\n",
    "    \n",
    "    # save\n",
    "    save_strategy='steps',\n",
    "    save_total_limit=2,\n",
    "    # save_steps=20, # default 500\n",
    "    load_best_model_at_end=True,              # cf. paper Sun et al.\n",
    "    # metric_for_best_model='eval_loss' \n",
    "    metric_for_best_model='f1'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f507f5d8",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 30,
     "id": "219545f7-5aff-4d0c-87e2-4ca28a6e7acc",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    }
   },
   "outputs": [],
   "source": [
    "trainer = CustomTrainer( # Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics\n",
    "    # callbacks=[EarlyStoppingCallback(early_stopping_patience=5)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cea63f2e",
   "metadata": {
    "gradient": {
     "editing": false,
     "execution_count": 31,
     "id": "b524a80a-ccf0-41f4-a015-e4ba2913fdc4",
     "kernelId": "5ae81875-6bf9-4a4d-b9d2-28830bffd5f4"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running training *****\n",
      "  Num examples = 3016\n",
      "  Num Epochs = 6\n",
      "  Instantaneous batch size per device = 48\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 48\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 378\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='378' max='378' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [378/378 10:10, Epoch 6/6]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.921800</td>\n",
       "      <td>0.897850</td>\n",
       "      <td>0.257662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.881900</td>\n",
       "      <td>0.860345</td>\n",
       "      <td>0.257662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.767500</td>\n",
       "      <td>0.693912</td>\n",
       "      <td>0.555336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.642900</td>\n",
       "      <td>0.584077</td>\n",
       "      <td>0.706025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.577100</td>\n",
       "      <td>0.551745</td>\n",
       "      <td>0.627097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.550600</td>\n",
       "      <td>0.517195</td>\n",
       "      <td>0.710884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.512100</td>\n",
       "      <td>0.510231</td>\n",
       "      <td>0.726736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.505500</td>\n",
       "      <td>0.520325</td>\n",
       "      <td>0.677141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.459800</td>\n",
       "      <td>0.509204</td>\n",
       "      <td>0.759773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.497800</td>\n",
       "      <td>0.509078</td>\n",
       "      <td>0.736145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.465100</td>\n",
       "      <td>0.522719</td>\n",
       "      <td>0.733267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.466400</td>\n",
       "      <td>0.501599</td>\n",
       "      <td>0.757818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.438600</td>\n",
       "      <td>0.510043</td>\n",
       "      <td>0.750256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.451500</td>\n",
       "      <td>0.508151</td>\n",
       "      <td>0.767890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.429100</td>\n",
       "      <td>0.503261</td>\n",
       "      <td>0.765954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>0.451300</td>\n",
       "      <td>0.479678</td>\n",
       "      <td>0.778412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>0.435600</td>\n",
       "      <td>0.476043</td>\n",
       "      <td>0.772677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>0.412000</td>\n",
       "      <td>0.477318</td>\n",
       "      <td>0.775898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 754\n",
      "  Batch size = 48\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e777afc7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "71f905b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No `TrainingArguments` passed, using `output_dir=tmp_trainer`.\n",
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "The following columns in the test set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: argument_bound_2, essay_nr, strct_pos_syn_deps_component, tokens_count_covering_sentence, essay, is_in_conclusion, text, nr_preceeding_comps_in_para, strct_fts_and_component_pos, relative_position_in_para_char, relative_position_in_para_token, strct_fts_and_component_syn_deps, relation_SupportAttack, argument_bound_1, sentence_syn_deps, token_count, component_POS, split, total_paras, argument_id, is_in_intro, label_LinkedNotLinked, sentence_POS, component_id, label_and_comp_idxs, succeeding_tokens_in_sentence_count, component_syn_deps, paragraph, label_x, strct_fts_and_sentence_syn_deps, structural_fts_as_text, nr_following_comps_in_para, strct_fts_and_sentence_pos, label_ComponentType, is_first_in_para, sentence, preceeding_tokens_in_sentence_count, is_last_in_para, structural_fts_as_text_combined, label_RelationType, token_count_covering_para, strct_pos_syn_deps_sentence, para_nr, token_ratio.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 1260\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='158' max='158' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [158/158 00:10]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_trainer = Trainer(model, data_collator=DataCollatorWithPadding(tokenizer))\n",
    "test_raw_preds, test_labels, _ = test_trainer.predict(test_dataset)\n",
    "test_preds = np.argmax(test_raw_preds, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ec9447b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1260"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e06f5cf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Premise      0.922     0.898     0.910       805\n",
      "  MajorClaim      0.741     0.974     0.842       153\n",
      "       Claim      0.702     0.639     0.669       302\n",
      "\n",
      "    accuracy                          0.845      1260\n",
      "   macro avg      0.788     0.837     0.807      1260\n",
      "weighted avg      0.847     0.845     0.844      1260\n",
      "\n"
     ]
    }
   ],
   "source": [
    "target_name = labels.int2str([0,1,2])\n",
    "print(classification_report(test_labels, test_preds, target_names=target_name, digits=3))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "bdb4e496",
   "metadata": {},
   "source": [
    "precision    recall  f1-score   support\n",
    "\n",
    "       Claim      0.704     0.662     0.683       302\n",
    "  MajorClaim      0.770     0.961     0.855       153\n",
    "     Premise      0.922     0.899     0.911       805\n",
    "\n",
    "    accuracy                          0.850      1260\n",
    "   macro avg      0.799     0.841     0.816      1260\n",
    "weighted avg      0.851     0.850     0.849      1260\n",
    "\n",
    "field used: strct_fts_and_component_pos, 1e-5, 48, 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d57cd1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "25e03036",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[723,   4,  78],\n",
       "       [  0, 149,   4],\n",
       "       [ 61,  48, 193]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_labels, test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6f845b66",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import ConfusionMatrixDisplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ee455308",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f56c0791eb0>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEGCAYAAADxD4m3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiE0lEQVR4nO3deZwV1Zn/8c/T3UDbgEALts2i4k8GJSqLKLgGxSRqZoLJS41LlJ/BIIlbnMREk5lojJMfZjO4xAkuCWZcokYjmcEtqJHk5wIigqhoi0I3NrLvsnT3M3/UabhAL/fS91L3Vn/fvurVVefWrXroFzyeU6fOOebuiIgkUVHcAYiI5IoSnIgklhKciCSWEpyIJJYSnIgkVkncAaTqWV7sB/frEHcYeeu9uWVxh5D3rFPHuEPIa59uW8fW+k3Wlmt84ZTOvnJVfVrnvj53yzPufnpb7tcWeZXgDu7Xgdee6Rd3GHnrC32Gxh1C3is+qH/cIeS1lxdNafM1Vqyq59Vn+qZ1bofKD3q2+YZtkFcJTkQKgVPvDXEHkRYlOBHJiAMNFMYAAXUyiEjGGtL8ryVmNtDM5qRs68zs22ZWbmbPmdn74WePcL6Z2W1mVmVmc81sWGtxKsGJSEYcZ5s3pLW1eB33Be4+xN2HAEcDm4AngOuA6e4+AJgejgHOAAaEbTxwV2uxKsGJSEYcqMfT2jIwGvjA3RcBY4DG3pApwFlhfwxwv0deAbqbWWVLF9UzOBHJWAbP4Hqa2ayU48nuPrmJ884DHgr7Fe5eG/aXAhVhvw9QnfKdmlBWSzOU4EQkIw7Upz8L0Qp3H97SCWbWEfgScP1u93J3M9vjHg01UUUkYw1pbmk6A5jt7p+E408am57h57JQvgRIfVG2byhrlhKciGTE03z+lsEzuPPZ0TwFmAqMDftjgSdTyi8OvakjgbUpTdkmqYkqIhlxh21Zeg3OzDoDnwMuSymeCDxiZuOARcC5oXwacCZQRdTjeklr11eCE5EMGfW0aTjrdu6+Edhvl7KVRL2qu57rwOWZXF8JTkQy4kBDYQxkUIITkcxlqwaXa0pwIpKR6EVfJTgRSSAHtnlhvIChBCciGXGM+gJ5w0wJTkQy1uBqoopIAukZnIgkmFGvZ3AikkTRjL5KcCKSQO7GVi+OO4y0KMGJSMYa9AxORJIo6mRQE1VEEkmdDCKSUOpkEJFEq9eLviKSRI6xzQsjdRRGlCKSN9TJICKJ5ZiaqCKSXOpkyHPVVZ346YSDtx8vXdyRi65dysraDrzy3L506OhUHrSF79xaTZdu9bz7RhmTro1WLHPgou8s5YQz1sYTfJ4oKnJuf+o9Vi7twI/GHhJ3OLHq0289190wc/txZe+N/OG+w5k3pydX/OscOnRsoKHeuPPWwbz3bnmMkbadO3pNBMDMTgcmAcXAPe4+MZf3y0S/Q7dw118XAFBfDxcO+wwnnLGGmqpSvv6DjykugXturuTh2/fn0n+r5eCBn3LH0wsoLoGVn5TwzdMGMvJzaylut/+LgLMuXU71+50o65rBCpgJtaS6K1deeioQJf77H3uKl2f05qpr3+DBKYcx69UDGD5iKV+fMJ/rvn1SzNG2TdTJUBhDtXKWhs2sGLiTaFHXQcD5ZjYoV/drizkzulJ50BYq+m7j6FHrtyetw4/exIraDgCUlvn28m1birDCeASRMz0rt3Ls6HU89dB+rZ/czgwetoylH3dm2SdluENZWR0AnbtsY9XK0pijy456itLaWmNm3c3sMTN718zeMbPjzKzczJ4zs/fDzx7hXDOz28ysyszmmtmw1q6fy3rmsUCVuy90963Aw8CYHN5vj734ZHdGnbVmt/JnHirnmFPXbz9+d3YZ3xg1kMtOHchVt9S069rbhB8v4Z6be+OqvO3ms6NreHF6XwAm33EkX//mW0x59GnGffMtfj/5MzFH13aO0eDpbWmYBDzt7ocBg4F3gOuA6e4+AJgejiGqLA0I23jgrtYunssE1weoTjmuCWV5ZdtW45Vnu3Hyv6zZqfzBSRUUlzinfmX19rLDhm3i7hcXcPtT7/Hw7fuzdXP7rMaNOG0ta1aUUDWvLO5Q8k5JSQMjjl/K31+M/qqfOeZD7r7jSMaeczp333kkV39vdswRZkc2anBm1g04GbgXwN23uvsaoorQlHDaFOCssD8GuN8jrwDdzayypXvE/qTQzMab2Swzm7V8Zf1ev//M57ty6JGb6NGrbnvZs38s57W/7sv371jUZFP0wAFb2KdzAx8tSEZzI1ODhm9k5OfXMeWV+Vz/m0UMPmE937ttUdxh5YXhI5bywfvdWbM6+rtx2hcW84+XegMw44U+DDx8dUtfLwjRuqhFaW1Az8Z/32Ebn3Kp/sBy4Hdm9oaZ3RNWuq9w99pwzlKgIuxnXGnKZSNrCdAv5bhvKNuJu08GJgMMH1y615eTffHPPXZqns58oSuP/mZ/fv74+5SW7Qhn6eKO9Oq9leIS+KSmA9VVpVT03bq3w80Lv5vYm99NjP7RHnXces6esJyfXXVQzFHlh8+OruFvoXkKsHJlKUcOWcG8Ob0YPGw5S2q6xBhdtmS0sv0Kdx/ezGclwDDgSnd/1cwmsaM5CkSr2ZvZHueFXCa4mcAAM+tPlNjOAy7I4f0ytnlTEbNndOXqn+34n8KdP+zLti3G9V89FIDDjt7I1bfU8NZrnfnjHf0pKYl6ya78aQ3d9tv7NU7JX51K6xg6fBm3/3Lo9rLbfj6Uy66cR3FxA9u2FnP7L4bEF2CWRMsGZqUXtQaocfdXw/FjRAnuEzOrdPfa0ARdFj5Pq9KUKmcJzt3rzOwK4Bmi10Tuc/f5ubrfnigta+Cx+W/tVPb7//9Ok+eedvZqTju78JsX2Tb35a7Mfblr3GHkhS2bSzjvS/+8U9nb83py9fhTYoooN9ytsfnZxuv4UjOrNrOB7r4AGA28HbaxwMTw88nwlanAFWb2MDACWJvSlG1STvsB3X0aMC2X9xCRvS+LL/peCTxgZh2BhcAlRH0Dj5jZOGARcG44dxpwJlAFbArntqgdv+ggInsimg8uO28QuPscoKlndKObONeByzO5vhKciGRIM/qKSEJFr4kUxjugSnAikpFCGouqBCciGdN0SSKSSNF0SWqiikhC6RmciCRSNJuImqgikkDRUC0lOBFJJNXgRCTBsjWSIdeU4EQkI+pFFZFEUxNVRBKpcU2GQqAEJyIZcaBONTgRSSo1UUUkmdJfEjB2SnAikpFsTniZa0pwIpIx1eBEJJE04aWIJJZj1DWok0FEEqpQnsEVRhoWkfzhURM1na01ZvaRmc0zszlmNiuUlZvZc2b2fvjZI5Sbmd1mZlVmNtfMhrV2fSU4EclI4zO4bCS44BR3H+LujcsHXgdMd/cBwPRwDHAGMCBs44G7WruwEpyIZCzLCW5XY4ApYX8KcFZK+f0eeQXobmaVLV1ICU5EMuIY9Q1FaW1ATzOblbKN3+1y8KyZvZ7yWYW714b9pUBF2O8DVKd8tyaUNUudDCKSsQw6GVakND2bcqK7LzGz/YHnzOzd1A/d3c3M9zROJTgRyYh79t6Dc/cl4ecyM3sCOBb4xMwq3b02NEGXhdOXAP1Svt43lDVLTVQRyZi7pbW1xMw6m1nXxn3g88BbwFRgbDhtLPBk2J8KXBx6U0cCa1Oask1SDU5EMpS1wfYVwBNmBlEuetDdnzazmcAjZjYOWAScG86fBpwJVAGbgEtau4ESnIhkrLXaWXrX8IXA4CbKVwKjmyh34PJM7pFXCe69uWV8ofeQuMPIW3WjW32vUZ6fHXcEec0btrb9Gg71DYUxkiGvEpyIFIZCGaqlBCciGXGy00TdG5TgRCRDmtFXRBLM9/jV271LCU5EMqYmqogkUtSLWhhjBJTgRCRjaqKKSGKpiSoiieS0Ps40XyjBiUjGCqSFqgQnIhlycA3VEpGkUhNVRBKr4HtRzex2Wmhqu/tVOYlIRPJaUsaiztprUYhI4XCg0BOcu09JPTazMnfflPuQRCTfFUoTtdXxFmZ2nJm9Dbwbjgeb2W9yHpmI5CnDG9Lb4pbOgLJfA18AVgK4+5vAyTmMSUTynae5xSytXlR3rw4LQzSqz004IpL3PBmdDI2qzex4wM2sA3A18E5uwxKRvJYHtbN0pNNEnUC0kk0f4GNgCBmubCMiSWNpbmlcyazYzN4ws/8Ox/3N7FUzqzKzP5pZx1DeKRxXhc8Pbu3arSY4d1/h7he6e4W793L3r4VlvUSkvWpIc0vPrq3CW4Bb3f1QYDUwLpSPA1aH8lvDeS1Kpxf1EDP7i5ktN7NlZvakmR2SdugikiyN78Gls7XCzPoCXwTuCccGnAo8Fk6ZApwV9seEY8Lno22XzoFdpdNEfRB4BKgEegOPAg+l8T0RSSj39LY0/Br4Hjvqe/sBa9y9LhzXED0eI/ysju7vdcDacH6z0klwZe7+B3evC9t/AaVphS4iyZT+ayI9zWxWyja+8RJm9s/AMnd/PVdhtjQWtTzsPmVm1wEPh5C/CkzLVUAiUgDSf01khbsPb+azE4AvmdmZRJWmfYFJQHczKwm1tL7AknD+EqAfUGNmJUA3wvu5zWnpNZHXiRJa45/kspTPHLi+pQuLSHJZFl4TcffrCXnEzEYB33X3C83sUeBsokrVWODJ8JWp4fjl8Pnz7i03hFsai9q/jfGLSBK5QW6HYX0feNjMbgbeAO4N5fcCfzCzKmAVcF5rF0prJIOZHQEMIuXZm7vfn2HQIpIUWX7R191fBF4M+wuBY5s4ZzNwTibXbTXBmdkNwCiiBDcNOAP4O6AEJ9JeJWgkw9nAaGCpu18CDCZ6uCci7VWCBtt/6u4NZlZnZvsCy4h6MhJr+Kh1TPjJxxQXOU89VM4jd1TEHVIsvnvpDEYOrWbNulIuvf4rO312zhnzmHDBTL78zQtYt6GULmVbuPYbM+i9/3q2bivm5/ecxEc1PWKKPD8UFTm3P/UeK5d24EdjE/RufAFNeJlODW6WmXUH7ibqWZ1N1IvRIjO7L4x8eKttIe5dRUXO5T9dwr9d2J9vjBrIKWPWcOCAzXGHFYtnZgzg+p99frfyXuUbOPqIj/lkReftZRd86U2qFu/HN374ZSb+9mQu/9orezPUvHTWpcupfr9T3GHkhHl6W9zSGYv6LXdf4+7/CXwOGBuaqq35PXB6G+Pb6wYO3cTHH3Vk6eJO1G0r4sUnu3PcF9bGHVYs5i04gHUbd/8H+q0LX2PyH4fvNGXOQX3WMGd+JQDVtd05oOcGeuz76V6LNd/0rNzKsaPX8dRDLb5oX7gKpInabIIzs2G7bkA5UBL2W+TuLxF15RaU/Q7YxvKPO24/XlHbgZ6V22KMKL8cP2wRK1aXsXDxzv9wFy4u58RjFgEw8JDlVPTcQM/yjXGEmBcm/HgJ99zcG09/wHlBKZQaXEvP4H7ZwmdONCC2zcLQjfEApZRl45KSI5061nHBl97k+7fsXjF/6C9HcflFr/Lbm//Mh9U9eH/RfjTkwZTVcRhx2lrWrCihal4ZRx23Pu5wcqNAnsG19KLvKXsjAHefDEwG2NfKY8/5K5d2oFfvrduPe1ZuY0Vthxgjyh+991/HAb02MPk//gxAr/KN/OdPnuTyG/+F1WvL+PndJ4UznQd+9Si1y7rGFmucBg3fyMjPr+OYU+fTsZNT1rWe7922iJ9ddVDcoWVHnjQ/06GFn3exYE4ZffpvpaLfFlYu7cCoMWuYeHlC/mK20Yc15Zx9+QXbjx/41SN880dfYt2GUjqXbWHLlhLq6os5c9R7zF1QwabNHVu4WnL9bmJvfjexNwBHHbeesycsT05ya6QEV5ga6o07f9iHnz64kKJiePbhcha91z4nT/nht15g8OFL6dZlMw9Pepgpjw/jqb/9U5PnHtR7Ld8f/xIOfFTTg1/cc+LeDVb2KiuQZ4vWyljVPb+w2UNEIyB6Ap8AN7j7vS19Z18r9xE2OifxJEHd6KPjDiHvlTw/O+4Q8tqrDX9lna9q0wO0Tv36ed+rr0nr3IXXfuf1FmYTybl0hmoZcCFwiLvfZGYHAge4+2stfc/dz89SjCKSR/KlhzQd6bzo+xvgOKAxYa0H7sxZRCKS/7I0ZXmupfMMboS7DzOzNwDcfXXjKjci0k4VSA0unQS3zcyKCX8kM+tFJuvliEjiFEoTNZ0EdxvwBLC/mf0H0ewi/5bTqEQkf3nh9KK2muDc/QEze51oyiQDznJ3rWwv0p4lpQYXek03AX9JLXP3xbkMTETyWFISHPA/7Fh8phToDywAPpPDuEQkjyXmGZy7H5l6HGYS+VbOIhIRyZKMh2q5+2wzG5GLYESkQCSlBmdm/5pyWAQMAz7OWUQikt+y1ItqZqXAS0Anolz0mLvfYGb9idZE3Y9oFvGL3H2rmXUiWuzqaKIFn7/q7h+1dI90RjJ0Tdk6ET2TG7NHfyIRSYbszOi7BTjV3QcDQ4DTzWwkcAtwq7sfCqwGxoXzxwGrQ/mt4bwWtViDCy/4dnX377Yaqoi0C0bWVrZ3YEM47BC2xsl0G+flmgLcCNxFVLG6MZQ/BtxhZtbS6vYtTVle4u71wAl7/kcQkUTK0poMZlZsZnOIVut7DvgAWOPudeGUGqBP2O8DVAOEz9cSNWOb1VIN7jWi521zzGwq8CiwfZJ9d3+89fBFJHEym02kp5nNSjmeHGbxji4VVaKGhJX7ngAOy1aYkF4vainRA71T2fE+nANKcCLtVfqdDCvSmQ/O3deY2QtEMxd1Dy3IOqAvsCSctoRoTeYaMyshWoB+ZUvXbSnB7R96UN9iR2LbHk9rAYtIcmXjGVyYuGNbSG77EC1LegvwAtGY94eBscCT4StTw/HL4fPnW3r+Bi0nuGKgCzsntkZKcCLtWXYyQCUwJXRmFgGPuPt/m9nbwMNmdjPwBtA4E/i9wB/MrIpoSdLzWrtBSwmu1t1valP4IpI8WVpVy93nAkObKF8IHNtE+WbgnEzu0VKCi386ThHJS0kYi6rVX0SkaYWe4Nx91d4MREQKR2ImvBQR2YlWtheRpDIK5wG9EpyIZE41OBFJqiT0ooqINE0JTkQSKUnLBoqI7EY1OBFJKj2DE5HkUoLLnHXsQMkBfeMOI28Vv1Uddwh5b+3Zu43RlhQNz72cleuoBiciyeRkMuFlrJTgRCQj2Vp0Zm9QghORzCnBiUhSWcszhecNJTgRyYxmExGRJNMzOBFJLA3VEpHkKpAaXFHcAYhIgQkr26eztcTM+pnZC2b2tpnNN7OrQ3m5mT1nZu+Hnz1CuZnZbWZWZWZzzWxYa6EqwYlI5jzNrWV1wHfcfRAwErjczAYB1wHT3X0AMD0cA5wBDAjbeOCu1m6gBCciGWl80betNTh3r3X32WF/PfAO0AcYA0wJp00Bzgr7Y4D7PfIK0N3MKlu6h57BiUjGrCG7D+HM7GCiRaBfBSrcvTZ8tBSoCPt9gNQB2TWhrJZmKMGJSGYyew+up5nNSjme7O6TU08wsy7An4Bvu/s6sx1L2ri7m+35SylKcCKSsQxeE1nh7sObvY5ZB6Lk9oC7Px6KPzGzSnevDU3QZaF8CdAv5et9Q1mz9AxORDKXhU4Gi6pq9wLvuPuvUj6aCowN+2OBJ1PKLw69qSOBtSlN2SapBiciGcvSSIYTgIuAeWY2J5T9AJgIPGJm44BFwLnhs2nAmUAVsAm4pLUbKMGJSGYcyMJge3f/O82vIT26ifMduDyTeyjBiUjGNFRLRBJJE16KSHK5Z6WJujcowYlIxlSDE5HkUoITkaRSDU5EksmB+sLIcEpwIpIx1eBEJLnUiyoiSaUanIgkk5YNFJGkMsDUySAiSaWV7UUkmdRELTydu2zjqh/O5aBD1oPDr28eTM/9N3PBN96j38EbuOaSE6h6t3vcYcaqqMiZ9MCrrFzWiRuvHsrgY1cy7tvvY0XO5k0l/OqGz1BbXRZ3mHvN9ee/yAmfWczqDftw0cRzADi090quPXcG+3TaRu2qrvz4/lPZtKUjhx+4jO9/dUb0RXPue/poXprbP8bo20JjUTGzfsD9RAtGONFc7JNydb+2Gv+v83n95V78v+uPpqSkgU6l9WzcUMJ/fP9orrhuXtzh5YUxFyym+sPOlHWuA+CKH7zLTdcMpvrDLnzxnGrOu3Qht95wRMxR7j3TXhvIn2Ycwb9/7YXtZded/xJ3/HkEcz7ozRdHvMuFo9/k7mnHsLC2nHG//DL1DUXst+8mpnzvMf7x1kHUNxTmpNqF0ouay99uc2se5p2yzts4Yugqnp0aTfdeV1fExg0dqP6oK0sWd4k5uvyw3/6bOebEFTzzRJ/tZe5Q1rkegM5d61i1vFNc4cXizQ8qWbdp5z9zv15rmPNBtJLdzAV9+ezgDwHYsq1kezLrWFKHNzvPY4FonFGktS1mOavBhbnSa8P+ejNrXPPw7Vzdc08d0HsTa1d35Jp/n0v/Aeuoercbv/3VILZsVgu+0WXXLuC+SQPYp6xue9mkmwbx49vfYOuWIjZtLOGai4+NMcL88OHSck46chEz5h3MKUMWUtF94/bPBh20jB+c/zcqytfzk/86pWBrb3jh9KLuld/wLmse5p2iYufQgeuY9viBXHXxSWzeXMw5Yz+IO6y8cexJy1mzqiNV7+y7U/lZFy7mhiuHcvHpJ/Pck70Z/50FMUWYP3764Gf5yonzufe7j1NWuo1t9Tv+ib29aH++NvEcLv3ll7notDl0LKlr4Up5Ljsr2+dczqsou6552MTn44HxAKXFXXMdTpNWLitlxbJSFszvAcA/nq/knIurYoklHw0asoaRn13OMSeuoEPHBso613HjbW/Q7+CNLHirGwAvPVvBT+58I+ZI47d4WXeuueuLQNRcPX7Q4t3OWfRJDz7d0oFDKlfzbnWvvR1iVhTKayI5rcE1s+bhTtx9srsPd/fhHYv3yWU4zVq9qpTly0rpc+AGAAYPX8HiD+NJtvno97cP4OLTT+aSL57ELdcdydyZ5dx0zWDKutTR58CoCTZ05CqqP+wcc6Tx697lUwDMnLGff4M//+NwACrL11FcFC1kUNFjPQdVrKF2VQH/HWvvz+BaWPMwL/32F5/h2pvmUFLSwNKPy/j1TwZz3GeXMuG78+nWfSs33jqThe/ty4+uHhF3qHmhob6I234yiB/+Yi4NDhvWdeDXN+ZlH1LO3HjxdIYe+jHdu2zmiR8/wL1PHc0+nbbxlROjx8x/m3sw//PqQACOOmQpF532JnX1RTQ4/OLRE1m7sTTO8PecAwWy6Ix5jrKsmZ0IzADmsePX8QN3n9bcd7p1qvDjD7ggJ/EkgW/dGncIeW/tyYX6btneMe+5SWxYVd2mLtxunXv7yEGXpXXus7NufL2Vle3vA/4ZWObuR4SycuCPwMHAR8C57r46VJomEa2Nugn4v+4+u6X756yJ6u5/d3dz96PcfUjYmk1uIlJAGhrS21r3e+D0XcquA6a7+wBgejgGOAMYELbxwF2tXbxA+6lFJDaNTdR0ttYu5f4SsGqX4jHAlLA/BTgrpfx+j7wCdDezypaurxe9RCRjGfSi9jSzWSnHk919civfqQjv0QIsJRoNBdF7tNUp59WEslqaoQQnIplLP8GtaOkZXOu3cTfb84FhaqKKSIbSfEVkzzswP2lseoafy0L5EqBfynl9Q1mzlOBEJDONq2qls+2ZqcDYsD8WeDKl/GKLjATWpjRlm6QmqohkLFsjGczsIWAU0bO6GuAGYCLwiJmNAxYB54bTpxG9IlJF9JrIJa1dXwlORDKXpQTn7uc389HoJs514PJMrq8EJyKZcaAh/mFY6VCCE5EM5cc403QowYlI5pTgRCSRHKgvjNH2SnAikiEHV4ITkaRSE1VEEkm9qCKSaKrBiUhiKcGJSCK5Q3193FGkRQlORDKnGpyIJJYSnIgkk6sXVUQSysH1oq+IJJaGaolIIrmnuyRg7JTgRCRz6mQQkaRy1eBEJJk04aWIJJUG24tIUjngBTJUS+uiikhmPEx4mc7WCjM73cwWmFmVmV2X7VBVgxORjHkWmqhmVgzcCXwOqAFmmtlUd3+7zRcPVIMTkcxlpwZ3LFDl7gvdfSvwMDAmm2Ga51FviJktJ1rJOl/0BFbEHUQe0++ndfn2OzrI3Xu15QJm9jTRnysdpcDmlOPJ7j45XOds4HR3vzQcXwSMcPcr2hJfqrxqorb1F59tZjbL3YfHHUe+0u+ndUn8Hbn76XHHkC41UUUkLkuAfinHfUNZ1ijBiUhcZgIDzKy/mXUEzgOmZvMGedVEzUOT4w4gz+n30zr9jprh7nVmdgXwDFAM3Ofu87N5j7zqZBARySY1UUUksZTgRCSxlOCakOvhI4XOzO4zs2Vm9lbcseQjM+tnZi+Y2dtmNt/Mro47pvZKz+B2EYaPvEfK8BHg/GwOHyl0ZnYysAG4392PiDuefGNmlUClu882s67A68BZ+ju096kGt7ucDx8pdO7+ErAq7jjylbvXuvvssL8eeAfoE29U7ZMS3O76ANUpxzXoL6fsITM7GBgKvBpzKO2SEpxIjphZF+BPwLfdfV3c8bRHSnC7y/nwEUk+M+tAlNwecPfH446nvVKC213Oh49IspmZAfcC77j7r+KOpz1TgtuFu9cBjcNH3gEeyfbwkUJnZg8BLwMDzazGzMbFHVOeOQG4CDjVzOaE7cy4g2qP9JqIiCSWanAiklhKcCKSWEpwIpJYSnAiklhKcCKSWEpwBcTM6sMrB2+Z2aNmVtaGa/0+rGqEmd1jZoNaOHeUmR2/B/f4yMx2W32pufJdztmQ4b1uNLPvZhqjJJsSXGH51N2HhBk8tgITUj80sz2agt7dL21lpotRQMYJTiRuSnCFawZwaKhdzTCzqcDbZlZsZj83s5lmNtfMLoPo7XozuyPMc/dXYP/GC5nZi2Y2POyfbmazzexNM5seBotPAK4JtceTzKyXmf0p3GOmmZ0QvrufmT0b5kC7B7DW/hBm9mczez18Z/wun90ayqebWa9Q9n/M7OnwnRlmdlhWfpuSSFp0pgCFmtoZwNOhaBhwhLt/GJLEWnc/xsw6Af8ws2eJZrQYCAwCKoC3gft2uW4v4G7g5HCtcndfZWb/CWxw91+E8x4EbnX3v5vZgUSjPg4HbgD+7u43mdkXgXRGOHw93GMfYKaZ/cndVwKdgVnufo2Z/Shc+wqiRVwmuPv7ZjYC+A1w6h78GqUdUIIrLPuY2ZywP4NovOPxwGvu/mEo/zxwVOPzNaAbMAA4GXjI3euBj83s+SauPxJ4qfFa7t7cnG+nAYOiIZcA7BtmzjgZ+Er47v+Y2eo0/kxXmdmXw36/EOtKoAH4Yyj/L+DxcI/jgUdT7t0pjXtIO6UEV1g+dfchqQXhH/rG1CLgSnd/ZpfzsjkWsggY6e6bm4glbWY2iihZHufum8zsRaC0mdM93HfNrr8DkeboGVzyPAN8M0zXg5n9k5l1Bl4Cvhqe0VUCpzTx3VeAk82sf/hueShfD3RNOe9Z4MrGAzMbEnZfAi4IZWcAPVqJtRuwOiS3w4hqkI2KgMZa6AVETd91wIdmdk64h5nZ4FbuIe2YElzy3EP0fG22RYvC/Jaopv4E8H747H6i2UB24u7LgfFEzcE32dFE/Avw5cZOBuAqYHjoxHibHb25PyZKkPOJmqqLW4n1aaDEzN4BJhIl2EYbgWPDn+FU4KZQfiEwLsQ3H00nLy3QbCIikliqwYlIYinBiUhiKcGJSGIpwYlIYinBiUhiKcGJSGIpwYlIYv0v9EV4WpWV7f4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ConfusionMatrixDisplay.from_predictions(test_labels, test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "17ac8e72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 2, ..., 2, 0, 0])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "37fd2c0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 2, ..., 0, 2, 0])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_preds"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}