cassava-validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# About this notebook  \n",
    "\n",
    "TBD...\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Directory settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Directory settings\n",
    "# ====================================================\n",
    "import os\n",
    "\n",
    "OUTPUT_DIR = \"./\"\n",
    "MODEL_DIR = \"../input/cassava-model/\"\n",
    "if not os.path.exists(OUTPUT_DIR):\n",
    "    os.makedirs(OUTPUT_DIR)\n",
    "\n",
    "TRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n",
    "TEST_PATH = \"../input/cassava-leaf-disease-merged/train\"\n",
    "# TEST_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n",
    "# TEST_PATH = \"../input/cassava-leaf-disease-classification/test_images\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "# CFG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# CFG\n",
    "# ====================================================\n",
    "class CFG:\n",
    "    debug = False\n",
    "    num_workers = 4\n",
    "    models = [\n",
    "        # \"tf_efficientnet_b3_ns\",\n",
    "        \"tf_efficientnet_b4_ns\",\n",
    "        \"vit_base_patch16_384\",\n",
    "        # \"deit_base_patch16_384\",\n",
    "        \"seresnext50_32x4d\",\n",
    "    ]\n",
    "    size = {\n",
    "        \"tf_efficientnet_b3_ns\": 512,\n",
    "        \"tf_efficientnet_b4_ns\": 512,\n",
    "        \"vit_base_patch16_384\": 384,\n",
    "        \"deit_base_patch16_384\": 384,\n",
    "        \"seresnext50_32x4d\": 512,\n",
    "    }\n",
    "    batch_size = 32\n",
    "    seed = 22\n",
    "    target_size = 5\n",
    "    target_col = \"label\"\n",
    "    n_fold = 5\n",
    "    trn_fold = {  # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "        \"tf_efficientnet_b3_ns\": {\n",
    "            \"best\": [0, 1, 2, 3, 4],\n",
    "            \"final\": [],\n",
    "        },\n",
    "        \"tf_efficientnet_b4_ns\": {\n",
    "            \"best\": [0, 1, 2, 3, 4],\n",
    "            \"final\": [],\n",
    "        },\n",
    "        \"vit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n",
    "        \"deit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n",
    "        \"seresnext50_32x4d\": {\"best\": [5, 6, 7, 8, 9], \"final\": []},\n",
    "    }\n",
    "    data_parallel = {\n",
    "        \"tf_efficientnet_b3_ns\": False,\n",
    "        \"tf_efficientnet_b4_ns\": True,\n",
    "        \"vit_base_patch16_384\": False,\n",
    "        \"deit_base_patch16_384\": False,\n",
    "        \"seresnext50_32x4d\": False,\n",
    "    }\n",
    "    weight = {\n",
    "        # \"tf_efficientnet_b3_ns\": None,\n",
    "        \"tf_efficientnet_b4_ns\": 1,\n",
    "        \"vit_base_patch16_384\": 1,\n",
    "        # \"deit_base_patch16_384\": None,\n",
    "        \"seresnext50_32x4d\": 1,\n",
    "    }\n",
    "    tta = 3  # 1: no TTA, >1: TTA\n",
    "    train = False\n",
    "    inference = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_sum = sum([CFG.weight[model] for model in CFG.models]) * CFG.tta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import sys\n",
    "\n",
    "sys.path.append(\"../input/pytorch-image-models/pytorch-image-models-master\")\n",
    "\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import time\n",
    "import warnings\n",
    "from collections import Counter, defaultdict\n",
    "from contextlib import contextmanager\n",
    "from functools import partial\n",
    "from pathlib import Path\n",
    "\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import seaborn as sns\n",
    "import timm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "from albumentations import (\n",
    "    CenterCrop,\n",
    "    CoarseDropout,\n",
    "    Compose,\n",
    "    Cutout,\n",
    "    HorizontalFlip,\n",
    "    HueSaturationValue,\n",
    "    IAAAdditiveGaussianNoise,\n",
    "    ImageOnlyTransform,\n",
    "    Normalize,\n",
    "    OneOf,\n",
    "    RandomBrightness,\n",
    "    RandomBrightnessContrast,\n",
    "    RandomContrast,\n",
    "    RandomCrop,\n",
    "    RandomResizedCrop,\n",
    "    Resize,\n",
    "    Rotate,\n",
    "    ShiftScaleRotate,\n",
    "    Transpose,\n",
    "    VerticalFlip,\n",
    ")\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "from PIL import Image\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from torch.nn.parameter import Parameter\n",
    "from torch.optim import SGD, Adam\n",
    "from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Utils\n",
    "# ====================================================\n",
    "def get_score(y_true, y_pred):\n",
    "    return accuracy_score(y_true, y_pred)\n",
    "\n",
    "\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    LOGGER.info(f\"[{name}] start\")\n",
    "    yield\n",
    "    LOGGER.info(f\"[{name}] done in {time.time() - t0:.0f} s.\")\n",
    "\n",
    "\n",
    "def init_logger(log_file=OUTPUT_DIR + \"inference.log\"):\n",
    "    from logging import INFO, FileHandler, Formatter, StreamHandler, getLogger\n",
    "\n",
    "    logger = getLogger(__name__)\n",
    "    logger.setLevel(INFO)\n",
    "    handler1 = StreamHandler()\n",
    "    handler1.setFormatter(Formatter(\"%(message)s\"))\n",
    "    handler2 = FileHandler(filename=log_file)\n",
    "    handler2.setFormatter(Formatter(\"%(message)s\"))\n",
    "    logger.addHandler(handler1)\n",
    "    logger.addHandler(handler2)\n",
    "    return logger\n",
    "\n",
    "\n",
    "LOGGER = init_logger()\n",
    "\n",
    "\n",
    "def seed_torch(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "\n",
    "seed_torch(seed=CFG.seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000015157.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000201771.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100042118.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000723321.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000812911.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_id  label  source\n",
       "0  1000015157.jpg      0    2020\n",
       "1  1000201771.jpg      3    2020\n",
       "2   100042118.jpg      1    2020\n",
       "3  1000723321.jpg      1    2020\n",
       "4  1000812911.jpg      3    2020"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n",
    "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n",
    "test = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Dataset\n",
    "# ====================================================\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self, df, transform=None):\n",
    "        self.df = df\n",
    "        self.file_names = df[\"image_id\"].values\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        file_name = self.file_names[idx]\n",
    "        file_path = f\"{TEST_PATH}/{file_name}\"\n",
    "        image = cv2.imread(file_path)\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "        if self.transform:\n",
    "            augmented = self.transform(image=image)\n",
    "            image = augmented[\"image\"]\n",
    "        return image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Transforms\n",
    "# ====================================================\n",
    "def get_transforms(*, data, size):\n",
    "\n",
    "    if data == \"train\":\n",
    "        return Compose(\n",
    "            [\n",
    "                # Resize(size, size),\n",
    "                RandomResizedCrop(size, size),\n",
    "                Transpose(p=0.5),\n",
    "                HorizontalFlip(p=0.5),\n",
    "                VerticalFlip(p=0.5),\n",
    "                ShiftScaleRotate(p=0.5),\n",
    "                HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n",
    "                RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n",
    "                CoarseDropout(p=0.5),\n",
    "                Cutout(p=0.5),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    if data == \"valid\":\n",
    "        return Compose(\n",
    "            [\n",
    "                Resize(size, size),\n",
    "                CenterCrop(size, size),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    if data == \"inference\":\n",
    "        return Compose(\n",
    "            [\n",
    "                # Resize(size, size),\n",
    "                RandomResizedCrop(size, size),\n",
    "                Transpose(p=0.5),\n",
    "                HorizontalFlip(p=0.5),\n",
    "                VerticalFlip(p=0.5),\n",
    "                # ShiftScaleRotate(p=0.5),\n",
    "                # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n",
    "                # RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n",
    "                # CoarseDropout(p=0.5),\n",
    "                # Cutout(p=0.5),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# MODEL\n",
    "# ====================================================\n",
    "class CassvaImgClassifier(nn.Module):\n",
    "    def __init__(self, model_name=\"resnext50_32x4d\", pretrained=False):\n",
    "        super().__init__()\n",
    "\n",
    "        if model_name == \"deit_base_patch16_384\":\n",
    "            # self.model = torch.hub.load(\"facebookresearch/deit:main\", model_name, pretrained=pretrained)\n",
    "            self.model = torch.hub.load(\"../input/fair-deit\", model_name, pretrained=pretrained, source=\"local\")\n",
    "            n_features = self.model.head.in_features\n",
    "            self.model.head = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "        else:\n",
    "            self.model = timm.create_model(model_name, pretrained=pretrained)\n",
    "\n",
    "            if \"resnext50_32x4d\" in model_name:\n",
    "                n_features = self.model.fc.in_features\n",
    "                self.model.fc = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "            elif model_name.startswith(\"tf_efficientnet\"):\n",
    "                n_features = self.model.classifier.in_features\n",
    "                self.model.classifier = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "            elif model_name.startswith(\"vit_\"):\n",
    "                n_features = self.model.head.in_features\n",
    "                self.model.head = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Helper functions\n",
    "# ====================================================\n",
    "def inference(model, states, test_loader, device, data_parallel):\n",
    "    model.to(device)\n",
    "\n",
    "    # Use multi GPU\n",
    "    if device == torch.device(\"cuda\") and data_parallel:\n",
    "        model = torch.nn.DataParallel(model)  # make parallel\n",
    "        # torch.backends.cudnn.benchmark=True\n",
    "\n",
    "    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n",
    "    probs = []\n",
    "    for i, (images) in tk0:\n",
    "        images = images.to(device)\n",
    "        avg_preds = []\n",
    "        for state in states:\n",
    "            model.load_state_dict(state[\"model\"])\n",
    "            model.eval()\n",
    "            with torch.no_grad():\n",
    "                y_preds = model(images)\n",
    "            avg_preds.append(y_preds.softmax(1).to(\"cpu\").numpy())\n",
    "        avg_preds = np.mean(avg_preds, axis=0)\n",
    "        probs.append(avg_preds)\n",
    "    probs = np.concatenate(probs)\n",
    "    return probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: tf_efficientnet_b4_ns, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58d55b8ca5c846149e456b41c315cd19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.4113303  0.16465071 0.19031808 0.01710353 0.21659732]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: tf_efficientnet_b4_ns, TTA: 1, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbf9dc5086cf41b38b11a0b58600cb93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.48532075 0.15037784 0.23644516 0.02698353 0.10087272]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: tf_efficientnet_b4_ns, TTA: 2, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9adb6555ccaa422ab6c6f309984c5008",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.3294753  0.21423021 0.29620296 0.02863643 0.1314551 ]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: vit_base_patch16_384, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62e5739f4406404c9c70105f4e18984d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.44151726 0.12938017 0.24077013 0.04045077 0.14788169]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: vit_base_patch16_384, TTA: 1, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b99e0794c77545aa8584ed31c106770e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.42230922 0.08196902 0.32993802 0.05142972 0.11435406]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: vit_base_patch16_384, TTA: 2, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e96780b70774c02b8c6e7ce4b2d7e45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.32653087 0.31568345 0.15996166 0.05122393 0.14660004]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: seresnext50_32x4d, TTA: 0, Saved: best, Fold: [5, 6, 7, 8, 9] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3151a607aa2d453980e1080562dab3eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.2990355  0.3897496  0.09844354 0.02557188 0.1871995 ]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: seresnext50_32x4d, TTA: 1, Saved: best, Fold: [5, 6, 7, 8, 9] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b2ec540c81943448e80123288890328",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.26858085 0.20290053 0.23426536 0.03780187 0.2564514 ]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: seresnext50_32x4d, TTA: 2, Saved: best, Fold: [5, 6, 7, 8, 9] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1b16d7129d444fcb2a87af63c5ecfb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.2038373  0.23594086 0.16766731 0.07201287 0.32054168]\n",
      "========== Overall ==========\n",
      "Submission example: [0.35421526 0.20943135 0.21711247 0.03902384 0.18021706]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000015157.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000201771.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100042118.jpg</td>\n",
       "      <td>4</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000723321.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000812911.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_id  label  source\n",
       "0  1000015157.jpg      0    2020\n",
       "1  1000201771.jpg      3    2020\n",
       "2   100042118.jpg      4    2020\n",
       "3  1000723321.jpg      1    2020\n",
       "4  1000812911.jpg      3    2020"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# inference\n",
    "# ====================================================\n",
    "predictions = None\n",
    "for model_name in CFG.models:\n",
    "    for i in range(CFG.tta):\n",
    "        model = CassvaImgClassifier(model_name, pretrained=False)\n",
    "        states = []\n",
    "        for saved_model in [\"best\", \"final\"]:\n",
    "            if CFG.trn_fold[model_name][saved_model] != []:\n",
    "                LOGGER.info(\n",
    "                    f\"========== Model: {model_name}, TTA: {i}, Saved: {saved_model}, Fold: {CFG.trn_fold[model_name][saved_model]} ==========\"\n",
    "                )\n",
    "                states += [\n",
    "                    torch.load(MODEL_DIR + f\"{model_name}_fold{fold}_{saved_model}.pth\")\n",
    "                    for fold in CFG.trn_fold[model_name][saved_model]\n",
    "                ]\n",
    "\n",
    "        if i == 0:  # no TTA\n",
    "            test_dataset = TestDataset(test, transform=get_transforms(data=\"valid\", size=CFG.size[model_name]))\n",
    "        else:\n",
    "            test_dataset = TestDataset(test, transform=get_transforms(data=\"inference\", size=CFG.size[model_name]))\n",
    "\n",
    "        test_loader = DataLoader(\n",
    "            test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True\n",
    "        )\n",
    "\n",
    "        inf = inference(model, states, test_loader, device, CFG.data_parallel[model_name])\n",
    "\n",
    "        LOGGER.info(f\"Inference example: {inf[0]}\")\n",
    "\n",
    "        if predictions is None:\n",
    "            predictions = inf[np.newaxis] * CFG.weight[model_name]\n",
    "        else:\n",
    "            predictions = np.append(predictions, inf[np.newaxis] * CFG.weight[model_name], axis=0)\n",
    "\n",
    "sub = np.sum(predictions, axis=0) / weight_sum\n",
    "LOGGER.info(f\"========== Overall ==========\")\n",
    "LOGGER.info(f\"Submission example: {sub[0]}\")\n",
    "np.savetxt(f\"{OUTPUT_DIR}/predictions.csv\", sub, delimiter=\",\")\n",
    "\n",
    "# submission\n",
    "test[\"label\"] = sub.argmax(1)\n",
    "test[[\"image_id\", \"label\"]].to_csv(OUTPUT_DIR + \"submission.csv\", index=False)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n",
    "train = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = confusion_matrix(train[\"label\"], test[\"label\"])\n",
    "cm = cm / cm.astype(np.float).sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD4CAYAAABPLjVeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5mElEQVR4nO3dd3wUVdfA8d9Noyeh7QZIUCEoVUUBC4+kQBolhSYgWBGxAiIgoqJIkQ5KURAsjxWUEkggIKFLfSwUsQAiJJANXZok2dz3jw0hmw5kC/uer5/5mNm5M3sOMzm5e2d2RmmtEUIIYR9ujg5ACCH+P5GiK4QQdiRFVwgh7EiKrhBC2JEUXSGEsCMPW7+B4ckFLnd5xF8fdHN0CDbhppSjQyhzLpgSAP9mmh0dgk34VnC/4T1WofkLpa45l36aYfcjRHq6QghhRzbv6QohhF0p5+5LStEVQrgWN3dHR1AsKbpCCNfi5AP5UnSFEK5FhheEEMKOpKcrhBB2JD1dIYSwI+npCiGEHcnVC0IIYUcyvCCEEHYkwwtCCGFH0tMVQgg7kqIrhBB25C4n0oQQwn5kTFcIIexIhheEEMKOnLyn69x/EooQ0tSPH8ZGsm1cFC+2b1hgeZ1qFVk0JJg1I8NY93Y4bZv5OSDKwm3etJG4TpFEtw/n44/mFFiekZHBsFcGEd0+nEd7dedoagoAW3/YTK/uneke14le3TuzfdvW3HVmvDeVqHbBtG51j93yKM7mTRuI6RhBp6gw5heR49DBA+kUFUbvnt1IzcnxzJnT9H2iDw+0bM64MaPsHTYAmzduILpDBB0jw5g3t/DYhwweSMfIMB7pcTV2gHlzP6RjZBjRHSLYvGmj1Xpms5nuXWJ54blncl/76ovP6RgZxl1N7uD06VO2SyqfLZs30i2mPV06RfDp/LkFlmdkZDBi6Mt06RTBk70f5mhqqtXytGNHCX7gXj7/dH7ua7FR7ejVNYbe3eN4rJeDn6yi3Eo/OcBNV3TdlGJ873voOXUj/3k9ic731eX22t5WbQZ1akT8jiO0fXs1/T7cyvg+9zooWmtms5nxY0bx/qy5fLd0OStXJHDwwH6rNksWfYu3tzfxiat4pM9jTJ86GQDfqlWZPmM2CxYvY9SYd3njtaG567QJCuGzrxbYNZeimM1mxo0exczZH7EoPoGVics5kC/HxYsW4u3tzbIVq+nd53GmT5kEQDmvcjz/4gBefmVoYZu2ObPZzNgxo5j1wUcsvhL7/nyxf2eJffnK1fR+9HGm5cR+YP9+ViYmsCg+gVkffsTY0W9jNl99pM4X//2MevXqW23r7nvu4cN5H1O7dh3bJ5fDbDYzcdxops38kK8XLWPVysQCx2D84u+o4u3Nd8uS6NH7MWZOn2y1fNrkCTzQ+qEC25419xM+X7CYT79caNMcSqRU6ScHKLHoKqUaKqWGKaXey5mGKaUa2SO4wtxTrxp/pZ/n7+MXyDRns3jbYSLvrm3dSEOVCp4AeFfwxHTmkgMiLWjP7l34162Lf0AAnp5eRES1Z93aNVZt1q1dQ8foWADahkWwY9sWtNY0bNSYmgYjAPUDG3D538tkZGQAcOddd1OzpsGuuRRlz+5dBNS9JU+OHViXnC/H5GQ6xcQB0C48gu05OVaoWJHm97TAq1w5R4RuiT0gJ3YvLyLbdyiwf9YmJxOdE3tYeATbt1piX7d2DZHtO+Dl5YW/fwABAbewZ/cuAExpaWzcsI64Ll2tttWoUWPq1PG3T3I5ft2zG/+AutTxt+yfsIgoNqxLtmqzYV0yHTrFAhDaLpwd27eiteWxY+uTv6d27TrUqx9o17iviZt76SdHhFfcQqXUMOBrQAHbcyYFfKWUetX24RXk51uB1FMXc+ePnb5EraoVrNpMWLqXLg/U5edJHfly4EMM/+Ine4dZqOPpJvz8auXOG4x+pJtM+dqk57bx8PCgcuUqnDlzxqrNmtVJNGzUGC8vL5vHfK3S0034+V0dzjEajaSnmwppkz/H03aNszDpJhN+ta7GbjAaMZlKiL2KJXaTyYQxb95+xtx9O+HdsQwaPAQ3N8d/sExPt47TYPTjeHq6VZvj6SYMOW2u7J+zZ85w8eIFPvtkHn37P1dww0rx0rN9ebRnVxZ/6+BPXU4+vFDSibSngCZa68y8LyqlpgB7gXcLW0kp1Q/oB1D5waepcEe7Mgi19DrfV5dvNh9idtIftKhfnZlPt6LNG0loF3gu8YH9f/Le1MnMnDPP0aGIUli/bi3VqlWjcZOm7Ni+zdHh3JC5H8yk5yOPUrFipQLL5nz8OQajkVOnTvJi/77cels9mt/bwgFRctOfSMsGahfyeq2cZYXSWs/RWrfQWrco64KbduYSdapVvBpI1QocO209fNDrodtYuv0IADsPnKS8pzvVKzvmI2teNQ1G0tKO5c6nm9IwGI352hhy22RlZXH+/Dl8fX0By8fUwQNfYNTY8QQE1LVb3NfCYDCSlpaWO28ymTAYjIW0yZ9jVbvGWRiD0Ujasauxp5tMGI0lxH7OErvRaMSUN+80EwajkZ9/+pF165KJCgtl2Csvs2PbVoYPe8U+CRXCYLCOM92URk2D9dBUTYOR9Jw2V/aPj68ve3fvYsa0ycRGtePrL/7Lp/PmsPDrLyzbzfl3qlatOsEhbdm7Z5edMiqEk/d0S3rXgcAapdQKpdScnGklsAYYYPPoCvHTX6eoZ6xM3RqV8HR3I+6+uiT9fNSqTeqpizzU2HIQNKhVhXKe7pw4d9kR4Vpp0rQZR/7+m9SUFDIzM0hakUhQcKhVm6DgUJbHLwEswwgtW92PUopz//zDS88/w4sDB3N3c+e4SqEwTZo24/DhQ6SmHMnJMYGgkHw5hoSybOliAL5flUTL+yw5OtqV2FNSjpCZkcHKxIKxB4eEEp8T++pVSbTKiT0oJJSViQlkZGSQknKEw4cP0bTZnQwYNJjVyRtYsTqZ8ZOm0PK++xk3fpIj0gOgUZOmHDn8N0dTLcfg6qQVtAkKsWrzUFAICcuWAJD8/SpatLwPpRRzPv6cJSu+Z8mK7+nxSB8ee6of3Xo8wqVLF7lw4QIAly5dZNuWH6gf2MDeqV3l5EW32OEFrfVKpdTtQCvgyinWVGCH1tpc9Jq2Y87WvPr5j3zzchvc3RRfbvqL34/+w7DYJvx86DRJPx9l5De/MOWxFvQPvx2tNS/N2+6IUAvw8PBg2Gtv8Hz/p8g2ZxMd14X6gQ2YPeM9GjdpSlBIKLGdu/LG8KFEtw/Hx8eHcROmAPDNV19w5Mhh5n4wi7kfzAJg1ofzqFa9OtOmTGRlwnL+/fcSkW2DiO3Slf7PveiwHF997U2efaYv2WYzMXFdCAxswKwZ02ncpCnBIW2J69yVEcOH0CkqDG8fH8ZPnJq7flR4KBfOnyczM5O1yd8ze8586tvppI2HhwfDR7zJs/36kp1tJjYn9pnvT6dJk6YEh7YlrktXRrw6hI6RltgnTLLEHhjYgPDIKOKi2+Pu7s5rr7+JewlfR/3i88/4ZP5HnDxxgm5x0fynTRBvjRpj8xxfeXUELz37NNnZ2XSKiaNeYAM+nPU+jRo3oU1wKNFxXXhrxDC6dIrA29uX0SX8kTh18iRDX34JAHNWFhFRHQq9usFunPx+ukrbeKDT8OQCFxhJtfbXBw6+DtFG3Jygt1nWXDAlAP7NdEifx+Z8K7jf8B6rEDun1DXn0pJ+dj9C5BtpQgjXIl8DFkIIO3LyjzdSdIUQLsUZTsoWR4quEMKlSNEVQgg7Um5SdIUQwm6kpyuEEHYkRVcIIexIiq4QQtiTc9dcKbpCCNciPV0hhLAjZ7hvcXGcOzohhLhGSqlST6XYVqRS6nel1P7CHtyglKqrlFqrlPpJKbVLKdW+pG1K0RVCuBZ1DVNxm1HKHZgJRAGNgZ5Kqcb5mr0OLNBaNwd6ALNKCk+KrhDCpZRhT7cVsF9rfVBrnYHl0WUx+dpo4MqTcX2Ao5RAxnSFEC7lWk6k5X20WI45Wus5OT/XAY7kWZYC3JdvE28Bq5RSLwKVgBIflSNFVwjhUq7la8A5BXZOiQ2L1hP4RGs9WSn1APBfpVRTrXWRjzOzedH97f0utn4Lu6sRM83RIdjEyfiBjg6hzClnv2jzOrniDefLShleMpYKBOSZ9895La+ngEgArfUWpVR5oAaQThFkTFcI4VLKcEx3B9BAKXWbUsoLy4my+HxtDgNtc963EVAeOF7cRmV4QQjhUsqqp6u1zlJKvQAkAe7AfK31XqXUKGCn1joeGAzMVUoNwnJS7XFdwjPQpOgKIVxKWX4jTWudCCTme+3NPD//CrS+lm1K0RVCuBYnH+6WoiuEcCnO/jVgKbpCCJciN7wRQgh7cu6aK0VXCOFapKcrhBB2JEVXCCHsSIquEELYkTyCXQgh7Eh6ukIIYUdSdIUQwo6cvOZK0RVCuBbp6QohhB25yYk0IYSwHyfv6Dpv0d2yeSPTJo3DbDYTHdeVR5942mp5RkYGo954ld/27cXH15fR706hVu06HDuaSo8uHbnlllsBaNLsLoaNeIsLFy7w7FO9c9dPTzcREdWJQUOG2zOtIoXdewuT+gfj7ubGJyv3MGnhDqvldQ1V+GBQODV8KnD63L88OXElqSfOOyhaa5s3bWTiu2PINmcT26UrT/btZ7U8IyODN4YPY9+vln01ftIUatfx58yZ0wwZNIC9e/YQHRvLqyNy75hHZmYG7455h507tuPm5sbzLw2kXViE7XPZuIHxObnEdenGU08XzGXE8KHs22vJZcLkqdSp4w/AvLkfsvi7b3Fzd2PY8Ndp/Z+HOPTXQYYOHpS7fkrKEZ574SV6P/o4QwYP5O+//gLg3LlzVKlShQWLlto8xx82b2Ty+LFkZ2cTE9eVx58q+Ls1csQwftv3Kz4+voydMIXadeqwd/cuxrwz0tJIa57u/zwhbcMA+PK/n7Bk0bcopQhscDtvjhpLuXLlbJ5LYaSnex3MZjOTx49m+qyPMBiNPNn7YR4KCuG2eoG5bZYt+Y4q3t58G5/E6qREZk6fzOjxUwDw9w/gs68XW22zUqVKVq893qsrwaFh9kmoBG5uimnPh9LhtUWknjjHpum9WL7tAL8dPpXbZlzfNnyxZh9ffP8rQXcFMOrx//DUpJUOjNrCbDbz7uhRzJ47H6OfkUce7kZQSCj161/dV0sWfUsVb2/iV6xiZWIC06dMZvzkqZTzKsdzLw5g/59/cmD/H1bb/ejDD6hWrTpLE5LIzs7m7Nmzdsll7JhRfDj3Y4xGI70e7kpwSCj1A6/msvi7hXh7e7N85WpWJCYwbcokJk6exoH9+1mZmMCi+ATS00080/cJ4hOSuPW2ermF1Gw2ExbShtB2luNu4uRpududNOFdKleubJccJ4x9hxkfzsNoNPJYr+60CQ6hXp79tXTxt3h7+7B4eRKrViTw/rRJjJs4lfqBDfjsy4V4eHhw4ng6vbrF8VBQCKdOnuSbLz/nm8XLKV++PMOHDGLVykQ6xcTZPJ/COHtP1ynvgfbrnt34+9eljn8Anp5etIuIYsO6ZKs2G9cl075jLAAhbcPZuWMrJdywPdfhvw9x+vQp7r7n3rIO/bq0vN2PA0fPcCjtLJlZ2Sxc/zsd769v1aZh3eqs//kwAOt/OULHB+o5ItQC9uzeRUDduvgHWPZVRFR71iWvsWqzLnkNnWJiAWgXHsH2bVvQWlOhYkWa33Mv5cp5Fdju0sWLcnvMbm5uVK1a1T65BNxiycXLi8j2HVi31jqXtcnJROcUk7DwCLZvteSybu0aItt3wMvLC3//AAICbmHP7l1W627buoWAgABq165j9brWmlVJK4jq0NG2CQJ79+wiIKAu/jm/W2GR7Vmf73drw9pkOkRbnjQeGhbBju2W363yFSrg4WHpp12+nGF1wirLbOby5X/Jysri30uXqFnTYPNcilKGj+uxCacsusePmzD4+eXOGwx+HE9PL9DGmNPGw8ODypWrcPbMGQCOpqbyaM/OPNv3UX7+cWeB7a9OSqRteKTTnOWsXaMyKcfP5c6nnjhPnerWvZ7dB48T07oBADEPBuJdsRzVqpS3a5yFSU83YfSrlTtvNPpxPN2Ur006fjltruyrMzn7qjDn/vkHgJkzptOzW2eGvDyAkydOlH3w+aSbTPjVynPcGY2YTPlzMVnnUqUKZ86cxmS6ejwCGP2MpOdbd+WKBCLbFyysP/5vJ9WrV88dErOl4+np1nEajBwvJEdjvv115Xdrz65f6B7XkZ5dY3j19ZF4eHhgMBrp/dgTdIpoS1S7NlSqUoX7H7ymhymUKaVKPznCdRddpdQTZRlIWaleoyZLEtfw2VeLGPDyMEaOGMqF89Zjn98nJRIe0cFBEV6f4R9t4KFmddgy4xEeauZP6olzmLNL17O/2WSZzZhMadx1d3O+WriIO++6m6mTJjg6rBuSmZHB+rXJhEdEFli2InF5ocXYGTW98y4WLF7Op18u4JN5c7l8+TL//HOWDWuTWZq4mhWr1/PvpUskLs///Eb7cXNzK/XkkPhuYN23i1qglOqnlNqplNr56fy517zhmjWNpKel5c6np6dR02Ao0MaU0yYrK4vz58/h4+uLl5cXPr6+ADRs3IQ6/gEcPnwod70///gNs9lMw8ZNrjkuWzl64jz+NavkztepUZnUk9Z/KI6dukCP0ct54IUvGPnpZgDOXrhs1zgLYzAYMaUdy503mdKoaTDma2MgLafNlX3lm7OPCuPr60v5ChVo2y4cgLDwSPbt+7Xsg8/HYDSSdizPcWcyYTTmz8Voncu5c/j6VsVovHo8ApjSTBjyrLtp0wYaNm5C9Ro1rLaXlZXFmu9XExnZ3hYpFVDTYLCOM91EzUJyNOXbXz759tdt9epTsWJFDuz/k+1bt1C7Th2qVquGh6cnIW3bseuXn2yeS1Fu6p6uUmpXEdNuwFjUelrrOVrrFlrrFo89+XRRzYrUqElTjhz5m6OpKWRmZvB90goeCgqxavOfoBASly8BYO2aVdzb8j6UUpw+fQqz2QxAasoRjhz+m9o5Z5cBVq9MJCzCPgd4ae38I43A2lW5xeiNp4cb3YLuIGHrQas21b3L5x4kQx5uyaer9jog0oKaNG3G4cN/k5pi2VdJKxIJDgm1ahMUEsqypUsA+H5VEi3vu7/YoR2lFG2CQti5YzsA27dtoV79+kW2LyuWXA6RknKEzIwMViYmEJQvl+CQUOKXWk7Irl6VRKucXIJCQlmZmEBGRgYpKUc4fPgQTZvdmbveisQEotoX/HS1bcsP3HZbPauP/LbUuIn1/lq9MpE2+X63HgoOISHecvIveXUSLVtZckxNSSErKwuAY0dTOXToILVr18HPrxa7d/3Cv5cuobVmx7at3Hab7fdXUZx9TLekqxeMQARwOt/rCvjBJhFhGUcaPGwEA59/muzsbDpGx1GvfgPmzH6fRo2b8FBQKJ1iu/D2G8PoGh2Bt48v74ybBMDPP+5k7uz38fDwQLm5MfS1kfj4+OZue83qlUx+7wNbhX5dzNmaQbOTWTa6M+7uik9X7WXf4ZO80ecBfvzDRMK2g7S5M4BRj7dGa9i0J4WBs9Y6OmzAsq+GvfYGzz3zFNnmbGLiulA/sAGzZrxH4yZNCQ4JJbZzV14fPpToqHC8fXx4d+KU3PXbh4dy4fwFMjMzWZu8hllz5lG/fiADXh7M68OHMendsVStVo23Ro+1Sy7DR7zJs/36kp1tJjauC4GBDZj5/nSaNGlKcGhb4rp0ZcSrQ+gYGYa3jw8TJk0FIDCwAeGRUcRFt8fd3Z3XXn8Td3d3AC5evMjWH37gjZGjCrznyhWJRBZSjG2Z49Dhr/PSs30xZ2cTHduZ+oEN+GDmezRq0pSg4FBi4roycsQw4jpG4O3tw5gJkwH45af/8cn8uXh4euKmFMNeexPfqlXxrVqVtmER9O7RBXd3d+5o2Ii4rt3tllN+TnKqpkiquDP+Sql5wMda602FLPtSa92rpDc4dcHscgOPdbq+5+gQbOJk/EBHh1Dm3Jz9N/A6ZWRlOzoEm/Auf+MX2d77ztpS15z/vRFi9wOk2J6u1vqpYpaVWHCFEMLenP3vrFN+OUIIIa6XfCNNCCHsyFmuvy+KFF0hhEtx8porRVcI4VqkpyuEEHbk5DVXiq4QwrXIiTQhhLAjGV4QQgg7kqIrhBB25OQ1V4quEMK1SE9XCCHsyMlrrhRdIYRrcfarF5zycT1CCHG93JQq9VQSpVSkUup3pdR+pdSrRbTprpT6VSm1Vyn1ZUnblJ6uEMKllNXwglLKHZgJhAEpwA6lVLzW+tc8bRoAw4HWWuvTSqkSn8gpPV0hhEspwydHtAL2a60Paq0zgK+BmHxtngZmaq1PA2it0ymBFF0hhEtxU6Wf8j7PMWfql2dTdYAjeeZTcl7L63bgdqXUZqXUVqVUwSeP5mPz4YXyXq5X10/FD3J0CDZRrctsR4dQ5k5+19/RIdiEs5+hd6RrOZGmtZ4DzLmBt/MAGgDBgD+wQSnVTGt9psj4buDNhBDC6ahr+K8EqUBAnnn/nNfySgHitdaZWuu/gD+wFOEiSdEVQriUaxleKMEOoIFS6jallBfQA4jP12YJll4uSqkaWIYbDlIMuXpBCOFSyuobaVrrLKXUC0AS4A7M11rvVUqNAnZqreNzloUrpX4FzMAQrfXJ4rYrRVcI4VLKcrxba50IJOZ77c08P2vg5ZypVKToCiFcSmm+9OBIUnSFEC7F2b8GLEVXCOFSnLyjK0VXCOFaZHhBCCHsyLlLrhRdIYSLkZuYCyGEHTn5eTQpukII1yJXLwghhB3J8IIQQtiRk3d0pegKIVyL9HSFEMKOnLvkStEVQrgYdycfX3Da++lu3rSR2I6RREeFM/+jgjd2z8jIYNjgQURHhdOnZ3eOpqYAcObMaZ5+4lEebHkP744ZZbVO38f7ENsxkoe7xPJwl1hOnSz2Dmw2t3nTBmI6RtApKqzIHIcOHkinqDB69+xGap4c+z7RhwdaNmdcvhydQdg9Afwyqyd7PuzFK12aF1geUKMyK0dHs2VaV7a/152Ie+sC4OHuxtyBoex4rzs/zezBK10LrmtPrnoM/rBpI507RRHbIYJP5s0tsDwjI4PhQwYR2yGCx3o9zNFUy3279+zeRa9ucfTqFkfPrrGsXbM6d5233xxBWFBrusd1slseRSnDZ6TZhFMWXbPZzLujRzFj9ly+i1/OysQEDhzYb9VmyaJvqeLtTfyKVTzS5zGmT5kMQDmvcjz34gAGvTK00G2PeXci33y3hG++W0K16tVtnktRzGYz40aPYubsj1gUn8DKxOUFcly8aCHe3t4sW7Ga3n0eZ/qUSYAlx+dfHMDLReToSG5uimnPPETM28tp/vzXdGsTSMOAqlZthj18L99tPsADA7/l0Ymrmd7/IQC6tK5POQ83Wr60gAcHfUvfiMbUNVRxRBouewyazWbGj32H92bPYeGSZSStSOBgvryWLvqWKt4+LElIolefR3l/muW4CwxswGdfLeTLhYt5f/Ycxo56i6ysLAA6Rcfy/uwbeepN2VGq9JMjlFh0lVINlVJtlVKV871e4gPYrtee3bsIqFsX/4AAPD29iIhqz7rkNVZt1iWvoVNMLADtwiPYvm0LWmsqVKxI83vupVw5L1uFVyYsOd6SJ8cOheSYTKeYOKCwHFvgVa6cI0IvVssGBg4cO8sh0zkys7JZuHE/He+71aqN1hrvCp4A+FT04tipi5bX0VQs74m7m6JCOXcysrI5dzHD3ikArnsM7t2Tk5e/Ja/wyPasX5ts1Wb9umQ6Rlseets2LILt27aitaZ8hQp4eFhGJC9fzrDqKd7ToiXePr52y6M4bkqVenJIfMUtVEq9BCwFXgT2KKXyPn54rK2CSk83YfSrlTtvNPpxPN2Ur006fjltPDw8qFy5CmfOnClx22+98RoPd4llzgezsNx/2DHS0034+fnlzhuNRtIL5GgqJMfTdo3zWtWuXomUExdy51NPXKBO9UpWbcZ8tZMewbezf34fFo/swMtzNgKwaPNBLv6byV+fPsYf8/owbcnPnD5/2a7xX+Gqx2C6KR2j8epxZyjsuDOZMBqt8zqbk9eeXb/QPa4jPbrEMPyNkblF2Jnc7D3dp4F7tdaxWJ4D9IZSakDOsiJDzvtY48LGwhxl7PhJLFy8jPmffc5P/9vJ8viljg7p/6XubQL5PPl3Ap/8L3FvJzBvUFuUgpa3GzBna+o9/hmNnv6CATF3c6vRMcMLtnKzH4NN77yLBYuX89lXC/h43lwuX3bMH8Xi3Oxjum5a6/MAWutDWApvlFJqCsUUXa31HK11C611iyf79iuqWZEMBiOmtGO58yZTGjUNxnxtDKTltMnKyuL8+XP4+voWv12jZRuVKlUmqkNH9u7Zdc2xlRWDwUhaWlruvMlkwlAgR2MhOVqPjzqboycv4F/jas+2To1KpJ68YNXmsbBGfLfJMo647XcT5b08qOFdge5tGrDqxyNkmbM5fvYSW347xr2BBrvGf4WrHoMGowGT6epxl17YcWc0YjJZ5+WTL6/b6tWnYoWKHNj/p81jvlbuSpV6coSSiq5JKXX3lZmcAtwRqAE0s1VQTZo24/Dhv0lNSSEzM4OkFYkEh4RatQkKCWXZ0iUAfL8qiZb33V/sX66srCxOn7Z8NM/MzGTD+nXUD7zdVimUyJLjIVJTjuTkmEBQoTkuBkqXozPY+Wc6gbV9ucVYBU8PN7o9FEjCtkNWbY4cP0/wnf4A3OHvS3lPd46fvUTK8XME31kHgIrlPGh1u5HfUx0znOKqx2DjJs048vfVvFatTKRNcIhVmzbBIbk98DWrk2jZypJXakpK7omzY0dTOXToILVr17Fr/KVRhk8DtglV3JiSUsofyNJapxWyrLXWenNJb3Ax8/oGrTZuWM+k8WPJNmcTE9eFvs/0Z9aM92jcpCnBIaFcvnyZ14cP5fd9+/D28eHdiVPwD7A8or59eCgXzl8gMzOTKt5VmDVnHrVr1eapx3uTlZmFOTub++5/gMFDX8Xd3f2aY1NldPn1xg3rmTh+LNlmMzFxXXj6mWeZNWN6To5tuXz5MiOGD8nNcfzEqbk5RoWHcuH8+dwcZ8+ZT/36gTcUT7Uus8siLSLurcvEvq1xd1N8+v1vTFj4I2/0asmP+4+TsP0QDQOqMuuFICqV90RrGPHJFtb8nEKl8h7MGRBKw4CqKOC/a35n6uKfbyiWk9/1v+51nfkYNGdf/1jwpo3rmTJhHGZzNtGxnXmqX38+mPkejRo3JSgnrzdfG8bvv1nyGjthMv7+ASQsW8qn8+fi4eGJUoqn+z9HcGg7AF4bOpj/7dzOmTNnqF6tOv2ee4HYzl2vObYq5W68FL4c/1up/3GmRDe0e+kttuiWhestus6srIqusymroutMbqToOrMbKbrOrCyK7uBlv5f6H2dypzvs/svsfKcehRDiBjj5F9Kk6AohXIuTn/aQoiuEcC0eTl51pegKIVyKk9dcKbpCCNcij2AXQgg7cvKaK0VXCOFa5OoFIYSwI2e/ibkUXSGES3HymitFVwjhWpz9G6NSdIUQLkV6ukIIYUdSdIUQwo6c/fanTvlgSiGEuF7ubqWfSqKUilRK/a6U2q+UerWYdl2UUlop1aKkbUpPVwjhUsrqG2lKKXdgJhAGpAA7lFLxWutf87WrAgwAtpUqvjKJTgghnEQZPjmiFbBfa31Qa50BfA3EFNLuHWA88G9p4pOe7nXQuOYNpNMXPOPoEMpc9VYvOjoEmzi57X1Hh+C0rqWjq5TqB+R9kOMcrfWVp+nWAY7kWZYC3Jdv/XuAAK11glJqSGneU4quEMKluF3Ddbo5Bfa6HlmulHIDpgCPX8t6UnSFEC6lDC9eSAUC8sz757x2RRWgKbAu54oJPyBeKRWttd5Z1Eal6AohXIpH2V2ouwNooJS6DUux7QH0urJQa30Wy5PRAVBKrQNeKa7ggpxIE0K4GKVKPxVHa50FvAAkAfuABVrrvUqpUUqp6OuNT3q6QgiXUpY3MddaJwKJ+V57s4i2waXZphRdIYRLcfIvpEnRFUK4FmcfM5WiK4RwKfKMNCGEsCMpukIIYUfOXXKl6AohXIyTd3Sl6AohXIuz309Xiq4QwqXI1QtCCGFHciJNCCHsSIYXhBDCjmR4QQgh7MjZe7pO+0dh86aNxHaMJDoqnPkfFbzHcEZGBsMGDyI6Kpw+PbtzNDUFgDNnTvP0E4/yYMt7eHfMKKt1nn+mL907x9AlpiOj3x6J2Wy2Sy552SKvKwa88CxdYzvZNP6i/LB5I52jo4jtGMEn8+YWWJ6RkcHwIYOI7RjBY488zNFUy21J9+zeRa/ucfTqHkfPbrGsXbPaaj2z2Uyv7p0Z+EJ/u+RRlLAHG/HL4jfYs3QkrzwRVmB53VpVSfzgRbZ/M5ykuQOoY/DNXTb6pRh2LnyNnQtfo2v4PXaMunCbN20ktlMk0e2LOQZfGUR0+3D69Mp3DD75KA+2KngMrkhcTre4TnTvHM3z/fty+vRpu+RSGHUNkyM4ZdE1m828O3oUM2bP5bv45axMTODAgf1WbZYs+pYq3t7Er1jFI30eY/qUyQCU8yrHcy8OYNArQwtsd/zkaSxYtJRvlyzj9OlTrE5aaZd8rrBVXgBrVq+iYsWKNs+hMGazmfFj3+G9WXNYuHgZSSsTOJgvr6WLv6WKtw9LlifRq/ejvD9tEgCBgQ347MuFfLlgMe/PmsPYd94iKysrd72vvvgvt9WrZ9d88nNzU0x7tTsxL8yieZfRdIu8l4b1/KzajBsUxxcJ22n18DjGzlnBqBctd/6L/E8T7m4UwH093qVNn0kMfLQtVSqVd0QaQM4xOGYUM2bN5buly1m5ophjMDHnGJya5xh8oeAxmJWVxcTxY5kz/zMWLIqnwe138M1Xn9stp/zclSr15AhOWXT37N5FQN26+AcE4OnpRURUe9Ylr7Fqsy55DZ1iYgFoFx7B9m1b0FpToWJFmt9zL+XKeRXYbuXKlQHLQZKVmWn3jyG2yuvixQt8/tkn9H3mWXukUcDePbsICKiLv78lr/DI9qxfl2zVZv3aZDpGW57p1zYsgu3bt6K1pnyFCnh4WEa5Ll/OsNonJlMamzeuJzauq/2SKUTLprdy4MgJDqWeJDPLzMKkH+kYfKdVm4b1arF+++8ArN/xBx2DmwHQqJ4fm37cj9mczcV/M9j9ZyrhDzayew5XFHoMrs13DK5dQ6foWADahRVyDHpZH4Naa7TWXLp0Ea0158+fp2ZNg71SKqCs7qdrKyUWXaVUK6VUy5yfGyulXlZKtbdlUOnpJox+tXLnjUY/jqeb8rVJxy+njYeHB5UrV+HMmTMlbvu5fk/RNqg1FStVol14RJnGXRJb5TXr/ffo89gTVCjvmB5Ueno6Rr+rPT+DwUi6KX9eV3O/ktfZnLz27PqF7nEd6dE1huGvj8wtwpMnjOOlQa+g3BzbN6ht8CHFdPXjcqrpNHVq+li12f1HKjGhdwMQE3oX3pUrUM2nErv+sBTZCuU9qe5biaAWt+PvV9We4Vsp9BgssK+u7Rj09PTktddH0r1zNOGhbTh44ACxnR33h1Jdw3+OUOzRrJQaCbwHzFZKjQNmAJWAV5VSI+wQX5mbNWceq9duJCMjgx3btjo6nBv2+2/7OHLkMKHtCo4z3iya3nkXCxYv57MvF/DxvLlcvnyZjevXUq1aNRo1buLo8Epl+NTFPHRvIFu+GsZD9waSajqN2ZzNmq2/sXLTr6z9ZDCfjnuCbbv+wmzOdnS4ZSozM5NvF3zNVwsXsyp5A7fffnuhY8X2crP3dLsCrYE2wPNArNb6HSACeLiolZRS/ZRSO5VSO6/nH99gMGJKO5Y7bzKlUdNgzNfGQFpOm6ysLM6fP4evr2+ptl+uXDmCQ9oW+Fhla7bI65eff+bXvXtoHx7KE48+wt+HDtH38T42ib8oBoMBU1pa7nx6ugmDMX9eV3O/kpdPvrxuq1efihUrcmD/n/zy809sWLeWTlFtGTFsMDt2bOON4YWPZ9va0fSz+Buv9k7rGKuSevysVZtjx8/S45WPeKDneEbOWAbA2fOXAJgwL4n7e7xLx2dnoJTiz8Pp9gs+n0KPwQL76tqOwT9+/w2AgIC6KKUIi4jil59/KvvgS8kNVerJMfEVL0trbdZaXwQOaK3/AdBaXwKK/HOttZ6jtW6htW7xZN9+RTUrUpOmzTh8+G9SU1LIzMwgaUUiwSGhVm2CQkJZtnQJAN+vSqLlffcXO0Z78eIFjh+3HOxZWVls2rCeW2+z7wkaW+TVvUdPVq/dSOKqZD7+7AtuufVWPvrkv7ZMo4DGTZpxJE9eq1Ym0iYoxKpNm+AQlscvBWDN6iRatrLklZqSknvi7NjRVA4dOkjt2nV4YcDLJK5ex7IVaxgzfjItW97HO+Mm2DWvK3bu/ZvAujW5pXZ1PD3c6RZxDwnrdlm1qe5bKXc/DXkygk+XWj5FubkpqvlUAqBpg9o0bVCb77f8Zt8E8mjStBmH/853DAbnOwaDQ1kWvwSA7/Psq6LUNBg4eOAAp06dAmDrlh8cevLT2Xu6JV2nm6GUqphTdO+98qJSyodiiu4NB+XhwbDX3uC5Z54i25xNTFwX6gc2YNaM92jcpCnBIaHEdu7K68OHEh0VjrePD+9OnJK7fvvwUC6cv0BmZiZrk9cwa848fH18GfjCc2RmZJCtNS1ataJr9x62SsFuedWvH2jXHArj4eHBkOGv8+KzfTFnZxMd25n6gQ34YOZ7NGrSlKDgUGLiuvLmiGHEdozA29uHsRMsZ8R//ul/fDp/Lh6eniilePW1N/Gt6rgxz8KYzdkMGr+AZbOex91N8enSrew7mMYbz3bgx18Pk7B+N21aNGDUi9FoDZt+3M/AcQsA8PRw5/v5AwE4d/5fnhzxqUOHF3KPwf6lOAbb5xyDE/IcgxGFH4P9nn2evo/3xsPDg1q1a/P26HEOy9HZvwastNZFL1SqnNb6ciGv1wBqaa13l/QGFzOLeQPhVMxm19tVhgdecnQINnFy2/uODsEmKnrdeMVc89uJUh/IbRvWsHuFLranW1jBzXn9BHDCJhEJIcQNcNRVCaUlXwMWQrgUJx9dkKIrhHAt0tMVQgg7cnPumitFVwjhWpz96gUpukIIl+LcJVeKrhDCxUhPVwgh7Mi5S64UXSGEq3HyqitFVwjhUmR4QQgh7Mi5S64UXSGEq3HyqitFVwjhUuQbaUIIYUdOPqTrnA+mFEKI61WWj2BXSkUqpX5XSu1XSr1ayPKXlVK/KqV2KaXWKKVuKWmbUnSFEC5FKVXqqYTtuAMzgSigMdBTKdU4X7OfgBZa6zuBb4ESH28iRVcI4VLK8HE9rYD9WuuDWusM4GsgJm8DrfXanCfrAGwF/EvaqM3HdM9dyrL1W9hd5fKuORTu5uy3Z7oOp7bPcHQINlGj58eODsEmLnz7xA1v41qOYqVUPyDvgxznaK2vPE23DnAkz7IU4L5iNvcUsKKk93TN6iGE+P/rGqpuToG94efFK6V6Ay2AoJLaStEVQriUMrxkLBUIyDPvn/Oa9fsp1Q4YAQQV9YizvGRMVwjhUspwTHcH0EApdZtSygvoAcRbv5dqDnwIRGut00sTn/R0hRAupayu09VaZymlXgCSAHdgvtZ6r1JqFLBTax0PTAQqAwtzroY4rLWOLm67UnSFEC6lLL+RprVOBBLzvfZmnp/bXes2pegKIVyKs38jTYquEMKlOHnNlaIrhHAxTl51pegKIVyK3MRcCCHsyLlLrhRdIYSrcfKqK0VXCOFS5CbmQghhR04+pCtFVwjhWpy85krRFUK4lpJuTu5oUnSFEC7FyWuuFF0hhGtx8pp789zacdsPm3ikS0d6xkXx+ScfFViekZHByOGD6RkXxTOP9+TYUcttL7OyMhnz1ms81iOO3t068fnHc+0dupXNmzYS1ymS6PbhfPxRwXsnZ2RkMOyVQUS3D+fRXt05mpoCwNYfNtOre2e6x3WiV/fObN+2NXedGe9NJapdMK1b3WO3PPKzRV7P9+/Lw11i6BrbkTGjRmI2m+2WT36bN20gpmMEnaLCmF9EfkMHD6RTVBi9e3YjNSe/M2dO0/eJPjzQsjnjxoyyd9glCru7Dj9N78yu97swOLZZgeX+NSqR+FYkP0yMZtvkGCKaW55GU7dmZU580YctE6PZMjGa6f0esHfoRSvLJ1PawE1RdM1mM1MnjGbi9Nl8tiCeNasSOXTwgFWbhKWLqOLtzVeLV9C9Vx8+eH8KAGu/X0VmRgaffr2Yj/67gPjFC3MLsr2ZzWbGjxnF+7Pm8t3S5axckcDBA/ut2ixZ9C3e3t7EJ67ikT6PMX3qZAB8q1Zl+ozZLFi8jFFj3uWN14bmrtMmKITPvlpg11zyslVe4ydN45vvlrJw8TJOnz7F96tW2jWvK8xmM+NGj2Lm7I9YFJ/AysTlHMiX3+JFC/H29mbZitX07vM406dMAqCcVzmef3EAL78ytLBNO5Sbm2JK3/uJG7OKewctptt/6tHQ38eqzbAud7Hoh794cEg8j01dx9Sn789d9pfpHA8MieeBIfEMmLPF3uEXSV3Df45wzUVXKfWZLQIpzr69u6kTUJfa/gF4enrSNiyKTeuTrdps2pBMZAfLM+OCQsP5ccc2tNYopfj30iWysrK4/O9lPDw9qVSpsr1TAGDP7l34162Lf0AAnp5eRES1Z93aNVZt1q1dQ8foWADahkWwY9sWtNY0bNSYmgYjAPUDG3D538tkZGQAcOddd1OzpsGuueRlq7wqV7bsp6ysLDIzMx02WLdn9y4C6t6SJ78OrEvOl19yMp1i4gBoFx7B9pz8KlSsSPN7WuBVrpwjQi9Wi8AaHEw7x6H082RmZfPt5oN0bFnXqo3W4F3RC7D8/9jpS44I9ZqU4U3MbaLYoquUis83LQM6X5m3U4ycOJ6OweiXO1/TaOT4ceubtJ9Iv9rGw8ODSpUrc/bsGYLbhlG+QgXiokLo1imMHo88jreP9V9zezmebsLPr1buvMHoR7rJlK9Nem4bDw8PKleuwpkzZ6zarFmdRMNGjfHy8rJ5zKVhy7yee+Yp2gW1plLFSrQLi7BdEsVITzfh53f1+DMajaSnmwppkz+/03aN81rVrlaRlBMXcudTT16kVrVKVm3GLviJHg/V548Pu7PotTAGz7s6/HOLoTI/TIxm5dtRPNjIaLe4S+KmSj85JL4SlvsD/wBTgMk507k8PxdKKdVPKbVTKbXzvx8XHH+1p317d+Pm5s7iFcl8s3Ql33zxKUdTjpS8opM6sP9P3ps6mREj33Z0KGWqqLxmfTiPVWs3kpGZwY48473CPrr9px6fr/uT259ZQOexq/noxTYoBWmnL9Kw/0IeHBLPq59u5+MBQVSp4OnocHM496BuSUW3BfA/LA9dO6u1Xgdc0lqv11qvL2olrfUcrXULrXWLPk/0veEga9Q0kG5Ky50/bjIV+Dhdw3C1TVZWFhfOn8fHx5fVKxO578HWeHh4UrVadZrddTe/7dt7wzFdj5oGI2lpx3Ln001pGIzGfG0MuW2ysrI4f/4cvr6+AJjS0hg88AVGjR1PQID1x0BHsnVe5cqVIzikbYEhC3sxGIykpV09/kwmEwaDsZA2+fOratc4r9XRUxfxr3G1Z1unekWOnbpg1ebRtg347odDAGz/4zjlvdypUaU8GVnZnDpveQbjzwdPctD0D4G1ve0We3Fu6uEFrXW21noq8AQwQik1AwdcZtawcVNSDh/maGoKmZmZrFm9gtZtQqzatH4ohJUJSwFYn7yKe1reh1IKo18tftyxHYBLly6yd88ubrn1NnunAECTps048vffpKakkJmZQdKKRIKCQ63aBAWHsjx+CWD5uN2y1f0opTj3zz+89PwzvDhwMHc3d9xVCoWxRV4XL17IHULKyspi44b13HpbPbvllFeTps04fPgQqSlHcvJLICgkX34hoSxbuhiA71cl0fK++53+Iv3/7T9B/Vre3GKojKeHG11b1yNhh/WnwJQTFwhpZhk2uaOOD+U93Tn+z7/U8C6HW87n81sNlQn08+aQ6ZzdcyiMc/dzQWmtS99YqQ5Aa631a6Vdx/RPZunfoBhbNm/g/SnjyTabaR8dx6NPPsO8D2ZwR6Mm/CcohMuXLzNm5HD+/H0fVbx9eGvMRGr7B3Dx4kXeHfU6hw4eQKNp3ymWnn2evKFYKpe//r87mzasZ9KEsWSbs4mO60Lffv2ZPeM9GjdpSlBIKJcvX+aN4UP57bd9+Pj4MG7CFPwDAvjow9nMnzeHunVvyd3WrA/nUa16daZNmcjKhOUcP55OzZoGYrt0pf9zL95Qjo7OS2vNgBf6k5GRgdaaFi1bMXjocDw8ru3fvqzurbpxw3omjh9LttlMTFwXnn7mWWbNmE7jJk0JDmnL5cuXGTF8CL/v24e3jw/jJ07FP8Dy9O6o8FAunD9PZmYmVbyrMHvOfOrXD7yheGr0/Lgs0iKiuT/jn2iFu5vis+Q/mbhoF68/3JwfD5wgcecRGvr7MKN/ayqX90Rrzeuf72TNL0eJue8WXu/RnKysbLI1jP7mJ1b878aH7S58+8QN77BjZzNKXXNq+XjZvfZeU9G9HmVVdJ3JjRRdYV/OfkPr61VWRdfZlEXRTbuGmuPn7Wn3A0SqhxDCpTj7n1kpukIIl+LsH26k6AohXIrcxFwIIezJuWuuFF0hhGtx8porRVcI4Vqc/YoVKbpCCJfi5DX35ri1oxBCuArp6QohXIqz93Sl6AohXIpcMiaEEHYkPV0hhLAjKbpCCGFHMrwghBB2JD1dIYSwIyevuVJ0hRAuxsmrrhRdIYRLcfavAdv8yRH2pJTqp7We4+g4ypor5uWKOYFr5uWKOTmSq30NuJ+jA7ARV8zLFXMC18zLFXNyGFcrukII4dSk6AohhB25WtF11XEnV8zLFXMC18zLFXNyGJc6kSaEEM7O1Xq6Qgjh1KToCiGEHblE0VVKRSqlfldK7VdKveroeMqCUmq+UipdKbXH0bGUJaVUgFJqrVLqV6XUXqXUAEfHdKOUUuWVUtuVUr/k5PS2o2MqS0opd6XUT0qp5Y6OxRXc9EVXKeUOzASigMZAT6VUY8dGVSY+ASIdHYQNZAGDtdaNgfuB511gf10GQrXWdwF3A5FKqfsdG1KZGgDsc3QQruKmL7pAK2C/1vqg1joD+BqIcXBMN0xrvQE45eg4yprW+pjW+secn89h+WWu49ioboy2OJ8z65kzucQZaqWUP9AB+MjRsbgKVyi6dYAjeeZTuMl/if+/UErdCjQHtjk4lBuW8xH8ZyAdWK21vulzyjENGApkOzgOl+EKRVfchJRSlYHvgIFa638cHc+N0lqbtdZ3A/5AK6VUUweHdMOUUh2BdK31/xwdiytxhaKbCgTkmffPeU04KaWUJ5aC+4XWepGj4ylLWuszwFpcYzy+NRCtlDqEZdguVCn1uWNDuvm5QtHdATRQSt2mlPICegDxDo5JFEEppYB5wD6t9RRHx1MWlFI1lVK+OT9XAMKA3xwaVBnQWg/XWvtrrW/F8nuVrLXu7eCwbno3fdHVWmcBLwBJWE7KLNBa73VsVDdOKfUVsAW4QymVopR6ytExlZHWQB8svaafc6b2jg7qBtUC1iqldmHpBKzWWsvlVaJQ8jVgIYSwo5u+pyuEEDcTKbpCCGFHUnSFEMKOpOgKIYQdSdEVQgg7kqIrhBB2JEVXCCHs6P8A2FN9sHxoJ14AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(cm, annot=True, cmap=\"Blues\")\n",
    "plt.savefig(f\"images/{model_name}.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.80      0.81      1492\n",
      "           1       0.93      0.90      0.91      3476\n",
      "           2       0.91      0.88      0.89      3017\n",
      "           3       0.96      0.99      0.97     15462\n",
      "           4       0.87      0.85      0.86      2890\n",
      "\n",
      "    accuracy                           0.94     26337\n",
      "   macro avg       0.90      0.88      0.89     26337\n",
      "weighted avg       0.94      0.94      0.94     26337\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(train[\"label\"], test[\"label\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9361734442039716\n"
     ]
    }
   ],
   "source": [
    "print(get_score(train[\"label\"], test[\"label\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}