cassava-validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# About this notebook  \n",
    "\n",
    "TBD...\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Directory settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Directory settings\n",
    "# ====================================================\n",
    "import os\n",
    "\n",
    "OUTPUT_DIR = \"./\"\n",
    "MODEL_DIR = \"../input/cassava-model/\"\n",
    "if not os.path.exists(OUTPUT_DIR):\n",
    "    os.makedirs(OUTPUT_DIR)\n",
    "\n",
    "TRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n",
    "TEST_PATH = \"../input/cassava-leaf-disease-merged/train\"\n",
    "# TEST_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n",
    "# TEST_PATH = \"../input/cassava-leaf-disease-classification/test_images\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "# CFG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# CFG\n",
    "# ====================================================\n",
    "class CFG:\n",
    "    debug = False\n",
    "    num_workers = 4\n",
    "    models = [\n",
    "        # \"tf_efficientnet_b3_ns\",\n",
    "        \"tf_efficientnet_b4_ns\",\n",
    "        \"vit_base_patch16_384\",\n",
    "        # \"deit_base_patch16_384\",\n",
    "        \"seresnext50_32x4d\",\n",
    "    ]\n",
    "    size = {\n",
    "        \"tf_efficientnet_b3_ns\": 512,\n",
    "        \"tf_efficientnet_b4_ns\": 512,\n",
    "        \"vit_base_patch16_384\": 384,\n",
    "        \"deit_base_patch16_384\": 384,\n",
    "        \"seresnext50_32x4d\": 512,\n",
    "    }\n",
    "    batch_size = 32\n",
    "    seed = 22\n",
    "    target_size = 5\n",
    "    target_col = \"label\"\n",
    "    n_fold = 5\n",
    "    trn_fold = {  # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "        \"tf_efficientnet_b3_ns\": {\n",
    "            \"best\": [0, 1, 2, 3, 4],\n",
    "            \"final\": [],\n",
    "        },\n",
    "        \"tf_efficientnet_b4_ns\": {\n",
    "            \"best\": [0, 1, 2, 3, 4],\n",
    "            \"final\": [],\n",
    "        },\n",
    "        \"vit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n",
    "        \"deit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n",
    "        \"seresnext50_32x4d\": {\"best\": [5, 6, 7, 8, 9], \"final\": []},\n",
    "    }\n",
    "    data_parallel = {\n",
    "        \"tf_efficientnet_b3_ns\": False,\n",
    "        \"tf_efficientnet_b4_ns\": True,\n",
    "        \"vit_base_patch16_384\": False,\n",
    "        \"deit_base_patch16_384\": False,\n",
    "        \"seresnext50_32x4d\": False,\n",
    "    }\n",
    "    weight = {\n",
    "        # \"tf_efficientnet_b3_ns\": None,\n",
    "        \"tf_efficientnet_b4_ns\": 1,\n",
    "        \"vit_base_patch16_384\": 1,\n",
    "        # \"deit_base_patch16_384\": None,\n",
    "        \"seresnext50_32x4d\": 1,\n",
    "    }\n",
    "    tta = 1  # 1: no TTA, >1: TTA\n",
    "    train = False\n",
    "    inference = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_sum = sum([CFG.weight[model] for model in CFG.models]) * CFG.tta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import sys\n",
    "\n",
    "sys.path.append(\"../input/pytorch-image-models/pytorch-image-models-master\")\n",
    "\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import time\n",
    "import warnings\n",
    "from collections import Counter, defaultdict\n",
    "from contextlib import contextmanager\n",
    "from functools import partial\n",
    "from pathlib import Path\n",
    "\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import seaborn as sns\n",
    "import timm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "from albumentations import (\n",
    "    CenterCrop,\n",
    "    CoarseDropout,\n",
    "    Compose,\n",
    "    Cutout,\n",
    "    HorizontalFlip,\n",
    "    HueSaturationValue,\n",
    "    IAAAdditiveGaussianNoise,\n",
    "    ImageOnlyTransform,\n",
    "    Normalize,\n",
    "    OneOf,\n",
    "    RandomBrightness,\n",
    "    RandomBrightnessContrast,\n",
    "    RandomContrast,\n",
    "    RandomCrop,\n",
    "    RandomResizedCrop,\n",
    "    Resize,\n",
    "    Rotate,\n",
    "    ShiftScaleRotate,\n",
    "    Transpose,\n",
    "    VerticalFlip,\n",
    ")\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "from PIL import Image\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from torch.nn.parameter import Parameter\n",
    "from torch.optim import SGD, Adam\n",
    "from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Utils\n",
    "# ====================================================\n",
    "def get_score(y_true, y_pred):\n",
    "    return accuracy_score(y_true, y_pred)\n",
    "\n",
    "\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    LOGGER.info(f\"[{name}] start\")\n",
    "    yield\n",
    "    LOGGER.info(f\"[{name}] done in {time.time() - t0:.0f} s.\")\n",
    "\n",
    "\n",
    "def init_logger(log_file=OUTPUT_DIR + \"inference.log\"):\n",
    "    from logging import INFO, FileHandler, Formatter, StreamHandler, getLogger\n",
    "\n",
    "    logger = getLogger(__name__)\n",
    "    logger.setLevel(INFO)\n",
    "    handler1 = StreamHandler()\n",
    "    handler1.setFormatter(Formatter(\"%(message)s\"))\n",
    "    handler2 = FileHandler(filename=log_file)\n",
    "    handler2.setFormatter(Formatter(\"%(message)s\"))\n",
    "    logger.addHandler(handler1)\n",
    "    logger.addHandler(handler2)\n",
    "    return logger\n",
    "\n",
    "\n",
    "LOGGER = init_logger()\n",
    "\n",
    "\n",
    "def seed_torch(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "\n",
    "seed_torch(seed=CFG.seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000015157.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000201771.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100042118.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000723321.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000812911.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_id  label  source\n",
       "0  1000015157.jpg      0    2020\n",
       "1  1000201771.jpg      3    2020\n",
       "2   100042118.jpg      1    2020\n",
       "3  1000723321.jpg      1    2020\n",
       "4  1000812911.jpg      3    2020"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n",
    "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n",
    "test = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Dataset\n",
    "# ====================================================\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self, df, transform=None):\n",
    "        self.df = df\n",
    "        self.file_names = df[\"image_id\"].values\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        file_name = self.file_names[idx]\n",
    "        file_path = f\"{TEST_PATH}/{file_name}\"\n",
    "        image = cv2.imread(file_path)\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "        if self.transform:\n",
    "            augmented = self.transform(image=image)\n",
    "            image = augmented[\"image\"]\n",
    "        return image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Transforms\n",
    "# ====================================================\n",
    "def get_transforms(*, data, size):\n",
    "\n",
    "    if data == \"train\":\n",
    "        return Compose(\n",
    "            [\n",
    "                # Resize(size, size),\n",
    "                RandomResizedCrop(size, size),\n",
    "                Transpose(p=0.5),\n",
    "                HorizontalFlip(p=0.5),\n",
    "                VerticalFlip(p=0.5),\n",
    "                ShiftScaleRotate(p=0.5),\n",
    "                HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n",
    "                RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n",
    "                CoarseDropout(p=0.5),\n",
    "                Cutout(p=0.5),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    if data == \"valid\":\n",
    "        return Compose(\n",
    "            [\n",
    "                Resize(size, size),\n",
    "                CenterCrop(size, size),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    if data == \"inference\":\n",
    "        return Compose(\n",
    "            [\n",
    "                # Resize(size, size),\n",
    "                RandomResizedCrop(size, size),\n",
    "                Transpose(p=0.5),\n",
    "                HorizontalFlip(p=0.5),\n",
    "                VerticalFlip(p=0.5),\n",
    "                # ShiftScaleRotate(p=0.5),\n",
    "                # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n",
    "                # RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n",
    "                # CoarseDropout(p=0.5),\n",
    "                # Cutout(p=0.5),\n",
    "                Normalize(\n",
    "                    mean=[0.485, 0.456, 0.406],\n",
    "                    std=[0.229, 0.224, 0.225],\n",
    "                ),\n",
    "                ToTensorV2(),\n",
    "            ]\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# MODEL\n",
    "# ====================================================\n",
    "class CassvaImgClassifier(nn.Module):\n",
    "    def __init__(self, model_name=\"resnext50_32x4d\", pretrained=False):\n",
    "        super().__init__()\n",
    "\n",
    "        if model_name == \"deit_base_patch16_384\":\n",
    "            # self.model = torch.hub.load(\"facebookresearch/deit:main\", model_name, pretrained=pretrained)\n",
    "            self.model = torch.hub.load(\"../input/fair-deit\", model_name, pretrained=pretrained, source=\"local\")\n",
    "            n_features = self.model.head.in_features\n",
    "            self.model.head = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "        else:\n",
    "            self.model = timm.create_model(model_name, pretrained=pretrained)\n",
    "\n",
    "            if \"resnext50_32x4d\" in model_name:\n",
    "                n_features = self.model.fc.in_features\n",
    "                self.model.fc = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "            elif model_name.startswith(\"tf_efficientnet\"):\n",
    "                n_features = self.model.classifier.in_features\n",
    "                self.model.classifier = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "            elif model_name.startswith(\"vit_\"):\n",
    "                n_features = self.model.head.in_features\n",
    "                self.model.head = nn.Linear(n_features, CFG.target_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Helper functions\n",
    "# ====================================================\n",
    "def inference(model, states, test_loader, device, data_parallel):\n",
    "    model.to(device)\n",
    "\n",
    "    # Use multi GPU\n",
    "    if device == torch.device(\"cuda\") and data_parallel:\n",
    "        model = torch.nn.DataParallel(model)  # make parallel\n",
    "        # torch.backends.cudnn.benchmark=True\n",
    "\n",
    "    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n",
    "    probs = []\n",
    "    for i, (images) in tk0:\n",
    "        images = images.to(device)\n",
    "        avg_preds = []\n",
    "        for state in states:\n",
    "            model.load_state_dict(state[\"model\"])\n",
    "            model.eval()\n",
    "            with torch.no_grad():\n",
    "                y_preds = model(images)\n",
    "            avg_preds.append(y_preds.softmax(1).to(\"cpu\").numpy())\n",
    "        avg_preds = np.mean(avg_preds, axis=0)\n",
    "        probs.append(avg_preds)\n",
    "    probs = np.concatenate(probs)\n",
    "    return probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: tf_efficientnet_b4_ns, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "519117ae6e3f433caedbcedd08d50ab4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.4113303  0.16465071 0.19031808 0.01710353 0.21659732]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: vit_base_patch16_384, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7075695aa1d044c3900a5e1bc761fe7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.44151726 0.12938017 0.24077013 0.04045077 0.14788169]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== Model: seresnext50_32x4d, TTA: 0, Saved: best, Fold: [5, 6, 7, 8, 9] ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5381a693b0f4c7d9c5a5be73f68b76a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Inference example: [0.2990355  0.3897496  0.09844354 0.02557188 0.1871995 ]\n",
      "========== Overall ==========\n",
      "Submission example: [0.38396105 0.2279268  0.17651057 0.02770873 0.18389285]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000015157.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000201771.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100042118.jpg</td>\n",
       "      <td>4</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000723321.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000812911.jpg</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_id  label  source\n",
       "0  1000015157.jpg      0    2020\n",
       "1  1000201771.jpg      3    2020\n",
       "2   100042118.jpg      4    2020\n",
       "3  1000723321.jpg      1    2020\n",
       "4  1000812911.jpg      3    2020"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# inference\n",
    "# ====================================================\n",
    "predictions = None\n",
    "for model_name in CFG.models:\n",
    "    for i in range(CFG.tta):\n",
    "        model = CassvaImgClassifier(model_name, pretrained=False)\n",
    "        states = []\n",
    "        for saved_model in [\"best\", \"final\"]:\n",
    "            if CFG.trn_fold[model_name][saved_model] != []:\n",
    "                LOGGER.info(\n",
    "                    f\"========== Model: {model_name}, TTA: {i}, Saved: {saved_model}, Fold: {CFG.trn_fold[model_name][saved_model]} ==========\"\n",
    "                )\n",
    "                states += [\n",
    "                    torch.load(MODEL_DIR + f\"{model_name}_fold{fold}_{saved_model}.pth\")\n",
    "                    for fold in CFG.trn_fold[model_name][saved_model]\n",
    "                ]\n",
    "\n",
    "        if i == 0:  # no TTA\n",
    "            test_dataset = TestDataset(test, transform=get_transforms(data=\"valid\", size=CFG.size[model_name]))\n",
    "        else:\n",
    "            test_dataset = TestDataset(test, transform=get_transforms(data=\"inference\", size=CFG.size[model_name]))\n",
    "\n",
    "        test_loader = DataLoader(\n",
    "            test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True\n",
    "        )\n",
    "\n",
    "        inf = inference(model, states, test_loader, device, CFG.data_parallel[model_name])\n",
    "\n",
    "        LOGGER.info(f\"Inference example: {inf[0]}\")\n",
    "\n",
    "        if predictions is None:\n",
    "            predictions = inf[np.newaxis] * CFG.weight[model_name]\n",
    "        else:\n",
    "            predictions = np.append(predictions, inf[np.newaxis] * CFG.weight[model_name], axis=0)\n",
    "\n",
    "sub = np.sum(predictions, axis=0) / weight_sum\n",
    "LOGGER.info(f\"========== Overall ==========\")\n",
    "LOGGER.info(f\"Submission example: {sub[0]}\")\n",
    "np.savetxt(f\"{OUTPUT_DIR}/predictions.csv\", sub, delimiter=\",\")\n",
    "\n",
    "# submission\n",
    "test[\"label\"] = sub.argmax(1)\n",
    "test[[\"image_id\", \"label\"]].to_csv(OUTPUT_DIR + \"submission.csv\", index=False)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n",
    "train = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = confusion_matrix(train[\"label\"], test[\"label\"])\n",
    "cm = cm / cm.astype(np.float).sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD4CAYAAABPLjVeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9+0lEQVR4nO3dd3gU1frA8e9JoycBJBsgAWlKV6RYEFIghBJS6KDgVaoFUemCqCBVughSLfdaLkgLJCT0KvVaAAWkSEkgG0RCN2Vzfn9sDNkkkADJ7rK/98Mzz5PZOTv7vszsu2fPzM4orTVCCCGsw8nWAQghxP8nUnSFEMKKpOgKIYQVSdEVQggrkqIrhBBW5FLYL1Cm5zcOd3rEucXdbR1CoVDK1hEUPCdHTApISUu3dQiFwr2o0wNvsGIN3sh3zbn10xyr7yDS0xVCCCsq9J6uEEJYlbLvvqQUXSGEY3FytnUEdyVFVwjhWOx8HF+KrhDCscjwghBCWJH0dIUQwoqkpyuEEFYkPV0hhLAiOXtBCCGsSIYXhBDCimR4QQghrEh6ukIIYUVSdIUQwoqc5UCaEEJYj4zpCiGEFcnwghBCWJH0dAtGi3rlmdCzIc5Oin9vPcmstb9ZLK9Ytjhz+z2LR3FXnJ0UHy79hY2/nMe/rjdjujyJm4sTKWnpvP/dT+z4zWijLCzt2rmDqZPHYzKlE9GhEy/36WexPCUlhffeHc6R337F09OTSR9Pp0JFH/b8sIvZM6eRlpqKi6srbw0eRpOnn7FRFma7du7g40njSTelE96xE6/klstIcy4enp5MnmrOJSnpMkPfHsSvhw8TGh7OiFFjMp/zev8+XLx4EZPJRIOnGjJy9BicC3m8bteO7UzOyCOiY2d6982Zx6iRwzjyqzmPKdNmULGiDwCLF85n5fLvcXJ2YvjI0TR9vhnJycm83OsFUlNSSDOZCGoVzGtvvAmA1po5s2eyPjYGZ2cnOnftzgsv9irU/AB+2LWDaZMnkJ6eTlhEJ/7Vu2+OHN8fNZyjR37Dw8OTCVOmU6FiRX49dJDx494nI3j6DnidgBZBnD79B+8Oeyfz+efjztHvtYH0ePGlQs8lV9LTfXBOSjHlpUZ0mLyZ83/dYtPYYGJ+jOPY+auZbYaE1WXVvjN8vukEj1dw579D/HnynUguXUumx/RtJCTdopaPB8uGBlB30CrbJZPBZDIxefxY5i5YgsHbwIvdOuMXEEjVatUz26xa8T3u7u5ERq8ndl0Us2ZMY/LUGXiWLs2sOfMo52XgxPHfeX1AH2I3bbdpLpM+Gsu8heZcXuhqzqVatlxKubsTuW49MdFRzJo+jcnTZlDErQivDRzEiePHOXnid4v1Tp42k5IlS6K1Zsjbb7IhNobWbdsVah4Txo9l/sLPMRgM9OjaCf+AQKpVv53HyuXLcHd3Z23MBtZFRzFz+lQ+njaTkydOEBMdxYrIKBITjfTv8zKRUbG4ubmxaMmXFC9RgtTUVP7VswfPN2tO/SeeZPWqFSQkXGD12nU4OTlx6dKlQssta45TJoxjzvzFGAwGXurRheb+ARb73eqV3+Pu7sHKtbGsXxfFJzOnMvHjGVSrXoOvvlmGi4sLf15MpEfnCJr5BfDoo1X4ZunKzPW3DfInILBloedyR3be083zI0EpVVMpNVwpNTtjGq6UqmWN4P7RsFpZ/jBe58zFG6Sa0lmx5wxtGvpYtNFaU6qoKwDuxd1ISLoFwKEzlzP/PhJ3hWJuzri52P6T8PChg/hUqoSPry+urm4Et2nL1i2bLNps3bKJkNBwAFoEBbN/72601tSsVZtyXgYAqlWvQfLfyaSkpFg7hUyHDx3EN3sum7PlsnkT7cPCAWjZKph9GbkUK16cBk81pEgRtxzrLVmyJABpaWmkpaaiCvnNdPjQQXx9K5vzcHOjddt2ObbJls2bCQ2LACCoVTD79pjz2LplE63btsPNzQ0fH198fStz+NBBlFIUL1Hidh5paZlFYel339J/wOs4OZn3x7JlyxZqfgC/Hj6Ir28lfHzM2yqodVu2bd1s0Wb7ls20Cw0DIDAomP379qC1pmixYri4mPtpyckpuW6P/Xv34OPrS/kKFQs9lztycs7/ZIvw7rZQKTUc+A5QwL6MSQHfKqVGFH54ZuVLFyP+rxuZ8+f/ukn50sUt2kxecYguTatweFY4/x3iz/CvDuRYT2hjX345fdkubup3MdGIt3f5zHkvgzeJRmO2NomZbVxcXChZshRJSUkWbTZtiKVmrdq4ueUsWtaSmGjEkCUXg8Gbi4nGbG3yziU3r/XrTQu/phQvUYKWrYILNO7sEo1GvMt7Z857GQwYjdnzMFrmUaoUSUmXMRqNGLxvP9fgbcjcniaTiS4dwgho9hzPPPsc9es/AUDcuXPExkTTvUsHXuvfhzNnThdqfmDepyzi9DJwMZccDdm21ZWMbXX44C90iQihe6cwRox+P7MI/2N9TDTBrQvv20i+KKf8TzaQ16v2BhprrSdprf+TMU0CmmQsy5VSqp9S6oBS6kDy8c13alagOj77KN/uOEXdQavoOnUrnw14zuJbRs2KHrzf9Une+XyfVeKxhpMnjjN7xjRGvf+hrUMpNHMXLGbDlh2kpKSwf+8eW4dzX5ydnVm6YjXrN2/j8KGDHD9uHkZJSUnBrUgRvl26gg6duvD+6HdtHGne6tZ/gqUr1/LlN0v5YvFCkpOTM5elpqawfdtmWhTyh2OelMr/ZAN5Fd10oEIuj5fPWJYrrfUCrXUjrXWjIjUCHyQ+AC5cvkXFMiUy5yuUKc6Fyzct2rzoV5VVe88CsP/EnxRxdaZsqSLm9qWL8dWgZrw2fzenE68/cDwFoZyXgYSEC5nzicYEvAyGbG28MtukpaVx/fo1PD09ATAmJDD4rTcYO2Eyvr6VrBZ3bry8DBiz5GI0JmQOf9xuc+dc8lKkSBH8A1rk+Kpf0LwMBhIuJGTOJxqNGAzZ8zBY5nHtGp6epTEYDBgTbj/XmGDMsT3d3d1p3ORpfti5AzD3hlu0DAKgRcsgjv9+rFDyyqqcl5dlnIlGyuWSozHbtvLItq2qVK1G8eLFOXnieOZjP+zcQc2atSlb9pHCSyA/HvKe7lvAJqXUOqXUgowpBtgEDCr06DL8eOoSVb1LUalcCVydnejwTGVifoy3aBN36SbN65h3nscquFPE1Yk/rybjXtyV74b4M3bpz+w9/qe1Qs5Tnbr1OHfmDPFxcaSmphC7Lho/f8sPKD//QNZGrgLMwwiNmzyDUoprV6/y5uv9GfjWYJ5s8JQNordUp249zp61zMU/IFsuAYGsWb0KgI3rY2n89DN3HaO9efMGFy8mAuY3/s7t23i0StVCywH+yeM0cXHnSE1JISY6Cr9sefgHBBK52nzQaMP6WJpk5OEXEEhMdBQpKSnExZ3j7NnT1K1Xn7/++ourV80HfP/++2/27P4hM4+AwJbs37cXgAP791G58qOFmh9A7TqW22pDTDTN/QIs2jTzDyAqcjUAm7Psd/FxceYxaeDC+XhOnz5FhSxjt7HromjVxsZDC2D3RVdpre/eQCknzMMJ//zvxgP7tdam/LxAmZ7f3P0F8qnlExWY8MJTODspvt5+iumRvzKyQz1++uMvYn6K5/EK7szs/TQlirqgNXzw3U9sOZzA4LA6vNW+DqcSrmWuq+OUzfx5Nfkur3Z35xZ3L4iU2Ll9G1OnTCDdlE5oREf69BvAvDmzqV2nLn4BgSQnJ/PeyGEcPXoEDw8PJk6Zjo+vL4vmz2PJ4gVUqlQ5c11z5y+mzAMeiHmQb1s7tm9j6mRzLmERHenTfwBzM3Lxz8hl9MhhHDtyBHcPDyZ9bM4FoG2rQG5cv0Fqaiql3Esxd8FiPD08efP1AaSmpJCuNY2aNGHIsJE5xhDz4nSPSe3Yvo0pkyaQnm4iPKIjffu/yqefzKJOnbr4B7YgOTmZUSOGcjQjjylTZ2TmsXD+PFatXI6zszPDRrzL8838+P3YUUa/O4L0dBPp6ZpWwa0Z8NobAFy9epV3hw/hwoULFC9enNFjPuTxmjXzFeeDHJfYtWMb06dMxJSeTmh4B17pO4DPPp1NrTp18fM3b6v3Rw3n2NEjuLt7MH7KNHx8fIles5ovlizExdUVJ6Xo0/81/DPOUrh18ybtWweyKmoDJUuVuu/Y3Is6PfB3/mJh8/Ndc26t7m/1MYY8i+6DKqiia08KqujaGzs/0+a+3GvRfVjYw8HgwlAgRTd8Qf6L7qp+Vt9BHorzdIUQIt/kxxFCCGFFdv7tRoquEMKhFPaPaB6UFF0hhEORoiuEEFakHvxYXKGSoiuEcCjS0xVCCCuSoiuEEFYkRVcIIazJvmuuFF0hhGORnq4QQljRPxeFt1dSdIUQDsXee7r2/ZEghBD3St3DlNeqlGqtlDqmlDqR291ylFKVlFJblFI/KaUOKqXa5rVOKbpCCIeilMr3lMd6nIFPgTZAbaC7Uqp2tmajgaVa6wZAN2BuXvFJ0RVCOJSCKrqYryN+Qmt9Smudgvl+kWHZ2mjAPeNvD+B8XiuVMV0hhEO5l58BK6X6Af2yPLRAa70g4++KwLksy+KAp7Ot4gNgvVJqIFACyPPe84VedE/O71rYL2F1j4RMtXUIheLS2iG2DqHg2fcxlfvmoGkViHs5kJZRYBfk2fDOugNfaK2nKaWeBf6tlKqrtb7jVealpyuEcCgFePZCPOCbZd4n47GsegOtAbTWu5VSRYFHgMQ7rVTGdIUQDqUAx3T3AzWUUlWUUm6YD5RFZmtzFmiR8bq1gKLAxbutVHq6QgiHUlA9Xa11mlLqDSAWcAaWaK1/VUqNBQ5orSOBwcBCpdTbmA+q/UvnceNJKbpCCMdSgAPeWutoIDrbY2Oy/P0b0PRe1ilFVwjhUORnwEIIYUX2/jNgKbpCCMdi3zVXiq4QwrFIT1cIIaxIiq4QQliRFF0hhLAiuQW7EEJYkfR0hRDCiqToCiGEFdl5zZWiK4RwLNLTFUIIK3KSA2lCCGE9dt7Rtd/r6e7etYMu4W3pFBrMV0sW5liekpLCqOHv0Ck0mFd6duX8ectrCydcOE/Acw35+qslmY99+58v6d6xPT06hfLeiCEkJycXeh53E9ToUX5Z3JvDn/dhSNcmOZZX8nInenIX9n32L2I/7krFR0pmLls9vhMXVgxk+dgO1gw5V7t27iC8fWtC27ZiyaKcF+FPSUlh+JC3CW3bip49unA+Pg6ApKTL9H2lF881eYpJ48daPCc1NYVxH7xHWEgwEe3bsHFDrHVy2bGd0HbBhLQOYvHC3HMZOvgtQloH8UK3zsRn5AKweOF8QloHEdoumF07d2Q+fvXqVQa/9SZhIa0Jb9+GX37+CYBjR4/Ss0dXOoa3Z+BrA7h+/XrhJwj8sGsHHULbEB4SzBeLc39vjRz6NuEhwbz0QlfOx5vfW4cPHaRHlwh6dImge+dwtmzaYPE8k8lEjy4deOuNAVbJ406cnFS+J5vEZ5NXzYPJZGLqpI+YMWc+3y5fw/qYaP44ecKiTeSq5biXcuf7yFi6v/ASn86aZrF81rQpPNu0WeZ8YqKRpd/+h8+/XsY330eSnm5iQ6zFFdusyslJMfONIMJGfU+Dvkvo7F+LmpXKWrSZ2M+frzf+SpMBXzDh692MfaV55rIZy/bRe4rt4v+HyWRi0vixzJm7kOWr1xKzLoqT2bbVqhXfU8rdncjo9bzQ8yVmzTBvqyJuRXjtjUG8PWRYjvUuWvAZZcqUZfXaWJavjqJho5wfSoWRy4TxY5n72SJWRkYRE72Wkycsc1m5fBnu7u6sjdnAi73+xczp5ls3nTxxgpjoKFZERjF3/iImfPQhJpMJgCkTx9P0+WasXhvDsuWrqVK1GgAfjhnFoLcHs3zVGgJbtuSLJYuskuPkCeOYPXcBy1auITYmilPZttfqld9Tyt2DVWtj6fFiLz6Zac6xevUafPXNMr5ZupJP5i5gwrgPSEtLy3zet1//mypVqxZ6DnlRKv+TLdhl0f3t8CF8fCtR0ccXV1c3goLbsH3rZos2O7Zupm37cAACWrbiwL49/HPt4G1bNlKhYkWqVKtu8RyTyURy8t+kpaXx999/U66cl1XyyU3jx8tz8vxlTidcITUtnWXbjhLynGW8NSuVZdvPZwHY9vNZQp69vXzrz2e5djPFqjHn5vChg/hWqoSPr3lbBbdpy9YtmyzabN2yifah4QC0DApm397daK0pVrw4DZ5qSBE3txzrXb1yBa/0Md8v0MnJidKlS1snF9/K5lzc3Gjdtl2OXLZs3kxoWAQAQa2C2bfHnMvWLZto3bYdbm5u+Pj44utbmcOHDnLt2jX+97/9RHTsBICrmxvu7uabx545c5qGjRoD8OyzTdm0YX2h5/jr4YP4+lbCJ+O91ap1W7Zle29t27KZkFDzTW9bBAWzL+O9VbRYMVxczCOSyckpFgesjMYEdu3YRnhEp0LPIS8FeOeIQmGXRfdiohEvg3fmvJfBm4sXE3O0MXib27i4uFCyZCmuJCVx8+YN/v35Ynr3f82ivZeXgRd6vUx4mxaEBPlRomRJnn72nq49XKAqPFKSuIvXMufjL16jYtmSFm0OnUokrGkNAMKa1sC9RBHKlCpq1TjzkphoxOBdPnPeYPDmotGYrU0i3hlt/tlWSUlJd1zntatXAfh0ziy6d+nA0HcGcenPPws++GwSjUa8y2fd7wwYc+RitMylVCmSki5jNN7eHwEM3gYSjUbi4+IoXboMY0aNpEvHcD4YM4qbN28CUK16DbZsNhf19bExJCRcKOwUSUxMtIjTy8scp2Wb29s063sL4PDBX+gSEUK3TmGMHP1+ZhGeNmUib749BGUH17J12J6uUurlggykoCz67FO6vdiL4sVLWDx+9eoVtm/dzIq1G1i7fit/37rFuqjstzuyLyMXbKVZfV92z+1Fs/q+xF+8hin9rncCcQhpJhNGYwJPPNmAb5euoP4TTzJj2hRbh3VfTKY0jh75jc7durN0+SqKFSuWOe794bjx/Pe7b+jWuQM3b97A1TVnj9/e1K3/BEtXruWrb5by+eKFJCcns2PbFsqUKUOt2nVsHR5g/maU38kWHuTshQ+Bz3NbkPVe8tM/mce/Xul7Tysu52Ug0ZiQOZ9oTMgxFFDOy4AxIQEvgzdpaWlcv34ND09Pfj18kM0b1zNn5jSuX7uGk5PCza0IZcqWpUKFipQuUwYA/8AgDv3yM23ahd5TbAXl/J/X8SlXKnO+YrlSxF+yPJBy4a8bdBu7GoASRV0Jf/4xrtyw7cG/7Ly8DBiz9NCMxgTKGQzZ2niRkHABg/ftbeXp6XnHdXp6elK0WDFatGwFQFBwa1atXF4o8VvEaTCQcCHrfmfEkCMXg2Uu167h6Vkag8G8P/7DmGDEy2DAYPDGYPCmfv0nzLm0ap1ZdKtUrcb8heYDvadP/8H2bVsLOUPztsgaZ2KiOU7LNuZtasj23sqqStVqFC9enJMnjvPLzz+xfesWdu3cTkpyCtdvXOe9kcMYN9E2H5QP9dkLSqmDd5gOAYY7PU9rvUBr3Uhr3eheCy5ArTp1OXf2DOfj40hNTWFD7Dqa+QdYtGnmF0D0mlUAbNm4nkaNn0Ypxfwl/2FV9EZWRW+k6ws9eal3Pzp3ewGDd3kOH/qFv2/dQmvNgX17eLSK7Qb9Dxy7QPWKpans7YGrixOd/WoStdvygEZZ92KZO9DQbk/zZewhG0R6d3Xq1uPsmTPEx5m3Vey6aPz9Ay3a+PkHsiZyFQAbN8TSuMkzdx1PU0rR3C+AA/v3AbBvz26qZhx8Kkx16tbj7NnTxMWdIzUlhZjoKPwCLHPxDwgkcvVKADasj6XJ0+Zc/AICiYmOIiUlhbi4c5w9e5q69erzSLlyGLy9Of3HKQD27tlN1WrmXC5dugRAeno6C+fPo3PXboWeY+069Th39vb2Wh8TTXM/y/dWc/8A1kaaP+w3Zdle8XFxmQfOLpyP5/TpU1SoUJE3Br1D9IatrFm3ifGTp9G48dM2K7hg/2O6efV0DUAwcDnb4wr4oVAiwjyONGT4KAa91pf09HRCwiKoWq0GC+Z+Qs3adWjuH0j78I58OHo4nUKDcXf3ZNykqXddZ916TxDYshUv9eiEs7Mzj9WsRXjHLoWVQp5M6Zq352xkzYROODs58WXsIY6cucR7vZry4+8JRO05SfMnfBn7SnO01uw8FMdbczZmPn/jtO485luGksVcOfH1AAZMj2Hj/05bPQ8XFxeGv/serw3oTbopnbCIjlSrXoO5c2ZTu05d/AMCCe/QidEjhxHathXuHh5MmjI98/ltgwO5cf0GqampbNm8ibkLFlOtWnUGvT2Y0SOHM3XyBEqXKcMH4yZYJZeRo8bwar8+pKebCI/oSPXqNfj0k1nUqVMX/8AWRHTsxKgRQwlpHYS7hwdTps4AzEf2W7VuQ0RoW5ydnXl39BicnZ0BGPHue4wcPoTU1FR8fHwZ+9FEAGKi1/Ldt98A0KJlEOERHa2S49CRoxn4ah9M6emEhnegWvUafPbpbGrVqYuffyBhEZ0YM2o44SHBuLt7MGGK+WyTn3/6H18uWYiLqytKKUa8OwZPKxzgvFf23tNVd7tbsFJqMfC51npnLsu+0Vr3yOsFLt80OdwgZIXw6Xk3eghdWjvE1iEUOHv/ddL9Sk1Lt3UIhaJU0QffYA3Hbcl3zfnfewFW30Hu2tPVWve+y7I8C64QQlibvfd05WfAQgiHYu/fbqToCiEcilxlTAghrMjOa64UXSGEY5GerhBCWJGd11wpukIIxyIH0oQQwopkeEEIIaxIiq4QQliRnddcKbpCCMciPV0hhLAiO6+5UnSFEI7F3s9esP29NYQQogA5KZXvKS9KqdZKqWNKqRNKqRF3aNNFKfWbUupXpdQ3ea1TerpCCIdSUMMLSiln4FMgCIgD9iulIrXWv2VpUwMYCTTVWl9WSuV5t1vp6QohHEoB3jmiCXBCa31Ka50CfAeEZWvTF/hUa30ZQGudSB6k6AohHIqTyv+klOqnlDqQZeqXZVUVgXNZ5uMyHsvqMeAxpdQupdQepVTrvOIr9OGFIq6OV9f/ihpq6xAKRZnw2bYOocBdWjXQ1iEUDvs+VmRT93IgTWu9AFjwAC/nAtQA/AEfYLtSqp7WOumO8T3AiwkhhN1R9/AvD/GAb5Z5n4zHsooDIrXWqVrrP4DfMRfhO5KiK4RwKPcyvJCH/UANpVQVpZQb0A2IzNZmFeZeLkqpRzAPN5y620rl7AUhhEMpqF+kaa3TlFJvALGAM7BEa/2rUmoscEBrHZmxrJVS6jfABAzVWl+623ql6AohHEpB/iJNax0NRGd7bEyWvzXwTsaUL1J0hRAOJT8/erAlKbpCCIdi7z8DlqIrhHAodt7RlaIrhHAsMrwghBBWZN8lV4quEMLByEXMhRDCiuz8OJoUXSGEY5GzF4QQwopkeEEIIazIzju6UnSFEI5FerpCCGFF9l1ypegKIRyMs52PL9jt9XR37dxBeEhrQtu0YsminBd2T0lJYfjgtwlt04qe3btwPj4OgKSky/R9uRfPNX6KSePHWjzn9f596NIhjI5hIXz04fuYTCar5HInu3ZuJywkmPZtgu6Y47DBb9G+TRAvdu9MfJYc+7zck2cbN2BithztQVDDyvwyvyeHF/ZiSOeGOZb7litJzMQO7J7dnX1zehDcqDIAri5OzH+rJfs/7cHeT7rTrF72O6NYl6Pugz/s3EGH9m0IbxfMF4sX5liekpLCyKFvE94umJd6dOV8vPm63Xt27+LFrh3p2iGUF7t2ZP/ePQDcuHGDHp0jMqcWzZ9l2uQJVs0pqwK8R1qhsMuiazKZmPTRWObMW8jyyLXEREdx8uQJizarVnxPKXd3Itet54WeLzFr+jQAirgV4bWBg3h7yLAc6508bSZLV6zm+1VruHz5LzbExlgln9yYTCYmfjSWT+ctYkVkFDHRa3PkuHLFMtzd3VmzbgMv9vwXs6ZPBcw5vj5wEO/kkqOtOTkpZr7qT9j7q2nw6n/o3PwxavqWsWgzvFsTlu84zrNvfkuvyTHMei0AgFeC6wLQ+PVvCBm9ikl9mtnsd/SOug+aTCYmTxjH7HkLWLZqDbHrojiVLa/VK76nlLsHq6Ji6dGzF5/MNO93np6lmfHJPP67IpIPPprImFHDAShRogTfLFuZOZUvX4GAFkFWzSsrpfI/2UKeRVcpVVMp1UIpVTLb43negO1+HT50EN9KlfDx9cXV1Y3gNm3ZunmTRZutmzfRPiwcgJatgtm3dzdaa4oVL06DpxpSpIhbjvWWLGlOIS0tjbTUVJsOuJtzrJwlx3a55LiZ9mERQG45NsKtSBFbhH5XjR8zcPJ8EqcTrpKals6y7ccJeaaqRRutNe7FzdvHo4QbF/66AUDNSmXY+ou5t3jxyi2uXE+mYQ2DdRPI4Kj74K+HM/LyMefVqnVbtm3ZbNFm29bNhISab3rbIiiYfXv3oLWmZq3alPMy32G8WvUaJP+dTEpKisVzz5z+g8t//UWDho2sk1AunJTK92ST+O62UCn1JrAaGAgcVkplvf1woX1/SEw0YvAunzlvMHhzMdGYrU0i3hltXFxcKFmyFElJSXmu+7V+vWnh15TiJUrQslVwgcZ9LxITjXh7e2fOGwwGEnPkaMwlx8tWjfNeVShbkrg/r2fOx/95nYplS1i0Gf/1XroFPM6JL19h5YehvPPZVgAO/XGRkGeq4OykqGxwp0F1L3wesfistxpH3QcTjYkYDLf3O6/c9jujEYPBMq8r2fLatGE9NWvVws3N8oNlfUw0QcFtbNqhedh7un2BhlrrcMz3AXpPKTUoY9kdQ856W+PcxsJsae6CxWzYsoOUlJTMMSlhXV38Huc/G49Q/aUlRLwfyeLBwSgFX67/jfg/r7NrVjc+7tecPUcuYErXtg63wD3s++DJE8f5ZOY03h3zYY5l62PWEdy2nQ2iuu1hH9N10lpfB9Ban8ZceNsopaZzl6KrtV6gtW6ktW70Sp9+d2p2R15eBowJFzLnjcYEynkZsrXxIiGjTVpaGtevX8PT0zNf6y9SpAj+AS3YumVT3o0LiZeXgYSEhMx5o9GIV44cDbnkWNqqcd6r85euW/ROKz5SkvhLNyzavNSqNst3HAdg79EEiro584h7MUzpmmELd/DMwG/pMm4tniWLcDw+yZrhZ3LUfdDL4IXReHu/S8xtvzMYMBot8/LIyMuYkMDQtwfy4fhJ+PhWsnje78eOYjKlUat2ncJNIg/OSuV7soW8iq5RKfXkPzMZBTgEeASoV1hB1albj7NnzxAfF0dqagqx66LxDwi0aOMXEMia1asA2Lg+lsZPP3PXT66bN29w8WIiYN6Rdm7fxqNVqt6xfWEz53ia+LhzGTlG4ZdrjiuB/OVoDw78bqR6RU8qG9xxdXGic/MaRO21vDnquYvX8H/SfGfrx31LU9TVmYtXblGsiAvFi5jPYgx80pc0UzpHz/1l9RzAcffB2nXqce7M7bzWx0TT3D/Aok1z/wDWRq4GYNOGWBo3Med17epV3npjAG8MeocnGzyVY92x66IIbm3bXi4U6N2AC4Uy31ftDguV8gHStNYJuSxrqrXeldcL3Ey9ywvcxY7t25g6eQLppnTCIjrSp/8A5s6ZTe06dfEPCCQ5OZnRI4dx7MgR3D08mPTxdHx8zW/ktq0CuXH9BqmpqZRyL8XcBYvx9PDkzdcHkJqSQrrWNGrShCHDRuLicu+nKqsCOv16x/ZtfDx5AukmE2ERHenb/1XmzpmVkWMLkpOTGTVyaGaOkz+ekZljm1aB3Lh+PTPHeQuWUK1a9QeKp0z47IJIi+BGlfm4X3OcnZz4csOvTPnvAd578Wl+PJ5I1N4/qOlbhrlvBlKiqCsaGLVkF5t+Okslr1KsGRdOutacv3SdV2du4uzFaw8Uy6VVA+/7ufa8Dz7IsMvOHduYPmUiJlM6oeEd6N1vAJ99Optatevil5HXmHeHc+yoOa8JU6bh4+PLogXz+GLRQipVrpy5rjmfLaJM2bIAhLUJYtbc+Q/0QVKqyIOXwncij+b7P2d6aE2rl967Ft2CcL9F154VVNG1NwVVdO3JgxRde+aIY91QMEV38Jpj+f7Pmdb+cau/meUXaUIIh2LnP0iToiuEcCx2fthDiq4QwrG42HnVlaIrhHAodl5zpegKIRyL3IJdCCGsyM5rrhRdIYRjkbMXhBDCiuz9IuZSdIUQDsXOa64UXSGEY7H3X4xK0RVCOBTp6QohhBVJ0RVCCCuy98uf2uWNKYUQ4n45O+V/yotSqrVS6phS6oRSasRd2nVUSmmlVJ43h5OerhDCoRTUL9KUUs7Ap0AQEAfsV0pFaq1/y9auFDAI2Juv+AokOiGEsBMFeOeIJsAJrfUprXUK8B0Qlku7ccBk4O/8xFfoPV17P33jfmgc8wLSicvfsHUIBa5sE8e8iPmlvZ/YOgS7dS8dXaVUPyDrjRwXaK3/uZtuReBclmVxwNPZnv8U4Ku1jlJKDc3Pa8rwghDCoTjdQ0cvo8De1y3LlVJOwHTgX/fyPCm6QgiHUoAnL8QDvlnmfTIe+0cpoC6wNeOMCW8gUikVqrU+cKeVStEVQjgUl4I7UXc/UEMpVQVzse0G9Phnodb6CuY7owOglNoKDLlbwQU5kCaEcDBK5X+6G611GvAGEAscAZZqrX9VSo1VSoXeb3zS0xVCOJSCvIi51joaiM722Jg7tPXPzzql6AohHIqd/yBNiq4QwrHY+5ipFF0hhEORe6QJIYQVSdEVQggrsu+SK0VXCOFg7LyjK0VXCOFY7P16ulJ0hRAORc5eEEIIK5IDaUIIYUUyvCCEEFYkwwtCCGFF9t7TtfcPhUy7dm4nLCSY9m2CWLIo5zWHU1JSGDb4Ldq3CeLF7p2Jj48DICnpMn1e7smzjRswcfxYa4edw66dOwgPaU1om1Z3zGP44LcJbdOKnt27cD5LHn1f7sVzjZ9iUpY8bt26xcBX+xPRvg0dw0KYNWOa1XK5kx927aBDaBvCQ4L5YvHCHMtTUlIYOfRtwkOCeemFrpyPN1+i9PChg/ToEkGPLhF07xzOlk0brB36HQU9V4tfVr7H4dXvM+TloBzLK5UvTfRnA9n335HELhxERS/PzGUfvRnGgWXvcmDZu3Rq9ZQVo87drp07CG/fmtC2d9kHh7xNaNtW9OyRbR98pRfPNbHcBwFSU1MY98F7hIUEE9G+DRs3xFoll9yoe5hs4aEouiaTiYkfjeXTeYtYERlFTPRaTp48YdFm5YpluLu7s2bdBl7s+S9mTZ8KQBG3Irw+cBDvDBlmi9AtmEwmJn00ljnzFrI8ci0x0VE58li14ntKubsTuW49L/R8iVnTzUW0iFsRXhs4iLdzyaPXyy+zcs06vvt+Bb/89CM7d2y3Sj65MZlMTJ4wjtlzF7Bs5RpiY6I4lS3H1Su/p5S7B6vWxtLjxV58MtO8rapXr8FX3yzjm6Ur+WTuAiaM+4C0tDRbpGHByUkxc0QXwt6YS4OOH9G5dUNqVvW2aDPx7Qi+jtpHk64TmbBgHWMHmq/81/r5OjxZy5enu02iec+pvNWrBaVKFLVFGkDGPjh+LHPmLmT56rXErLvLPhidsQ/OyLIPvpH7PrhowWeUKVOW1WtjWb46ioaNmlgln9w4K5XvyRYeiqJ7+NBBfCtVxsfXF1dXN4LbtGPr5k0WbbZu3kz7sAgAWrYKZt/e3WitKVa8OA2eaoRbkSK2CN2COY9KWfJom0sem2gfFg7klkdDihRxs2hfrFgxGjd5BgBXVzdq1qpNojHBKvnk5tfDB/H1rYSPjznHVq3bsm3rZos227ZsJiTUfH+/FkHB7Nu3B601RYsVw8XFPOKVnJxiN18TG9d9lJPn/uR0/CVS00wsi/2REP/6Fm1qVi3Ptn3HANi2/3dC/OsBUKuqNzt/PIHJlM7Nv1M4dDyeVs/VsnoO/8h1H9ySbR/cson2oeEAtAzKZR90c8ux3tUrV/BKH/OtxpycnChdunSh53InBXU93cKSZ9FVSjVRSjXO+Lu2UuodpVTbwg/ttsREI97et3sWBoOBxERjLm3KA+Di4kLJkqVISrpszTDzlJhoxJARI4DB4M3FHHkk5pJHUr7Wf+3qVbZv20KTp58tsJjvVWJiIoYs28rLy0CiMee2MmTL8UpGjocP/kKXiBC6dQpj5Oj3M4uwLVXw8iDOeHtfijdepmI5D4s2h36PJyzwSQDCAp/AvWQxyniU4ODv5iJbrKgrZT1L4NfoMXy8bVeQct0Hc2yfe9sHr129CsCnc2bRvUsHhr4ziEt//lnwweeTuod/tnDXoquUeh+YDcxTSk0E5gAlgBFKqVFWiE/kU1paGiOGDab7Cz3x8fXN+wl2qm79J1i6ci1ffbOUzxcvJDk52dYh5cvIGStp1rA6u78dTrOG1Yk3XsZkSmfTnqPE7PyNLV8M5suJL7P34B+YTOm2DrdApZlMGI0JPPFkA75duoL6TzzJjGlTbBbPw97T7QQ0BZoDrwPhWutxQDDQ9U5PUkr1U0odUEodWJzLQP298vIykJBw+yuz0WjEy8uQS5sLgLkAXb9+DU9P2/UocuPlZcCYESOA0ZhAuRx5eOWSh2ee6/7ogzFUqlSZF3q+VKAx3ysvLy+MWbZVYqIRL0PObWXMlqNHthyrVK1G8eLFOXnieKHHnJfziVfwMdzelyoaShN/8YpFmwsXr9BtyCKe7T6Z9+esAeDK9VsATFkcyzPdJhHy6hyUUhw/m2i94LPJdR/MsX3ubR/09PSkaLFitGjZCoCg4NYcOfJbwQefT06ofE+2ie/u0rTWJq31TeCk1voqgNb6FnDHj2ut9QKtdSOtdaPeffrdqVm+1albj7NnTxMfd47U1BRi10XhFxBo0cYvIJA1q1cCsHF9LI2ffsZuxgT/Yc7jDPFxcRl5ROOfax6rgPzn8ensmVy7fo2hI94trNDzrXadepzLkuP6mGia+wVYtGnuH8DayNUAbNoQS+Mm5hzj4+IyD5xdOB/P6dOnqFChotVzyO7Ar2eoXqkclSuUxdXFmc7BTxG19aBFm7KeJTK309BXgvly9R7AfBCujEcJAOrWqEDdGhXYuPuodRPIok7depw9k20f9M+2D/oHsiZyFQAbs2yfO1FK0dwvgAP79wGwb89uqlatVmg55MXee7pKa33nhUrtBQK01jeVUk5a6/SMxz2ALVrrPM9/uZXKnV/gHuzYvo2PJ08g3WQiLKIjffu/ytw5s6hdpy7+AS1ITk5m1MihHDtyBHcPDyZ/PCPza3abVoHcuH6d1NRUSrmXYt6CJVSrVv2+Y9EPkNKO7duYOnkC6aZ0wiI60qf/AObOmZ2RRyDJycmMHjksM49JH0/PzKNtq0BuXL+RmcfcBYspWaIkrVv6U6VKVVwzDnB07f4CHTp1vufYTKYC2VTs3LGN6VMmYkpPJzS8A737DuCzT2dTq05d/PzNOY4ZNZxjR4/g7u7BhCnT8PHxJWrNar5cshAXV1eUUvTt/xr+gS0fKBavZ98skJyCn6/Nx0M64eyk+HL1HqYsjuW9V9vx429nidp2iIiWTzJ2YChaw84fT/DWxKWkpKZRxM2F3d8OB+Da9b8ZOP47Dv4en8er5e3S3k/u+7k7tm9j6pQs+2C/O+yDRzP2wSlZ9sHgLPtgKfM+WK1adc6fj2f0yOFcv3aV0mXK8MG4CZQvX+GeYyvu9uClcMORP/O9IwfVesTqpTevoltEa51jUE0p9QhQXmt9KK8XKKiia08epOjas4IquvakoIquvXmQomvPCqLobjqa/6Lboqb1i+5dDw3nVnAzHv8TsN3hSSGEuANbnZWQX7Y/H0cIIQqQnR3KyUGKrhDCoUhPVwghrMjJvmuuFF0hhGORi5gLIYQV2XfJlaIrhHAw0tMVQggrsu+SK0VXCOFo7LzqStEVQjgUGV4QQggrsu+SK0VXCOFo7LzqStEVQjgU+UWaEEJYkZ0P6T4cN6YUQoj8KshbsCulWiuljimlTiilRuSy/B2l1G9KqYNKqU1Kqcp5rVOKrhDCoSil8j3lsR5n4FOgDVAb6K6Uqp2t2U9AI611feB7IM+bw0nRFUI4lAK8XU8T4ITW+pTWOgX4DgjL2kBrvSXjdmYAewCfvFZa6GO615PTCvslrK64m7OtQygUTvZ+eab78Ne+ObYOoVCU6fzgN3y1R7dWPfg9Fe9lL1ZK9QOyvugCrfU//7kVgXNZlsUBT99ldb2BdXm9phxIE0I4lnuouhkF9oE/wZRSLwKNAL+82krRFUI4lAI8ZSwe8M0y75PxmOXrKdUSGAX43ekWZ1nJmK4QwqEU4JjufqCGUqqKUsoN6AZEWr6WagDMB0K11on5iU96ukIIh1JQ5+lqrdOUUm8AsYAzsERr/atSaixwQGsdCXwMlASWZZwNcVZrHXq39UrRFUI4lIL8RZrWOhqIzvbYmCx/t7zXdUrRFUI4FHv/RZoUXSGEQ7HzmitFVwjhYOy86krRFUI4FLmIuRBCWJF9l1wpukIIR2PnVVeKrhDCochFzIUQworsfEhXiq4QwrHYec2VoiuEcCx5XZzc1qToCiEcip3XXCm6QgjHYuc1134v7bjnhx1079COrmGt+ffnC3MsT0lJYcyIwXQNa03fXt24cP72ZS5PHD9G/3/14MXOofTqEk5ysvkSl6mpKUz+6H26RbSlR4cQtm5ab7V8/rFr5w4i2rcmtG0rPl+U89rJKSkpDB/yNqFtW9GrRxfOx8cBsOeHXfTo0oEuEe3p0aUD+/buyXzOnNkzaNPSn6ZNnrJaHlndb05JSZfp90ovmjZ5iknjx1o8JzYmmi4dQukUHsKs6VOtkgfArp3bCQsJpn2bIJbcIZdhg9+ifZsgXuzemfiMXAAWL5xP+zZBhIUE88OuHZmPf/3vL+kYHkKHsHb8599fWKzv26//TXj71nQIa8eMaXneXqvABTXw4ZdPu3B4XleGdHgix3LfR0oQMy6E3dM7sG9mR4Ibmi8v2615dfbM6JA53VjRl/pVylo7/NwV5J0pC4Fd9nRNJhPTJ41nxtyFeBkM9OnZlef9AqhStXpmm7WrllPK3Z3/ro5hY2w082ZPZ+ykaaSlpTFu9AhGj5tIjcdqciUpCRcXc5pfLV5A6TJl+G5lNOnp6Vy9csXqeU0eP5a5C5Zg8DbwYrfO+AUEUrXa7bxWrfged3d3IqPXE7suilkzpjF56gw8S5dm1px5lPMycOL477w+oA+xm7YD0NwvgK7dXyC8XWur5vOgORVxK8Krbwzi5InjnDj+e2b7pKTLzJr2MV//dzmly5RhzKjh7N2zm6efebbQc5n40Vg+W/g5Bm8DL3TthF9AINWy5LJyxTLc3d1Zs24DMdFRzJo+lSnTZnLy5Ali10WxfHUUFxON9O/zMqujYvnj1ElWLF/Gf75dhqurK68P6ENzvwAqVarM/n172LplE0uXR+Lm5sZfly4Van7ZOTkpZvZ/nnbvRxF/6QY7P45g7b4zHI1LymwzvMtTLN91koUxR6jp48mqMW2o2e9bvtt+gu+2nwCgTuXSLB0ZzME/rBv/ndj7KWP33NNVSn1VGIFkdeTXQ/j4+lLRxxdXVzdatmrLzq1bLNrs3LaZNiHme8T5t2jF//btQWvN/j0/UK3GY9R4rCYAHp6eODub72kWFbmSni/3BcDJyQnP0qULOxULhw8dxKdSJXx8zXkFt2nL1i2bLNps3bKJkNBwAFoEBbN/72601tSsVZtyXgYAqlWvQfLfyaSkpABQ/4knKVfOy6q5/ONBcipWvDgNnmqIm5ubRfv4uDh8K1emdJkyADR55jk2byz8byWHDx3Et1LlLLm0Y+vmbLls3kz7sAgAWrYKZl9GLls3byK4TTvc3Nyo6OOLb6XKHD50kFOnTlKvXn2KFSuGi4sLDRs1ZlNGLkv/+y0v9+6XmX+ZstbtKTauUY6TF65w2niN1LR0lu08ScjTj1q00Rrci5nj8yjhxoW/buRYT5dm1Vm246Q1Qs6XAryIeaG4a9FVSkVmm9YAHf6ZL6ygLiYa8TKUz5wvZzBw8aLRss3FRLwM3gC4uLhQomQpriQlce7saZRSvPN6X17p0Ymvv1wMwLVrVwFYNO8TXunRidHD3uavS38WVgq5uphoxNv7dl5eBm8SjdnySkzMbOPi4kLJkqVISkqyaLNpQyw1a9XOUaxsoaByysrXtxJn/viD8/FxpKWlsXXzRhISLhRK/FklJhrx9vbOnDcYDCQmGnNpkz2Xy3d8bvXqj/Hjj/8jKekyt27dYueO7RgTEgA4c/o0P/7vAC9270zvf73I4UMHCz3HrCqUKUHcn7eLaPylG1QsU8KizfjvDtDNvwYnFvVg5XtteGfhDznW0+n5aizdcaLQ480vJ5X/ySbx5bHcB7gKTAemZUzXsvydK6VUP6XUAaXUga+W5ByPLUxpaSYO/vwjYz6awtzF/2b7lk0c2LcHU5qJRGMCdes/yZJvvqdu/Sf4dKb1xgoLyskTx5k9Yxqj3v/Q1qEUGncPD0a+9z4jhr5D75deoEKFijg7PZx3YK5arRovv9KHV/v15vUBfXj88Zo4OZnfdiaTiatXr/Dvb5by1uBhDBvyFlprG0dsqUuz6vxn8zGq9/mGiHHrWPxWgEUPsXGNctxMTuO3s5dtF2QO9j2om1fRbQT8D/NN165orbcCt7TW27TW2+70JK31Aq11I611o16v9L3noMp5GUg03u7ZXDQaKVfOYNmmnBeJRnOPIS0tjRvXr+Hh6YmXwcATDRriWbo0RYsV49mmzfj96G94eHpStGgx/AKDAAhoGcyxo7/dc2wPopyXwaLHlmhMwMuQLS8vr8w2aWlpXL9+DU9PTwCMCQkMfusNxk6YjK9vJavFfTcPmtOd+PkH8tU3S/ny6/9S+dEqVHr00YIOPQcvLwMJGb1QAKPRiJeXIZc22XMpfdfnRnTszLdLV7Dky68p5e5B5YxcDAYDLVoGoZSiXr36OCknLl+2XvE6/9cNfB653bOtWLYE8dmGD15q+TjLd50CYO+xRIq6OvOIe9HM5Z2bVberXi485MMLWut0rfUM4GVglFJqDlY4+Fazdl3OnTvL+fg4UlNT2Lg+mqZ+ARZtmvoFsG7tagC2blrPU42fRilFk2ebcurEcf6+dYu0tDR++vEAj1aphlKKps39+enAPgD+t28Pj1apVtipWKhTtx7nzpwhPs6cV+y6aPz8Ay3a+PkHsjZyFWAeRmjc5BmUUly7epU3X+/PwLcG82QD25ylkJsHyelu/jmodPXKFZb991siOnQqlPizqlO3HmfPniY+7lxGLlH4BWTLJSCQNatXArBxfSyNnzbn4hcQSOy6KFJSUoiPO8fZs6epW6++RS4XLpxn86b1tGnbHoCAwJbs37cXgDOn/yA1NZXSVjzOcOD4RaqX96CyVylcXZzo/Hw1ovadsWhz7uJ1/OtXBOBxH0+Kujlz8crfgLlodWxa1a7Gc8He+7mg7uXrjFKqHdBUa/1ufp9z8XrafX1f2r1zO7OmTSLdlE67sAhe6t2fRfM+oWbtOjzvF0hycjLj3hvB8WNHcPfw4IMJU6noYz6dJTZ6Df/+fCFKKZ5t2ozXBg0BIOHCeca9N4Lr167hWbo0I9//CO/yFe45tuJu9/9Vd+f2bUydMoF0UzqhER3p028A8+bMpnaduvgFmPN6b+Qwjh49goeHBxOnTMfH15dF8+exZPECKlWqnLmuufMXU6ZsWWZO/5iYqLVcvJhIuXJehHfsxIDXBt53jNbKCaBdcCA3rt8gNTWVUqVKMXfBYqpWq87IYe/w+7FjAPQb8BrBbdrdV2z3em3VHdu38fHkCaSbTIRFdKRv/1eZO2cWtevUxT+gBcnJyYwaOZRjR8z73eSPZ2TmsnD+PFavXI6zizNDh7/L8838AHi5V4/Ms2gGDxuZeRZGamoK749+l2PHjuLq6so7Q4bR5On8naFRpnPO09nuR3BDXz5+5VmcnZ34cuMxpnz/E+91b8iPJ/4kav8Zavp4Mvf15pQo6opGM+rLvWz62Xx6ZrO65fmoZxP8hq8ukFgAbq3q98C18MKVlHzXnPIeblavvfdUdO/H/RZde/YgRVdYl71f0Pp+FVTRtTcFUXQTrqbmu+Z4u7tafQexy/N0hRDiftn7x6wUXSGEQ7H3LzdSdIUQDsXef5EmRVcI4Vjsu+ZK0RVCOBY7r7lSdIUQjsXez1iRoiuEcCh2XnPt93q6QgjhiKSnK4RwKPbe05WiK4RwKHLKmBBCWJH0dIUQwoqk6AohhBXJ8IIQQliR9HSFEMKK7LzmStEVQjgYO6+6UnSFEA7F3n8GXOh3jrAmpVQ/rbXDXVLfEfNyxJzAMfNyxJxsydF+BtzP1gEUEkfMyxFzAsfMyxFzshlHK7pCCGHXpOgKIYQVOVrRddRxJ0fMyxFzAsfMyxFzshmHOpAmhBD2ztF6ukIIYdek6AohhBU5RNFVSrVWSh1TSp1QSo2wdTwFQSm1RCmVqJQ6bOtYCpJSylcptUUp9ZtS6lel1CBbx/SglFJFlVL7lFK/ZOT0oa1jKkhKKWel1E9KqbW2jsURPPRFVynlDHwKtAFqA92VUrVtG1WB+AJobesgCkEaMFhrXRt4BnjdAbZXMhCotX4CeBJorZR6xrYhFahBwBFbB+EoHvqiCzQBTmitT2mtU4DvgDAbx/TAtNbbgb9sHUdB01pf0Fr/mPH3Ncxv5oq2jerBaLPrGbOuGZNDHKFWSvkA7YBFto7FUThC0a0InMsyH8dD/ib+/0Ip9SjQANhr41AeWMZX8J+BRGCD1vqhzynDTGAYkG7jOByGIxRd8RBSSpUElgNvaa2v2jqeB6W1NmmtnwR8gCZKqbo2DumBKaVCgESt9f9sHYsjcYSiGw/4Zpn3yXhM2CmllCvmgvu11nqFreMpSFrrJGALjjEe3xQIVUqdxjxsF6iU+o9tQ3r4OULR3Q/UUEpVUUq5Ad2ASBvHJO5AKaWAxcARrfV0W8dTEJRS5ZRSnhl/FwOCgKM2DaoAaK1Haq19tNaPYn5fbdZav2jjsB56D33R1VqnAW8AsZgPyizVWv9q26genFLqW2A38LhSKk4p1dvWMRWQpkBPzL2mnzOmtrYO6gGVB7YopQ5i7gRs0FrL6VUiV/IzYCGEsKKHvqcrhBAPEym6QghhRVJ0hRDCiqToCiGEFUnRFUIIK5KiK4QQViRFVwghrOj/AFGaFWzXXmuSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(cm, annot=True, cmap=\"Blues\")\n",
    "plt.savefig(f\"images/{model_name}.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.82      0.84      1492\n",
      "           1       0.93      0.91      0.92      3476\n",
      "           2       0.92      0.89      0.90      3017\n",
      "           3       0.97      0.99      0.98     15462\n",
      "           4       0.88      0.87      0.88      2890\n",
      "\n",
      "    accuracy                           0.94     26337\n",
      "   macro avg       0.91      0.89      0.90     26337\n",
      "weighted avg       0.94      0.94      0.94     26337\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(train[\"label\"], test[\"label\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9429319968105707\n"
     ]
    }
   ],
   "source": [
    "print(get_score(train[\"label\"], test[\"label\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}