EfficientNet-kPiece-PCA-LSTM-Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fd1efdf1",
   "metadata": {},
   "source": [
    "### Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0938de48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import load_img, img_to_array\n",
    "\n",
    "from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAvgPool2D, BatchNormalization, LSTM, Concatenate\n",
    "from tensorflow.keras.applications import EfficientNetV2L, EfficientNetV2S, Xception\n",
    "from tensorflow.keras.regularizers import l2\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support as score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c954b162",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68302673",
   "metadata": {},
   "source": [
    "A detailed discussion and visualization of the data can be seen in [here](Data-Overview.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "75b06a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.read_csv(\"./Data/train_labels.csv\")\n",
    "images = ['./Data/train/train/'+str(name)+'.tif' for name in labels['id']]\n",
    "labels['images'] = images\n",
    "labels = labels[['id', 'images', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29190fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_labels = list(set(labels['label']))\n",
    "class_names = [\n",
    "    'letter', 'form', 'email', 'handwritten', 'advertisement', 'scientific report', 'scientific publication',\n",
    "    'specification', 'file folder', 'news article', 'budget', 'invoice', 'presentation', 'questionnaire', 'resume',\n",
    "    'memo'\n",
    "]\n",
    "label_names = pd.DataFrame({\n",
    "    'labels': class_labels,\n",
    "    'names': class_names\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b49810a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = pd.read_csv('./Data/train_labels_1.csv')\n",
    "test_labels = pd.read_csv('./Data/test_labels_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ca081f2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "roi_info = np.load('./Data/ROI-Extractions/train/train_roi_viz.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a77be477",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 100, 6)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_info.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2e251c1",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1a53ca",
   "metadata": {},
   "source": [
    "Below we load the visual features for each document as generated in [EfficientNet kPiece Model](EfficientNet-kPiece-Model.ipynb). We will use this for fine-tuning."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc3707c1",
   "metadata": {},
   "source": [
    "### Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "87c751ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 1280)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_4P = np.load('./Data/EffNet-4P-train-precompute/whole_train_data_precomp.npy')\n",
    "# fine_tune_data_4P = np.reshape(fine_tune_data_4P, (16000, -1))\n",
    "fine_tune_data_4P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e1b35850",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data = np.load(f'./Data/EffNet-kP-train-precompute/whole_train_data_precomp.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "945aad41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 100, 1280)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "568ffcb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels = labels['label'].to_numpy()\n",
    "fine_tune_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6c95996d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200, 5, 1280)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train_4P = fine_tune_data_4P[list(train_labels['id'])]\n",
    "fine_tune_data_train_4P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6e228ab3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200, 100, 1280)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train = fine_tune_data[list(train_labels['id'])]\n",
    "fine_tune_data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bcfdf149",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200,)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels_train = train_labels['label'].to_numpy()\n",
    "fine_tune_labels_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ea4370c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 5, 1280)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_test_4P = fine_tune_data_4P[list(test_labels['id'])]\n",
    "fine_tune_data_test_4P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ccfac9c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 100, 1280)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_test = fine_tune_data[list(test_labels['id'])]\n",
    "fine_tune_data_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "132bb5a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800,)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels_test = test_labels['label'].to_numpy()\n",
    "fine_tune_labels_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d052cb1",
   "metadata": {},
   "source": [
    "Now, we observe that the input vector for each image is $7680D$ which is pretty high dimensional. Since, it consistd of $6$ parts of $1280D$, on each part we use PCA to reduce the dimensions to a reasonable number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0e282176",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_COMPS = 512\n",
    "pcas = [PCA(N_COMPS) for _ in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2ed8047c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    pcas[i].fit(fine_tune_data_train_4P[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "737f5590",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data_train_4P_red = np.zeros((fine_tune_data_train_4P.shape[0], 5 * N_COMPS))\n",
    "for i in range(5):\n",
    "    fine_tune_data_train_4P_red[:,(N_COMPS*i):(N_COMPS*(i+1))] = pcas[i].transform(fine_tune_data_train_4P[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ec6fbb3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200, 2560)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train_4P_red.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bbd9ce98",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data_test_4P_red = np.zeros((fine_tune_data_test_4P.shape[0], 5 * N_COMPS))\n",
    "for i in range(5):\n",
    "    fine_tune_data_test_4P_red[:,(N_COMPS*i):(N_COMPS*(i+1))] = pcas[i].transform(fine_tune_data_test_4P[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fe643c0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 2560)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_test_4P_red.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48661622",
   "metadata": {},
   "source": [
    "### Validation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bc88528c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>images</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>850</th>\n",
       "      <td>./Data/validation/validation/17801.tif</td>\n",
       "      <td>17801</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>./Data/validation/validation/17802.tif</td>\n",
       "      <td>17802</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>./Data/validation/validation/17803.tif</td>\n",
       "      <td>17803</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>./Data/validation/validation/17804.tif</td>\n",
       "      <td>17804</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>./Data/validation/validation/17805.tif</td>\n",
       "      <td>17805</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>./Data/validation/validation/18696.tif</td>\n",
       "      <td>18696</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>./Data/validation/validation/18697.tif</td>\n",
       "      <td>18697</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>./Data/validation/validation/18698.tif</td>\n",
       "      <td>18698</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>787</th>\n",
       "      <td>./Data/validation/validation/18699.tif</td>\n",
       "      <td>18699</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>./Data/validation/validation/18700.tif</td>\n",
       "      <td>18700</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     images     id  label\n",
       "850  ./Data/validation/validation/17801.tif  17801     -1\n",
       "898  ./Data/validation/validation/17802.tif  17802     -1\n",
       "852  ./Data/validation/validation/17803.tif  17803     -1\n",
       "798  ./Data/validation/validation/17804.tif  17804     -1\n",
       "750  ./Data/validation/validation/17805.tif  17805     -1\n",
       "..                                      ...    ...    ...\n",
       "246  ./Data/validation/validation/18696.tif  18696     -1\n",
       "211  ./Data/validation/validation/18697.tif  18697     -1\n",
       "760  ./Data/validation/validation/18698.tif  18698     -1\n",
       "787  ./Data/validation/validation/18699.tif  18699     -1\n",
       "891  ./Data/validation/validation/18700.tif  18700     -1\n",
       "\n",
       "[900 rows x 3 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_data = pd.DataFrame({'images':['./Data/validation/validation/'+name for name in os.listdir('./Data/validation/validation/')]})\n",
    "validation_data['id'] = [name.split('.')[0] for name in os.listdir('./Data/validation/validation/')]\n",
    "validation_data['label'] = -1 # Simply added to prevent re-writing code\n",
    "validation_data.sort_values(by=['id'], inplace=True)\n",
    "validation_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bf4cf1be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 100, 6)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_info_validation = np.load('./Data/ROI-Extractions/validation/validation_roi_viz.npy')\n",
    "roi_info_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9c377e45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 1280)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_4P = np.load('./Data/EffNet-4P-validation-precompute/whole_validation_data_precomp.npy')\n",
    "# fine_tune_data_4P = np.reshape(fine_tune_data_4P, (16000, -1))\n",
    "fine_tune_data_validation_4P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "23851219",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data_validation = np.load(f'./Data/EffNet-kP-validation-precompute/whole_validation_data_precomp.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "53ed0146",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 100, 1280)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "44350f4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data_validation_4P_red = np.zeros((fine_tune_data_validation_4P.shape[0], 5 * N_COMPS))\n",
    "for i in range(5):\n",
    "    fine_tune_data_validation_4P_red[:,(N_COMPS*i):(N_COMPS*(i+1))] = pcas[i].transform(fine_tune_data_validation_4P[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cec17792",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 2560)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_4P_red.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dff1f776",
   "metadata": {},
   "source": [
    "## Fine-Tuning Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7380947f",
   "metadata": {},
   "source": [
    "The fine-tuning part of our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "96a2bdfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ftmodel():\n",
    "    inputs = Input(shape=(5*N_COMPS))\n",
    "    \n",
    "    inputs_roi = Input(shape=(100, fine_tune_data_train.shape[-1]))\n",
    "    y = LSTM(512, input_shape=(100, fine_tune_data_train.shape[-1]))(inputs_roi)\n",
    "    \n",
    "    x = Concatenate()([inputs, y])\n",
    "    x = BatchNormalization()(x)\n",
    "    top_dropout_rate = 0.3\n",
    "    # x = Dropout(top_dropout_rate, name='top_dropout_1')(x)\n",
    "    # x = Dense(1024, name='top_dense_1', kernel_initializer=\"he_normal\", kernel_regularizer=l2(5e-5), activation='relu')(x)\n",
    "    # x = BatchNormalization()(x)\n",
    "    x = Dropout(top_dropout_rate, name='top_dropout_2')(x)\n",
    "    x = Dense(512, name='top_dense_2', kernel_initializer=\"he_normal\", kernel_regularizer=l2(5e-5), activation='relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(top_dropout_rate, name='top_dropout_3')(x)\n",
    "    outputs = Dense(16, activation='softmax', name='pred', kernel_regularizer=l2(5e-5))(x)\n",
    "\n",
    "    model1 = tf.keras.Model([inputs, inputs_roi], outputs, name='EfficientNet-FineTune')\n",
    "    return model1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c6574006",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metal device set to: Apple M1 Max\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 02:34:06.101448: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2022-10-08 02:34:06.101587: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"EfficientNet-FineTune\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_2 (InputLayer)           [(None, 100, 1280)]  0           []                               \n",
      "                                                                                                  \n",
      " input_1 (InputLayer)           [(None, 2560)]       0           []                               \n",
      "                                                                                                  \n",
      " lstm (LSTM)                    (None, 512)          3672064     ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)      (None, 3072)         0           ['input_1[0][0]',                \n",
      "                                                                  'lstm[0][0]']                   \n",
      "                                                                                                  \n",
      " batch_normalization (BatchNorm  (None, 3072)        12288       ['concatenate[0][0]']            \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " top_dropout_2 (Dropout)        (None, 3072)         0           ['batch_normalization[0][0]']    \n",
      "                                                                                                  \n",
      " top_dense_2 (Dense)            (None, 512)          1573376     ['top_dropout_2[0][0]']          \n",
      "                                                                                                  \n",
      " batch_normalization_1 (BatchNo  (None, 512)         2048        ['top_dense_2[0][0]']            \n",
      " rmalization)                                                                                     \n",
      "                                                                                                  \n",
      " top_dropout_3 (Dropout)        (None, 512)          0           ['batch_normalization_1[0][0]']  \n",
      "                                                                                                  \n",
      " pred (Dense)                   (None, 16)           8208        ['top_dropout_3[0][0]']          \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 5,267,984\n",
      "Trainable params: 5,260,816\n",
      "Non-trainable params: 7,168\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model1 = ftmodel()\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "caca467d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.keras.utils.plot_model(model1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e56505e2",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70b7e206",
   "metadata": {},
   "source": [
    "The fine-tuning will be done on the EfficientNet's $1280D$ representation vectors for $5$ pieces of the image and the averaged ROI representation vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "96524578",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.compile(loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'], optimizer = Adam(learning_rate = 0.0003))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "20df8897",
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 1000\n",
    "checkpoint_filepaths = ['./Models/EfficientNet-kP-PCA-LSTM/auto-EffNet-FT-model-weight']\n",
    "model_checkpoint_callback = [tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    save_weights_only=True,\n",
    "    monitor = \"loss\",\n",
    "    mode='min',\n",
    "    save_best_only=True) for checkpoint_filepath in checkpoint_filepaths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "03354278",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 02:34:18.146884: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n",
      "2022-10-08 02:34:18.988575: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-10-08 02:34:20.031133: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-10-08 02:34:20.621947: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19/19 [==============================] - ETA: 0s - loss: 3.0615 - accuracy: 0.1964"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 02:34:40.815153: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-10-08 02:34:40.953167: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19/19 [==============================] - 23s 1s/step - loss: 3.0615 - accuracy: 0.1964 - val_loss: 2.7692 - val_accuracy: 0.3212\n",
      "Epoch 2/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 2.0185 - accuracy: 0.4309 - val_loss: 2.7213 - val_accuracy: 0.3625\n",
      "Epoch 3/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 1.6149 - accuracy: 0.5429 - val_loss: 2.6843 - val_accuracy: 0.3662\n",
      "Epoch 4/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 1.4051 - accuracy: 0.5997 - val_loss: 2.6486 - val_accuracy: 0.3662\n",
      "Epoch 5/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 1.2417 - accuracy: 0.6449 - val_loss: 2.6104 - val_accuracy: 0.3625\n",
      "Epoch 6/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 1.1422 - accuracy: 0.6694 - val_loss: 2.5686 - val_accuracy: 0.3562\n",
      "Epoch 7/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 1.0519 - accuracy: 0.6949 - val_loss: 2.5255 - val_accuracy: 0.3650\n",
      "Epoch 8/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.9761 - accuracy: 0.7201 - val_loss: 2.4768 - val_accuracy: 0.3862\n",
      "Epoch 9/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.9101 - accuracy: 0.7392 - val_loss: 2.4263 - val_accuracy: 0.3925\n",
      "Epoch 10/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.8635 - accuracy: 0.7489 - val_loss: 2.3711 - val_accuracy: 0.4062\n",
      "Epoch 11/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.7998 - accuracy: 0.7679 - val_loss: 2.3160 - val_accuracy: 0.4237\n",
      "Epoch 12/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.7512 - accuracy: 0.7842 - val_loss: 2.2593 - val_accuracy: 0.4412\n",
      "Epoch 13/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.7084 - accuracy: 0.7978 - val_loss: 2.2043 - val_accuracy: 0.4412\n",
      "Epoch 14/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.6811 - accuracy: 0.8083 - val_loss: 2.1514 - val_accuracy: 0.4562\n",
      "Epoch 15/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.6504 - accuracy: 0.8156 - val_loss: 2.0949 - val_accuracy: 0.4625\n",
      "Epoch 16/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.6109 - accuracy: 0.8295 - val_loss: 2.0358 - val_accuracy: 0.4775\n",
      "Epoch 17/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.5824 - accuracy: 0.8399 - val_loss: 1.9654 - val_accuracy: 0.5075\n",
      "Epoch 18/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.5421 - accuracy: 0.8538 - val_loss: 1.8987 - val_accuracy: 0.5200\n",
      "Epoch 19/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.5326 - accuracy: 0.8539 - val_loss: 1.8378 - val_accuracy: 0.5362\n",
      "Epoch 20/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.4960 - accuracy: 0.8666 - val_loss: 1.7736 - val_accuracy: 0.5512\n",
      "Epoch 21/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.4779 - accuracy: 0.8718 - val_loss: 1.7105 - val_accuracy: 0.5587\n",
      "Epoch 22/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.4474 - accuracy: 0.8831 - val_loss: 1.6465 - val_accuracy: 0.5725\n",
      "Epoch 23/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.4289 - accuracy: 0.8916 - val_loss: 1.5847 - val_accuracy: 0.5888\n",
      "Epoch 24/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.4099 - accuracy: 0.8943 - val_loss: 1.5140 - val_accuracy: 0.5975\n",
      "Epoch 25/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3989 - accuracy: 0.8972 - val_loss: 1.4566 - val_accuracy: 0.6075\n",
      "Epoch 26/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.3774 - accuracy: 0.9078 - val_loss: 1.3986 - val_accuracy: 0.6187\n",
      "Epoch 27/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3598 - accuracy: 0.9116 - val_loss: 1.3535 - val_accuracy: 0.6362\n",
      "Epoch 28/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3473 - accuracy: 0.9195 - val_loss: 1.3070 - val_accuracy: 0.6475\n",
      "Epoch 29/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3397 - accuracy: 0.9184 - val_loss: 1.2642 - val_accuracy: 0.6475\n",
      "Epoch 30/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3277 - accuracy: 0.9211 - val_loss: 1.2327 - val_accuracy: 0.6562\n",
      "Epoch 31/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.3025 - accuracy: 0.9309 - val_loss: 1.1986 - val_accuracy: 0.6600\n",
      "Epoch 32/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.2893 - accuracy: 0.9372 - val_loss: 1.1767 - val_accuracy: 0.6612\n",
      "Epoch 33/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.2814 - accuracy: 0.9374 - val_loss: 1.1513 - val_accuracy: 0.6625\n",
      "Epoch 34/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2728 - accuracy: 0.9401 - val_loss: 1.1324 - val_accuracy: 0.6637\n",
      "Epoch 35/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2575 - accuracy: 0.9461 - val_loss: 1.1210 - val_accuracy: 0.6675\n",
      "Epoch 36/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2551 - accuracy: 0.9470 - val_loss: 1.1057 - val_accuracy: 0.6650\n",
      "Epoch 37/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2421 - accuracy: 0.9526 - val_loss: 1.0927 - val_accuracy: 0.6712\n",
      "Epoch 38/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2360 - accuracy: 0.9522 - val_loss: 1.0864 - val_accuracy: 0.6737\n",
      "Epoch 39/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2307 - accuracy: 0.9548 - val_loss: 1.0822 - val_accuracy: 0.6850\n",
      "Epoch 40/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2189 - accuracy: 0.9594 - val_loss: 1.0784 - val_accuracy: 0.6887\n",
      "Epoch 41/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2162 - accuracy: 0.9601 - val_loss: 1.0810 - val_accuracy: 0.6900\n",
      "Epoch 42/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2071 - accuracy: 0.9609 - val_loss: 1.0821 - val_accuracy: 0.6912\n",
      "Epoch 43/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.2010 - accuracy: 0.9657 - val_loss: 1.0898 - val_accuracy: 0.6837\n",
      "Epoch 44/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1904 - accuracy: 0.9678 - val_loss: 1.0942 - val_accuracy: 0.6900\n",
      "Epoch 45/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1873 - accuracy: 0.9689 - val_loss: 1.0968 - val_accuracy: 0.6962\n",
      "Epoch 46/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1821 - accuracy: 0.9686 - val_loss: 1.0997 - val_accuracy: 0.6875\n",
      "Epoch 47/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1788 - accuracy: 0.9701 - val_loss: 1.1014 - val_accuracy: 0.6850\n",
      "Epoch 48/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.1696 - accuracy: 0.9730 - val_loss: 1.1050 - val_accuracy: 0.6875\n",
      "Epoch 49/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1692 - accuracy: 0.9740 - val_loss: 1.1104 - val_accuracy: 0.6925\n",
      "Epoch 50/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1613 - accuracy: 0.9757 - val_loss: 1.1141 - val_accuracy: 0.6912\n",
      "Epoch 51/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1629 - accuracy: 0.9738 - val_loss: 1.1113 - val_accuracy: 0.7000\n",
      "Epoch 52/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1547 - accuracy: 0.9774 - val_loss: 1.1189 - val_accuracy: 0.6975\n",
      "Epoch 53/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1515 - accuracy: 0.9782 - val_loss: 1.1191 - val_accuracy: 0.6962\n",
      "Epoch 54/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1480 - accuracy: 0.9784 - val_loss: 1.1256 - val_accuracy: 0.6937\n",
      "Epoch 55/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1468 - accuracy: 0.9771 - val_loss: 1.1351 - val_accuracy: 0.6912\n",
      "Epoch 56/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1428 - accuracy: 0.9814 - val_loss: 1.1384 - val_accuracy: 0.6925\n",
      "Epoch 57/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1396 - accuracy: 0.9807 - val_loss: 1.1367 - val_accuracy: 0.6937\n",
      "Epoch 58/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19/19 [==============================] - 20s 1s/step - loss: 0.1383 - accuracy: 0.9793 - val_loss: 1.1444 - val_accuracy: 0.6925\n",
      "Epoch 59/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1342 - accuracy: 0.9823 - val_loss: 1.1559 - val_accuracy: 0.6962\n",
      "Epoch 60/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.1335 - accuracy: 0.9825 - val_loss: 1.1423 - val_accuracy: 0.6875\n",
      "Epoch 61/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1280 - accuracy: 0.9839 - val_loss: 1.1403 - val_accuracy: 0.6850\n",
      "Epoch 62/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1257 - accuracy: 0.9846 - val_loss: 1.1473 - val_accuracy: 0.6875\n",
      "Epoch 63/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1211 - accuracy: 0.9861 - val_loss: 1.1510 - val_accuracy: 0.6850\n",
      "Epoch 64/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1241 - accuracy: 0.9844 - val_loss: 1.1559 - val_accuracy: 0.6912\n",
      "Epoch 65/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1206 - accuracy: 0.9864 - val_loss: 1.1612 - val_accuracy: 0.6937\n",
      "Epoch 66/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1174 - accuracy: 0.9867 - val_loss: 1.1579 - val_accuracy: 0.6912\n",
      "Epoch 67/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1201 - accuracy: 0.9854 - val_loss: 1.1641 - val_accuracy: 0.6850\n",
      "Epoch 68/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1135 - accuracy: 0.9876 - val_loss: 1.1724 - val_accuracy: 0.6862\n",
      "Epoch 69/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1158 - accuracy: 0.9857 - val_loss: 1.1678 - val_accuracy: 0.6900\n",
      "Epoch 70/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1113 - accuracy: 0.9875 - val_loss: 1.1704 - val_accuracy: 0.6975\n",
      "Epoch 71/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1092 - accuracy: 0.9881 - val_loss: 1.1793 - val_accuracy: 0.7000\n",
      "Epoch 72/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1102 - accuracy: 0.9879 - val_loss: 1.1817 - val_accuracy: 0.6937\n",
      "Epoch 73/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1092 - accuracy: 0.9887 - val_loss: 1.1768 - val_accuracy: 0.6925\n",
      "Epoch 74/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1065 - accuracy: 0.9893 - val_loss: 1.1808 - val_accuracy: 0.6950\n",
      "Epoch 75/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1052 - accuracy: 0.9881 - val_loss: 1.1879 - val_accuracy: 0.6912\n",
      "Epoch 76/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1046 - accuracy: 0.9878 - val_loss: 1.1932 - val_accuracy: 0.6925\n",
      "Epoch 77/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1023 - accuracy: 0.9899 - val_loss: 1.1956 - val_accuracy: 0.6950\n",
      "Epoch 78/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1033 - accuracy: 0.9890 - val_loss: 1.1951 - val_accuracy: 0.6925\n",
      "Epoch 79/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.1016 - accuracy: 0.9895 - val_loss: 1.1871 - val_accuracy: 0.6912\n",
      "Epoch 80/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0982 - accuracy: 0.9916 - val_loss: 1.1477 - val_accuracy: 0.6800\n",
      "Epoch 81/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0991 - accuracy: 0.9901 - val_loss: 1.1494 - val_accuracy: 0.6775\n",
      "Epoch 82/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0964 - accuracy: 0.9918 - val_loss: 1.1570 - val_accuracy: 0.6800\n",
      "Epoch 83/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0992 - accuracy: 0.9902 - val_loss: 1.1648 - val_accuracy: 0.6837\n",
      "Epoch 84/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0961 - accuracy: 0.9916 - val_loss: 1.1675 - val_accuracy: 0.6762\n",
      "Epoch 85/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0932 - accuracy: 0.9910 - val_loss: 1.1761 - val_accuracy: 0.6875\n",
      "Epoch 86/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0951 - accuracy: 0.9916 - val_loss: 1.1833 - val_accuracy: 0.6825\n",
      "Epoch 87/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0937 - accuracy: 0.9911 - val_loss: 1.2044 - val_accuracy: 0.6862\n",
      "Epoch 88/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0927 - accuracy: 0.9910 - val_loss: 1.2050 - val_accuracy: 0.6875\n",
      "Epoch 89/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0922 - accuracy: 0.9913 - val_loss: 1.2037 - val_accuracy: 0.6875\n",
      "Epoch 90/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0923 - accuracy: 0.9925 - val_loss: 1.2142 - val_accuracy: 0.6900\n",
      "Epoch 91/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0885 - accuracy: 0.9934 - val_loss: 1.2132 - val_accuracy: 0.6950\n",
      "Epoch 92/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0883 - accuracy: 0.9925 - val_loss: 1.2144 - val_accuracy: 0.6950\n",
      "Epoch 93/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0893 - accuracy: 0.9920 - val_loss: 1.2162 - val_accuracy: 0.6962\n",
      "Epoch 94/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0870 - accuracy: 0.9930 - val_loss: 1.2151 - val_accuracy: 0.6975\n",
      "Epoch 95/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0842 - accuracy: 0.9939 - val_loss: 1.2126 - val_accuracy: 0.7000\n",
      "Epoch 96/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0876 - accuracy: 0.9921 - val_loss: 1.1825 - val_accuracy: 0.6937\n",
      "Epoch 97/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0866 - accuracy: 0.9918 - val_loss: 1.1800 - val_accuracy: 0.6912\n",
      "Epoch 98/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0839 - accuracy: 0.9940 - val_loss: 1.1996 - val_accuracy: 0.7000\n",
      "Epoch 99/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0856 - accuracy: 0.9922 - val_loss: 1.2045 - val_accuracy: 0.6975\n",
      "Epoch 100/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0823 - accuracy: 0.9936 - val_loss: 1.1958 - val_accuracy: 0.6937\n",
      "Epoch 101/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0840 - accuracy: 0.9930 - val_loss: 1.1579 - val_accuracy: 0.6825\n",
      "Epoch 102/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0826 - accuracy: 0.9939 - val_loss: 1.1750 - val_accuracy: 0.6825\n",
      "Epoch 103/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0808 - accuracy: 0.9939 - val_loss: 1.1894 - val_accuracy: 0.6887\n",
      "Epoch 104/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0808 - accuracy: 0.9936 - val_loss: 1.1987 - val_accuracy: 0.6937\n",
      "Epoch 105/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0807 - accuracy: 0.9943 - val_loss: 1.1987 - val_accuracy: 0.6987\n",
      "Epoch 106/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0802 - accuracy: 0.9945 - val_loss: 1.2140 - val_accuracy: 0.6975\n",
      "Epoch 107/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0788 - accuracy: 0.9942 - val_loss: 1.2150 - val_accuracy: 0.6987\n",
      "Epoch 108/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0799 - accuracy: 0.9936 - val_loss: 1.2133 - val_accuracy: 0.6937\n",
      "Epoch 109/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0791 - accuracy: 0.9941 - val_loss: 1.2218 - val_accuracy: 0.6950\n",
      "Epoch 110/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0785 - accuracy: 0.9949 - val_loss: 1.2194 - val_accuracy: 0.7063\n",
      "Epoch 111/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0784 - accuracy: 0.9947 - val_loss: 1.2234 - val_accuracy: 0.7063\n",
      "Epoch 112/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0772 - accuracy: 0.9947 - val_loss: 1.2399 - val_accuracy: 0.7100\n",
      "Epoch 113/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0770 - accuracy: 0.9953 - val_loss: 1.2431 - val_accuracy: 0.7025\n",
      "Epoch 114/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0778 - accuracy: 0.9945 - val_loss: 1.2477 - val_accuracy: 0.7037\n",
      "Epoch 115/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19/19 [==============================] - 20s 1s/step - loss: 0.0740 - accuracy: 0.9957 - val_loss: 1.2534 - val_accuracy: 0.7063\n",
      "Epoch 116/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0744 - accuracy: 0.9950 - val_loss: 1.2365 - val_accuracy: 0.7000\n",
      "Epoch 117/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0742 - accuracy: 0.9957 - val_loss: 1.2302 - val_accuracy: 0.7075\n",
      "Epoch 118/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0748 - accuracy: 0.9951 - val_loss: 1.2239 - val_accuracy: 0.7012\n",
      "Epoch 119/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0741 - accuracy: 0.9950 - val_loss: 1.2353 - val_accuracy: 0.6987\n",
      "Epoch 120/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0731 - accuracy: 0.9956 - val_loss: 1.2439 - val_accuracy: 0.6987\n",
      "Epoch 121/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0735 - accuracy: 0.9948 - val_loss: 1.2401 - val_accuracy: 0.7000\n",
      "Epoch 122/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0731 - accuracy: 0.9951 - val_loss: 1.2360 - val_accuracy: 0.7025\n",
      "Epoch 123/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0728 - accuracy: 0.9956 - val_loss: 1.2392 - val_accuracy: 0.7113\n",
      "Epoch 124/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0704 - accuracy: 0.9958 - val_loss: 1.2450 - val_accuracy: 0.7100\n",
      "Epoch 125/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0719 - accuracy: 0.9952 - val_loss: 1.2292 - val_accuracy: 0.7150\n",
      "Epoch 126/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0721 - accuracy: 0.9958 - val_loss: 1.2156 - val_accuracy: 0.7113\n",
      "Epoch 127/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0714 - accuracy: 0.9957 - val_loss: 1.1966 - val_accuracy: 0.7100\n",
      "Epoch 128/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0728 - accuracy: 0.9946 - val_loss: 1.1682 - val_accuracy: 0.7050\n",
      "Epoch 129/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0718 - accuracy: 0.9952 - val_loss: 1.1503 - val_accuracy: 0.7012\n",
      "Epoch 130/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0703 - accuracy: 0.9963 - val_loss: 1.1610 - val_accuracy: 0.7012\n",
      "Epoch 131/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0706 - accuracy: 0.9948 - val_loss: 1.1743 - val_accuracy: 0.7075\n",
      "Epoch 132/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0717 - accuracy: 0.9958 - val_loss: 1.1808 - val_accuracy: 0.7075\n",
      "Epoch 133/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0695 - accuracy: 0.9960 - val_loss: 1.1842 - val_accuracy: 0.7088\n",
      "Epoch 134/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0688 - accuracy: 0.9962 - val_loss: 1.1954 - val_accuracy: 0.7063\n",
      "Epoch 135/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0716 - accuracy: 0.9945 - val_loss: 1.2002 - val_accuracy: 0.7100\n",
      "Epoch 136/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0665 - accuracy: 0.9975 - val_loss: 1.2026 - val_accuracy: 0.7012\n",
      "Epoch 137/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0686 - accuracy: 0.9958 - val_loss: 1.2078 - val_accuracy: 0.7063\n",
      "Epoch 138/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0671 - accuracy: 0.9965 - val_loss: 1.2169 - val_accuracy: 0.7025\n",
      "Epoch 139/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0688 - accuracy: 0.9958 - val_loss: 1.2231 - val_accuracy: 0.7050\n",
      "Epoch 140/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0671 - accuracy: 0.9961 - val_loss: 1.2242 - val_accuracy: 0.7025\n",
      "Epoch 141/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0662 - accuracy: 0.9968 - val_loss: 1.2164 - val_accuracy: 0.6975\n",
      "Epoch 142/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0660 - accuracy: 0.9967 - val_loss: 1.2147 - val_accuracy: 0.6962\n",
      "Epoch 143/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0677 - accuracy: 0.9961 - val_loss: 1.2175 - val_accuracy: 0.7025\n",
      "Epoch 144/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0651 - accuracy: 0.9972 - val_loss: 1.2209 - val_accuracy: 0.6937\n",
      "Epoch 145/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0656 - accuracy: 0.9969 - val_loss: 1.2205 - val_accuracy: 0.6900\n",
      "Epoch 146/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0647 - accuracy: 0.9973 - val_loss: 1.2310 - val_accuracy: 0.7000\n",
      "Epoch 147/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0633 - accuracy: 0.9976 - val_loss: 1.2458 - val_accuracy: 0.6962\n",
      "Epoch 148/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0661 - accuracy: 0.9961 - val_loss: 1.2505 - val_accuracy: 0.6950\n",
      "Epoch 149/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0646 - accuracy: 0.9966 - val_loss: 1.2245 - val_accuracy: 0.6912\n",
      "Epoch 150/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0649 - accuracy: 0.9968 - val_loss: 1.2034 - val_accuracy: 0.6875\n",
      "Epoch 151/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0653 - accuracy: 0.9962 - val_loss: 1.2121 - val_accuracy: 0.6825\n",
      "Epoch 152/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0668 - accuracy: 0.9959 - val_loss: 1.2256 - val_accuracy: 0.6925\n",
      "Epoch 153/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0658 - accuracy: 0.9958 - val_loss: 1.2595 - val_accuracy: 0.6937\n",
      "Epoch 154/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0640 - accuracy: 0.9965 - val_loss: 1.3065 - val_accuracy: 0.6950\n",
      "Epoch 155/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0636 - accuracy: 0.9972 - val_loss: 1.3139 - val_accuracy: 0.6950\n",
      "Epoch 156/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0642 - accuracy: 0.9961 - val_loss: 1.3090 - val_accuracy: 0.6962\n",
      "Epoch 157/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0641 - accuracy: 0.9969 - val_loss: 1.2790 - val_accuracy: 0.6962\n",
      "Epoch 158/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0622 - accuracy: 0.9971 - val_loss: 1.2587 - val_accuracy: 0.6937\n",
      "Epoch 159/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0634 - accuracy: 0.9965 - val_loss: 1.2555 - val_accuracy: 0.6900\n",
      "Epoch 160/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0648 - accuracy: 0.9961 - val_loss: 1.2563 - val_accuracy: 0.6912\n",
      "Epoch 161/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0636 - accuracy: 0.9971 - val_loss: 1.2524 - val_accuracy: 0.6950\n",
      "Epoch 162/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0612 - accuracy: 0.9974 - val_loss: 1.2522 - val_accuracy: 0.7000\n",
      "Epoch 163/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0626 - accuracy: 0.9967 - val_loss: 1.2764 - val_accuracy: 0.6937\n",
      "Epoch 164/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0615 - accuracy: 0.9968 - val_loss: 1.2704 - val_accuracy: 0.6912\n",
      "Epoch 165/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0621 - accuracy: 0.9972 - val_loss: 1.2638 - val_accuracy: 0.6900\n",
      "Epoch 166/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0621 - accuracy: 0.9972 - val_loss: 1.2704 - val_accuracy: 0.7000\n",
      "Epoch 167/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0619 - accuracy: 0.9969 - val_loss: 1.2806 - val_accuracy: 0.6962\n",
      "Epoch 168/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0613 - accuracy: 0.9969 - val_loss: 1.2784 - val_accuracy: 0.6950\n",
      "Epoch 169/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0598 - accuracy: 0.9978 - val_loss: 1.2760 - val_accuracy: 0.7025\n",
      "Epoch 170/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0608 - accuracy: 0.9972 - val_loss: 1.2883 - val_accuracy: 0.7000\n",
      "Epoch 171/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0607 - accuracy: 0.9968 - val_loss: 1.2932 - val_accuracy: 0.6937\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 172/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0613 - accuracy: 0.9965 - val_loss: 1.2971 - val_accuracy: 0.6975\n",
      "Epoch 173/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0610 - accuracy: 0.9969 - val_loss: 1.3172 - val_accuracy: 0.6962\n",
      "Epoch 174/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0601 - accuracy: 0.9972 - val_loss: 1.3235 - val_accuracy: 0.6912\n",
      "Epoch 175/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0603 - accuracy: 0.9968 - val_loss: 1.3112 - val_accuracy: 0.6950\n",
      "Epoch 176/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0600 - accuracy: 0.9970 - val_loss: 1.3142 - val_accuracy: 0.6962\n",
      "Epoch 177/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0589 - accuracy: 0.9978 - val_loss: 1.3111 - val_accuracy: 0.6962\n",
      "Epoch 178/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0612 - accuracy: 0.9964 - val_loss: 1.2927 - val_accuracy: 0.6925\n",
      "Epoch 179/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0587 - accuracy: 0.9970 - val_loss: 1.2577 - val_accuracy: 0.6962\n",
      "Epoch 180/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0599 - accuracy: 0.9968 - val_loss: 1.2512 - val_accuracy: 0.6925\n",
      "Epoch 181/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0595 - accuracy: 0.9971 - val_loss: 1.2629 - val_accuracy: 0.6900\n",
      "Epoch 182/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0588 - accuracy: 0.9972 - val_loss: 1.2739 - val_accuracy: 0.6937\n",
      "Epoch 183/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0601 - accuracy: 0.9972 - val_loss: 1.2810 - val_accuracy: 0.6962\n",
      "Epoch 184/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0584 - accuracy: 0.9976 - val_loss: 1.2788 - val_accuracy: 0.6975\n",
      "Epoch 185/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0575 - accuracy: 0.9976 - val_loss: 1.2860 - val_accuracy: 0.7000\n",
      "Epoch 186/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0583 - accuracy: 0.9970 - val_loss: 1.2787 - val_accuracy: 0.7000\n",
      "Epoch 187/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0568 - accuracy: 0.9982 - val_loss: 1.2721 - val_accuracy: 0.7063\n",
      "Epoch 188/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0573 - accuracy: 0.9972 - val_loss: 1.2757 - val_accuracy: 0.6987\n",
      "Epoch 189/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0596 - accuracy: 0.9965 - val_loss: 1.2816 - val_accuracy: 0.7000\n",
      "Epoch 190/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0591 - accuracy: 0.9973 - val_loss: 1.2641 - val_accuracy: 0.7063\n",
      "Epoch 191/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0581 - accuracy: 0.9971 - val_loss: 1.2621 - val_accuracy: 0.7138\n",
      "Epoch 192/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0572 - accuracy: 0.9974 - val_loss: 1.2668 - val_accuracy: 0.7125\n",
      "Epoch 193/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0594 - accuracy: 0.9966 - val_loss: 1.2853 - val_accuracy: 0.7138\n",
      "Epoch 194/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0562 - accuracy: 0.9978 - val_loss: 1.2993 - val_accuracy: 0.7150\n",
      "Epoch 195/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0599 - accuracy: 0.9965 - val_loss: 1.2941 - val_accuracy: 0.7075\n",
      "Epoch 196/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0559 - accuracy: 0.9976 - val_loss: 1.3035 - val_accuracy: 0.6975\n",
      "Epoch 197/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0573 - accuracy: 0.9972 - val_loss: 1.3125 - val_accuracy: 0.6962\n",
      "Epoch 198/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0577 - accuracy: 0.9969 - val_loss: 1.3011 - val_accuracy: 0.7012\n",
      "Epoch 199/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0569 - accuracy: 0.9975 - val_loss: 1.3003 - val_accuracy: 0.7037\n",
      "Epoch 200/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0571 - accuracy: 0.9971 - val_loss: 1.3086 - val_accuracy: 0.7100\n",
      "Epoch 201/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0576 - accuracy: 0.9970 - val_loss: 1.3133 - val_accuracy: 0.7088\n",
      "Epoch 202/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0562 - accuracy: 0.9974 - val_loss: 1.3134 - val_accuracy: 0.7012\n",
      "Epoch 203/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0555 - accuracy: 0.9972 - val_loss: 1.3191 - val_accuracy: 0.6975\n",
      "Epoch 204/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0560 - accuracy: 0.9974 - val_loss: 1.3289 - val_accuracy: 0.7012\n",
      "Epoch 205/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0575 - accuracy: 0.9966 - val_loss: 1.3329 - val_accuracy: 0.6950\n",
      "Epoch 206/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0563 - accuracy: 0.9972 - val_loss: 1.3259 - val_accuracy: 0.7000\n",
      "Epoch 207/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0560 - accuracy: 0.9974 - val_loss: 1.3167 - val_accuracy: 0.6962\n",
      "Epoch 208/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0586 - accuracy: 0.9964 - val_loss: 1.2979 - val_accuracy: 0.7037\n",
      "Epoch 209/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0556 - accuracy: 0.9978 - val_loss: 1.3083 - val_accuracy: 0.7037\n",
      "Epoch 210/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0567 - accuracy: 0.9970 - val_loss: 1.3112 - val_accuracy: 0.7050\n",
      "Epoch 211/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0563 - accuracy: 0.9972 - val_loss: 1.3098 - val_accuracy: 0.7050\n",
      "Epoch 212/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0572 - accuracy: 0.9964 - val_loss: 1.3194 - val_accuracy: 0.7025\n",
      "Epoch 213/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0554 - accuracy: 0.9972 - val_loss: 1.3171 - val_accuracy: 0.6950\n",
      "Epoch 214/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0581 - accuracy: 0.9966 - val_loss: 1.3161 - val_accuracy: 0.6950\n",
      "Epoch 215/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0562 - accuracy: 0.9972 - val_loss: 1.3353 - val_accuracy: 0.7000\n",
      "Epoch 216/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0566 - accuracy: 0.9968 - val_loss: 1.3394 - val_accuracy: 0.6912\n",
      "Epoch 217/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0550 - accuracy: 0.9972 - val_loss: 1.3380 - val_accuracy: 0.6900\n",
      "Epoch 218/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0550 - accuracy: 0.9974 - val_loss: 1.3288 - val_accuracy: 0.6962\n",
      "Epoch 219/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0559 - accuracy: 0.9969 - val_loss: 1.3209 - val_accuracy: 0.6987\n",
      "Epoch 220/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0571 - accuracy: 0.9970 - val_loss: 1.3220 - val_accuracy: 0.6950\n",
      "Epoch 221/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0557 - accuracy: 0.9966 - val_loss: 1.3229 - val_accuracy: 0.6987\n",
      "Epoch 222/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0542 - accuracy: 0.9979 - val_loss: 1.3298 - val_accuracy: 0.7075\n",
      "Epoch 223/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0555 - accuracy: 0.9966 - val_loss: 1.3367 - val_accuracy: 0.7037\n",
      "Epoch 224/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0544 - accuracy: 0.9980 - val_loss: 1.3447 - val_accuracy: 0.7012\n",
      "Epoch 225/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0541 - accuracy: 0.9974 - val_loss: 1.3535 - val_accuracy: 0.6912\n",
      "Epoch 226/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0547 - accuracy: 0.9972 - val_loss: 1.3603 - val_accuracy: 0.6987\n",
      "Epoch 227/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0545 - accuracy: 0.9969 - val_loss: 1.3284 - val_accuracy: 0.7012\n",
      "Epoch 228/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0547 - accuracy: 0.9972 - val_loss: 1.2889 - val_accuracy: 0.6987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 229/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0547 - accuracy: 0.9974 - val_loss: 1.2946 - val_accuracy: 0.6987\n",
      "Epoch 230/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0542 - accuracy: 0.9973 - val_loss: 1.3128 - val_accuracy: 0.7088\n",
      "Epoch 231/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0542 - accuracy: 0.9972 - val_loss: 1.3019 - val_accuracy: 0.7012\n",
      "Epoch 232/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0550 - accuracy: 0.9972 - val_loss: 1.3050 - val_accuracy: 0.7000\n",
      "Epoch 233/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0537 - accuracy: 0.9973 - val_loss: 1.3060 - val_accuracy: 0.6962\n",
      "Epoch 234/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0541 - accuracy: 0.9970 - val_loss: 1.3168 - val_accuracy: 0.6962\n",
      "Epoch 235/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0551 - accuracy: 0.9966 - val_loss: 1.3196 - val_accuracy: 0.6962\n",
      "Epoch 236/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0538 - accuracy: 0.9972 - val_loss: 1.3255 - val_accuracy: 0.6987\n",
      "Epoch 237/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0530 - accuracy: 0.9977 - val_loss: 1.3181 - val_accuracy: 0.7050\n",
      "Epoch 238/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0546 - accuracy: 0.9974 - val_loss: 1.3189 - val_accuracy: 0.7012\n",
      "Epoch 239/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0539 - accuracy: 0.9980 - val_loss: 1.3206 - val_accuracy: 0.7000\n",
      "Epoch 240/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0541 - accuracy: 0.9966 - val_loss: 1.3272 - val_accuracy: 0.6987\n",
      "Epoch 241/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0524 - accuracy: 0.9972 - val_loss: 1.3132 - val_accuracy: 0.7000\n",
      "Epoch 242/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0555 - accuracy: 0.9972 - val_loss: 1.3180 - val_accuracy: 0.6987\n",
      "Epoch 243/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0534 - accuracy: 0.9973 - val_loss: 1.3148 - val_accuracy: 0.6937\n",
      "Epoch 244/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0538 - accuracy: 0.9971 - val_loss: 1.3248 - val_accuracy: 0.6987\n",
      "Epoch 245/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0523 - accuracy: 0.9976 - val_loss: 1.3292 - val_accuracy: 0.7075\n",
      "Epoch 246/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0531 - accuracy: 0.9974 - val_loss: 1.3255 - val_accuracy: 0.6975\n",
      "Epoch 247/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0525 - accuracy: 0.9977 - val_loss: 1.3315 - val_accuracy: 0.6875\n",
      "Epoch 248/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0530 - accuracy: 0.9976 - val_loss: 1.3298 - val_accuracy: 0.6900\n",
      "Epoch 249/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0537 - accuracy: 0.9972 - val_loss: 1.3331 - val_accuracy: 0.6937\n",
      "Epoch 250/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0543 - accuracy: 0.9973 - val_loss: 1.3493 - val_accuracy: 0.6862\n",
      "Epoch 251/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0528 - accuracy: 0.9973 - val_loss: 1.3380 - val_accuracy: 0.6800\n",
      "Epoch 252/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0528 - accuracy: 0.9975 - val_loss: 1.3403 - val_accuracy: 0.6962\n",
      "Epoch 253/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0539 - accuracy: 0.9972 - val_loss: 1.3387 - val_accuracy: 0.6950\n",
      "Epoch 254/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0537 - accuracy: 0.9966 - val_loss: 1.3304 - val_accuracy: 0.6837\n",
      "Epoch 255/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0528 - accuracy: 0.9972 - val_loss: 1.3205 - val_accuracy: 0.6962\n",
      "Epoch 256/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0526 - accuracy: 0.9976 - val_loss: 1.3307 - val_accuracy: 0.6950\n",
      "Epoch 257/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0532 - accuracy: 0.9971 - val_loss: 1.3251 - val_accuracy: 0.6912\n",
      "Epoch 258/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0521 - accuracy: 0.9976 - val_loss: 1.3341 - val_accuracy: 0.6887\n",
      "Epoch 259/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0522 - accuracy: 0.9972 - val_loss: 1.3401 - val_accuracy: 0.6875\n",
      "Epoch 260/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0525 - accuracy: 0.9970 - val_loss: 1.3226 - val_accuracy: 0.6925\n",
      "Epoch 261/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0522 - accuracy: 0.9974 - val_loss: 1.3149 - val_accuracy: 0.6962\n",
      "Epoch 262/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0521 - accuracy: 0.9978 - val_loss: 1.3202 - val_accuracy: 0.7012\n",
      "Epoch 263/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0539 - accuracy: 0.9968 - val_loss: 1.2734 - val_accuracy: 0.7012\n",
      "Epoch 264/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0517 - accuracy: 0.9977 - val_loss: 1.2478 - val_accuracy: 0.6950\n",
      "Epoch 265/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0536 - accuracy: 0.9970 - val_loss: 1.2819 - val_accuracy: 0.6912\n",
      "Epoch 266/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0527 - accuracy: 0.9966 - val_loss: 1.3050 - val_accuracy: 0.6912\n",
      "Epoch 267/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0521 - accuracy: 0.9974 - val_loss: 1.3046 - val_accuracy: 0.6975\n",
      "Epoch 268/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0499 - accuracy: 0.9986 - val_loss: 1.3043 - val_accuracy: 0.6987\n",
      "Epoch 269/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0505 - accuracy: 0.9978 - val_loss: 1.3053 - val_accuracy: 0.7050\n",
      "Epoch 270/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0509 - accuracy: 0.9978 - val_loss: 1.3123 - val_accuracy: 0.7000\n",
      "Epoch 271/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0513 - accuracy: 0.9972 - val_loss: 1.3052 - val_accuracy: 0.7025\n",
      "Epoch 272/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0503 - accuracy: 0.9978 - val_loss: 1.3256 - val_accuracy: 0.7025\n",
      "Epoch 273/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0517 - accuracy: 0.9972 - val_loss: 1.3468 - val_accuracy: 0.7012\n",
      "Epoch 274/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0521 - accuracy: 0.9973 - val_loss: 1.3499 - val_accuracy: 0.7050\n",
      "Epoch 275/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0517 - accuracy: 0.9973 - val_loss: 1.3452 - val_accuracy: 0.7000\n",
      "Epoch 276/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0520 - accuracy: 0.9970 - val_loss: 1.3473 - val_accuracy: 0.6900\n",
      "Epoch 277/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0511 - accuracy: 0.9978 - val_loss: 1.2329 - val_accuracy: 0.6687\n",
      "Epoch 278/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0500 - accuracy: 0.9978 - val_loss: 1.2363 - val_accuracy: 0.6675\n",
      "Epoch 279/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0519 - accuracy: 0.9971 - val_loss: 1.2301 - val_accuracy: 0.6850\n",
      "Epoch 280/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0503 - accuracy: 0.9980 - val_loss: 1.2231 - val_accuracy: 0.6962\n",
      "Epoch 281/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0512 - accuracy: 0.9975 - val_loss: 1.2401 - val_accuracy: 0.6975\n",
      "Epoch 282/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0522 - accuracy: 0.9968 - val_loss: 1.2609 - val_accuracy: 0.6912\n",
      "Epoch 283/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0514 - accuracy: 0.9975 - val_loss: 1.2633 - val_accuracy: 0.6950\n",
      "Epoch 284/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0499 - accuracy: 0.9978 - val_loss: 1.2638 - val_accuracy: 0.6962\n",
      "Epoch 285/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0512 - accuracy: 0.9974 - val_loss: 1.2678 - val_accuracy: 0.6925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 286/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0511 - accuracy: 0.9977 - val_loss: 1.2994 - val_accuracy: 0.6937\n",
      "Epoch 287/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0513 - accuracy: 0.9971 - val_loss: 1.3234 - val_accuracy: 0.6987\n",
      "Epoch 288/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0495 - accuracy: 0.9976 - val_loss: 1.3240 - val_accuracy: 0.7063\n",
      "Epoch 289/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0503 - accuracy: 0.9976 - val_loss: 1.3374 - val_accuracy: 0.7037\n",
      "Epoch 290/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0511 - accuracy: 0.9972 - val_loss: 1.3522 - val_accuracy: 0.7000\n",
      "Epoch 291/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0503 - accuracy: 0.9976 - val_loss: 1.3479 - val_accuracy: 0.7075\n",
      "Epoch 292/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0505 - accuracy: 0.9976 - val_loss: 1.3430 - val_accuracy: 0.7037\n",
      "Epoch 293/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0492 - accuracy: 0.9980 - val_loss: 1.3293 - val_accuracy: 0.7000\n",
      "Epoch 294/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0508 - accuracy: 0.9971 - val_loss: 1.3115 - val_accuracy: 0.7050\n",
      "Epoch 295/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0503 - accuracy: 0.9977 - val_loss: 1.1906 - val_accuracy: 0.6800\n",
      "Epoch 296/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0501 - accuracy: 0.9977 - val_loss: 1.2131 - val_accuracy: 0.6712\n",
      "Epoch 297/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0498 - accuracy: 0.9979 - val_loss: 1.2130 - val_accuracy: 0.6800\n",
      "Epoch 298/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0494 - accuracy: 0.9980 - val_loss: 1.2367 - val_accuracy: 0.6762\n",
      "Epoch 299/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0490 - accuracy: 0.9978 - val_loss: 1.2545 - val_accuracy: 0.6775\n",
      "Epoch 300/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0494 - accuracy: 0.9978 - val_loss: 1.3130 - val_accuracy: 0.6875\n",
      "Epoch 301/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0493 - accuracy: 0.9981 - val_loss: 1.3357 - val_accuracy: 0.6850\n",
      "Epoch 302/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0494 - accuracy: 0.9978 - val_loss: 1.3438 - val_accuracy: 0.6887\n",
      "Epoch 303/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0494 - accuracy: 0.9974 - val_loss: 1.4239 - val_accuracy: 0.6900\n",
      "Epoch 304/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0508 - accuracy: 0.9967 - val_loss: 1.4732 - val_accuracy: 0.6925\n",
      "Epoch 305/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0508 - accuracy: 0.9974 - val_loss: 1.4614 - val_accuracy: 0.7025\n",
      "Epoch 306/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9977 - val_loss: 1.4554 - val_accuracy: 0.7000\n",
      "Epoch 307/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0514 - accuracy: 0.9970 - val_loss: 1.4393 - val_accuracy: 0.6987\n",
      "Epoch 308/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0492 - accuracy: 0.9977 - val_loss: 1.4441 - val_accuracy: 0.6987\n",
      "Epoch 309/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0497 - accuracy: 0.9978 - val_loss: 1.4252 - val_accuracy: 0.7088\n",
      "Epoch 310/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9978 - val_loss: 1.4478 - val_accuracy: 0.7050\n",
      "Epoch 311/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0491 - accuracy: 0.9978 - val_loss: 1.4368 - val_accuracy: 0.7025\n",
      "Epoch 312/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0498 - accuracy: 0.9975 - val_loss: 1.4544 - val_accuracy: 0.7050\n",
      "Epoch 313/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0505 - accuracy: 0.9971 - val_loss: 1.4299 - val_accuracy: 0.7012\n",
      "Epoch 314/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0497 - accuracy: 0.9981 - val_loss: 1.4102 - val_accuracy: 0.7000\n",
      "Epoch 315/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0502 - accuracy: 0.9975 - val_loss: 1.4048 - val_accuracy: 0.6975\n",
      "Epoch 316/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0493 - accuracy: 0.9975 - val_loss: 1.4310 - val_accuracy: 0.7050\n",
      "Epoch 317/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9981 - val_loss: 1.4480 - val_accuracy: 0.7000\n",
      "Epoch 318/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0491 - accuracy: 0.9976 - val_loss: 1.4419 - val_accuracy: 0.7012\n",
      "Epoch 319/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0492 - accuracy: 0.9978 - val_loss: 1.4208 - val_accuracy: 0.6987\n",
      "Epoch 320/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0499 - accuracy: 0.9977 - val_loss: 1.3915 - val_accuracy: 0.7000\n",
      "Epoch 321/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0491 - accuracy: 0.9974 - val_loss: 1.3802 - val_accuracy: 0.6975\n",
      "Epoch 322/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0492 - accuracy: 0.9972 - val_loss: 1.3643 - val_accuracy: 0.6950\n",
      "Epoch 323/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0504 - accuracy: 0.9974 - val_loss: 1.3467 - val_accuracy: 0.6937\n",
      "Epoch 324/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0486 - accuracy: 0.9978 - val_loss: 1.3511 - val_accuracy: 0.6975\n",
      "Epoch 325/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0501 - accuracy: 0.9972 - val_loss: 1.3468 - val_accuracy: 0.6975\n",
      "Epoch 326/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9974 - val_loss: 1.3677 - val_accuracy: 0.7012\n",
      "Epoch 327/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9982 - val_loss: 1.3577 - val_accuracy: 0.7088\n",
      "Epoch 328/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9982 - val_loss: 1.3559 - val_accuracy: 0.7012\n",
      "Epoch 329/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0481 - accuracy: 0.9980 - val_loss: 1.3551 - val_accuracy: 0.7000\n",
      "Epoch 330/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0480 - accuracy: 0.9980 - val_loss: 1.3225 - val_accuracy: 0.7012\n",
      "Epoch 331/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0488 - accuracy: 0.9974 - val_loss: 1.3294 - val_accuracy: 0.6950\n",
      "Epoch 332/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0501 - accuracy: 0.9971 - val_loss: 1.3218 - val_accuracy: 0.6950\n",
      "Epoch 333/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0482 - accuracy: 0.9979 - val_loss: 1.3351 - val_accuracy: 0.6925\n",
      "Epoch 334/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0501 - accuracy: 0.9976 - val_loss: 1.3428 - val_accuracy: 0.6962\n",
      "Epoch 335/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0480 - accuracy: 0.9978 - val_loss: 1.2507 - val_accuracy: 0.6762\n",
      "Epoch 336/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9976 - val_loss: 1.2545 - val_accuracy: 0.6675\n",
      "Epoch 337/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0482 - accuracy: 0.9985 - val_loss: 1.2570 - val_accuracy: 0.6775\n",
      "Epoch 338/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0491 - accuracy: 0.9976 - val_loss: 1.2681 - val_accuracy: 0.6887\n",
      "Epoch 339/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0476 - accuracy: 0.9979 - val_loss: 1.2842 - val_accuracy: 0.6875\n",
      "Epoch 340/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0504 - accuracy: 0.9968 - val_loss: 1.2964 - val_accuracy: 0.6850\n",
      "Epoch 341/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0493 - accuracy: 0.9973 - val_loss: 1.3197 - val_accuracy: 0.7000\n",
      "Epoch 342/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0480 - accuracy: 0.9978 - val_loss: 1.3597 - val_accuracy: 0.6925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 343/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9978 - val_loss: 1.3765 - val_accuracy: 0.6950\n",
      "Epoch 344/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0494 - accuracy: 0.9970 - val_loss: 1.3928 - val_accuracy: 0.6987\n",
      "Epoch 345/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9976 - val_loss: 1.4041 - val_accuracy: 0.6937\n",
      "Epoch 346/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0490 - accuracy: 0.9978 - val_loss: 1.3907 - val_accuracy: 0.6950\n",
      "Epoch 347/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0499 - accuracy: 0.9975 - val_loss: 1.3774 - val_accuracy: 0.6975\n",
      "Epoch 348/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0497 - accuracy: 0.9970 - val_loss: 1.3727 - val_accuracy: 0.6987\n",
      "Epoch 349/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0484 - accuracy: 0.9974 - val_loss: 1.3982 - val_accuracy: 0.6950\n",
      "Epoch 350/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0482 - accuracy: 0.9978 - val_loss: 1.4162 - val_accuracy: 0.6912\n",
      "Epoch 351/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0489 - accuracy: 0.9974 - val_loss: 1.3772 - val_accuracy: 0.6950\n",
      "Epoch 352/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0484 - accuracy: 0.9972 - val_loss: 1.3880 - val_accuracy: 0.7025\n",
      "Epoch 353/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0486 - accuracy: 0.9976 - val_loss: 1.3742 - val_accuracy: 0.7050\n",
      "Epoch 354/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0500 - accuracy: 0.9970 - val_loss: 1.3789 - val_accuracy: 0.7075\n",
      "Epoch 355/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0483 - accuracy: 0.9978 - val_loss: 1.3466 - val_accuracy: 0.7050\n",
      "Epoch 356/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0489 - accuracy: 0.9973 - val_loss: 1.3484 - val_accuracy: 0.7000\n",
      "Epoch 357/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0486 - accuracy: 0.9974 - val_loss: 1.3557 - val_accuracy: 0.6962\n",
      "Epoch 358/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0486 - accuracy: 0.9975 - val_loss: 1.3467 - val_accuracy: 0.6912\n",
      "Epoch 359/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9971 - val_loss: 1.3596 - val_accuracy: 0.6900\n",
      "Epoch 360/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0493 - accuracy: 0.9975 - val_loss: 1.3687 - val_accuracy: 0.6975\n",
      "Epoch 361/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9982 - val_loss: 1.3976 - val_accuracy: 0.6800\n",
      "Epoch 362/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0477 - accuracy: 0.9977 - val_loss: 1.4352 - val_accuracy: 0.6900\n",
      "Epoch 363/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0487 - accuracy: 0.9974 - val_loss: 1.4601 - val_accuracy: 0.6937\n",
      "Epoch 364/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0486 - accuracy: 0.9976 - val_loss: 1.4312 - val_accuracy: 0.6925\n",
      "Epoch 365/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0482 - accuracy: 0.9978 - val_loss: 1.3204 - val_accuracy: 0.6862\n",
      "Epoch 366/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0484 - accuracy: 0.9977 - val_loss: 1.3330 - val_accuracy: 0.6812\n",
      "Epoch 367/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0483 - accuracy: 0.9976 - val_loss: 1.3513 - val_accuracy: 0.6787\n",
      "Epoch 368/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0489 - accuracy: 0.9971 - val_loss: 1.3518 - val_accuracy: 0.6837\n",
      "Epoch 369/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0491 - accuracy: 0.9977 - val_loss: 1.3822 - val_accuracy: 0.6812\n",
      "Epoch 370/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9981 - val_loss: 1.3961 - val_accuracy: 0.6812\n",
      "Epoch 371/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9972 - val_loss: 1.4081 - val_accuracy: 0.6800\n",
      "Epoch 372/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0480 - accuracy: 0.9974 - val_loss: 1.4155 - val_accuracy: 0.6800\n",
      "Epoch 373/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0496 - accuracy: 0.9970 - val_loss: 1.4350 - val_accuracy: 0.6812\n",
      "Epoch 374/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0495 - accuracy: 0.9975 - val_loss: 1.4560 - val_accuracy: 0.6850\n",
      "Epoch 375/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9980 - val_loss: 1.4430 - val_accuracy: 0.6812\n",
      "Epoch 376/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9976 - val_loss: 1.4200 - val_accuracy: 0.6875\n",
      "Epoch 377/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0474 - accuracy: 0.9979 - val_loss: 1.4213 - val_accuracy: 0.6887\n",
      "Epoch 378/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9972 - val_loss: 1.4295 - val_accuracy: 0.6887\n",
      "Epoch 379/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0485 - accuracy: 0.9974 - val_loss: 1.4366 - val_accuracy: 0.6925\n",
      "Epoch 380/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0481 - accuracy: 0.9978 - val_loss: 1.4228 - val_accuracy: 0.6875\n",
      "Epoch 381/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0482 - accuracy: 0.9980 - val_loss: 1.4051 - val_accuracy: 0.6887\n",
      "Epoch 382/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9977 - val_loss: 1.4214 - val_accuracy: 0.6837\n",
      "Epoch 383/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0476 - accuracy: 0.9978 - val_loss: 1.4279 - val_accuracy: 0.6912\n",
      "Epoch 384/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0482 - accuracy: 0.9970 - val_loss: 1.4470 - val_accuracy: 0.6975\n",
      "Epoch 385/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0496 - accuracy: 0.9968 - val_loss: 1.4493 - val_accuracy: 0.6925\n",
      "Epoch 386/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0477 - accuracy: 0.9978 - val_loss: 1.4690 - val_accuracy: 0.6900\n",
      "Epoch 387/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0491 - accuracy: 0.9972 - val_loss: 1.4991 - val_accuracy: 0.6925\n",
      "Epoch 388/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0486 - accuracy: 0.9972 - val_loss: 1.4943 - val_accuracy: 0.6925\n",
      "Epoch 389/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0477 - accuracy: 0.9976 - val_loss: 1.4857 - val_accuracy: 0.6925\n",
      "Epoch 390/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0497 - accuracy: 0.9972 - val_loss: 1.4785 - val_accuracy: 0.6912\n",
      "Epoch 391/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0490 - accuracy: 0.9970 - val_loss: 1.4819 - val_accuracy: 0.6962\n",
      "Epoch 392/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0482 - accuracy: 0.9971 - val_loss: 1.4719 - val_accuracy: 0.6950\n",
      "Epoch 393/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0479 - accuracy: 0.9978 - val_loss: 1.4873 - val_accuracy: 0.6875\n",
      "Epoch 394/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9980 - val_loss: 1.4799 - val_accuracy: 0.6937\n",
      "Epoch 395/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0481 - accuracy: 0.9975 - val_loss: 1.4543 - val_accuracy: 0.6975\n",
      "Epoch 396/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0495 - accuracy: 0.9972 - val_loss: 1.4375 - val_accuracy: 0.6987\n",
      "Epoch 397/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0468 - accuracy: 0.9982 - val_loss: 1.4427 - val_accuracy: 0.6950\n",
      "Epoch 398/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0488 - accuracy: 0.9970 - val_loss: 1.4124 - val_accuracy: 0.6912\n",
      "Epoch 399/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9976 - val_loss: 1.4090 - val_accuracy: 0.6887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 400/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0487 - accuracy: 0.9975 - val_loss: 1.4223 - val_accuracy: 0.6812\n",
      "Epoch 401/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0483 - accuracy: 0.9979 - val_loss: 1.4282 - val_accuracy: 0.6962\n",
      "Epoch 402/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0477 - accuracy: 0.9980 - val_loss: 1.4272 - val_accuracy: 0.6912\n",
      "Epoch 403/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0472 - accuracy: 0.9980 - val_loss: 1.4352 - val_accuracy: 0.6950\n",
      "Epoch 404/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0473 - accuracy: 0.9978 - val_loss: 1.4424 - val_accuracy: 0.6912\n",
      "Epoch 405/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9972 - val_loss: 1.4374 - val_accuracy: 0.6950\n",
      "Epoch 406/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0469 - accuracy: 0.9982 - val_loss: 1.4185 - val_accuracy: 0.6950\n",
      "Epoch 407/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9976 - val_loss: 1.4075 - val_accuracy: 0.6937\n",
      "Epoch 408/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0473 - accuracy: 0.9976 - val_loss: 1.3986 - val_accuracy: 0.6950\n",
      "Epoch 409/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9978 - val_loss: 1.4230 - val_accuracy: 0.6887\n",
      "Epoch 410/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9975 - val_loss: 1.4392 - val_accuracy: 0.6837\n",
      "Epoch 411/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0472 - accuracy: 0.9980 - val_loss: 1.4426 - val_accuracy: 0.6787\n",
      "Epoch 412/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0479 - accuracy: 0.9976 - val_loss: 1.3510 - val_accuracy: 0.6837\n",
      "Epoch 413/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0468 - accuracy: 0.9979 - val_loss: 1.3258 - val_accuracy: 0.6750\n",
      "Epoch 414/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9983 - val_loss: 1.3244 - val_accuracy: 0.6775\n",
      "Epoch 415/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0472 - accuracy: 0.9980 - val_loss: 1.3401 - val_accuracy: 0.6850\n",
      "Epoch 416/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9972 - val_loss: 1.3543 - val_accuracy: 0.6900\n",
      "Epoch 417/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9974 - val_loss: 1.3594 - val_accuracy: 0.6925\n",
      "Epoch 418/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0481 - accuracy: 0.9975 - val_loss: 1.3789 - val_accuracy: 0.6925\n",
      "Epoch 419/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0461 - accuracy: 0.9982 - val_loss: 1.3986 - val_accuracy: 0.6850\n",
      "Epoch 420/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9984 - val_loss: 1.4072 - val_accuracy: 0.6837\n",
      "Epoch 421/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9983 - val_loss: 1.4090 - val_accuracy: 0.6900\n",
      "Epoch 422/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9975 - val_loss: 1.4078 - val_accuracy: 0.6925\n",
      "Epoch 423/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0480 - accuracy: 0.9976 - val_loss: 1.3985 - val_accuracy: 0.6987\n",
      "Epoch 424/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0467 - accuracy: 0.9976 - val_loss: 1.3797 - val_accuracy: 0.6987\n",
      "Epoch 425/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9982 - val_loss: 1.3950 - val_accuracy: 0.6975\n",
      "Epoch 426/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0462 - accuracy: 0.9978 - val_loss: 1.4021 - val_accuracy: 0.6975\n",
      "Epoch 427/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9981 - val_loss: 1.4075 - val_accuracy: 0.6975\n",
      "Epoch 428/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9977 - val_loss: 1.4058 - val_accuracy: 0.7012\n",
      "Epoch 429/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0484 - accuracy: 0.9972 - val_loss: 1.4127 - val_accuracy: 0.7025\n",
      "Epoch 430/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0467 - accuracy: 0.9976 - val_loss: 1.4183 - val_accuracy: 0.7000\n",
      "Epoch 431/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0461 - accuracy: 0.9981 - val_loss: 1.4318 - val_accuracy: 0.6950\n",
      "Epoch 432/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0463 - accuracy: 0.9978 - val_loss: 1.4574 - val_accuracy: 0.6925\n",
      "Epoch 433/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9980 - val_loss: 1.4574 - val_accuracy: 0.6875\n",
      "Epoch 434/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9973 - val_loss: 1.4439 - val_accuracy: 0.6912\n",
      "Epoch 435/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0475 - accuracy: 0.9977 - val_loss: 1.4354 - val_accuracy: 0.6950\n",
      "Epoch 436/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9975 - val_loss: 1.4248 - val_accuracy: 0.7025\n",
      "Epoch 437/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9977 - val_loss: 1.4333 - val_accuracy: 0.7025\n",
      "Epoch 438/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9976 - val_loss: 1.4471 - val_accuracy: 0.7012\n",
      "Epoch 439/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0460 - accuracy: 0.9982 - val_loss: 1.4436 - val_accuracy: 0.6950\n",
      "Epoch 440/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0460 - accuracy: 0.9982 - val_loss: 1.4354 - val_accuracy: 0.7012\n",
      "Epoch 441/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9982 - val_loss: 1.4313 - val_accuracy: 0.6987\n",
      "Epoch 442/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9980 - val_loss: 1.4358 - val_accuracy: 0.6925\n",
      "Epoch 443/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0451 - accuracy: 0.9984 - val_loss: 1.4464 - val_accuracy: 0.6950\n",
      "Epoch 444/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0470 - accuracy: 0.9973 - val_loss: 1.4324 - val_accuracy: 0.7025\n",
      "Epoch 445/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9979 - val_loss: 1.4234 - val_accuracy: 0.6975\n",
      "Epoch 446/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9978 - val_loss: 1.4062 - val_accuracy: 0.7000\n",
      "Epoch 447/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9978 - val_loss: 1.3971 - val_accuracy: 0.6912\n",
      "Epoch 448/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0460 - accuracy: 0.9980 - val_loss: 1.3939 - val_accuracy: 0.6900\n",
      "Epoch 449/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9979 - val_loss: 1.3878 - val_accuracy: 0.6950\n",
      "Epoch 450/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9984 - val_loss: 1.3938 - val_accuracy: 0.7050\n",
      "Epoch 451/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0480 - accuracy: 0.9970 - val_loss: 1.3912 - val_accuracy: 0.6987\n",
      "Epoch 452/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9974 - val_loss: 1.3984 - val_accuracy: 0.7037\n",
      "Epoch 453/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9974 - val_loss: 1.4008 - val_accuracy: 0.7012\n",
      "Epoch 454/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0467 - accuracy: 0.9974 - val_loss: 1.4056 - val_accuracy: 0.6962\n",
      "Epoch 455/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0466 - accuracy: 0.9973 - val_loss: 1.4179 - val_accuracy: 0.7012\n",
      "Epoch 456/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0460 - accuracy: 0.9976 - val_loss: 1.4353 - val_accuracy: 0.7025\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 457/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9978 - val_loss: 1.5090 - val_accuracy: 0.7113\n",
      "Epoch 458/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0466 - accuracy: 0.9974 - val_loss: 1.5100 - val_accuracy: 0.6987\n",
      "Epoch 459/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0470 - accuracy: 0.9979 - val_loss: 1.4727 - val_accuracy: 0.7000\n",
      "Epoch 460/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0467 - accuracy: 0.9973 - val_loss: 1.4696 - val_accuracy: 0.7012\n",
      "Epoch 461/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0457 - accuracy: 0.9981 - val_loss: 1.4394 - val_accuracy: 0.7050\n",
      "Epoch 462/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0465 - accuracy: 0.9976 - val_loss: 1.4014 - val_accuracy: 0.7012\n",
      "Epoch 463/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0474 - accuracy: 0.9974 - val_loss: 1.4027 - val_accuracy: 0.6975\n",
      "Epoch 464/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0482 - accuracy: 0.9974 - val_loss: 1.4139 - val_accuracy: 0.6987\n",
      "Epoch 465/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9981 - val_loss: 1.4737 - val_accuracy: 0.6962\n",
      "Epoch 466/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0454 - accuracy: 0.9980 - val_loss: 1.5647 - val_accuracy: 0.7100\n",
      "Epoch 467/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0466 - accuracy: 0.9980 - val_loss: 1.5541 - val_accuracy: 0.7063\n",
      "Epoch 468/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0468 - accuracy: 0.9978 - val_loss: 1.5334 - val_accuracy: 0.7037\n",
      "Epoch 469/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9978 - val_loss: 1.4954 - val_accuracy: 0.7050\n",
      "Epoch 470/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9976 - val_loss: 1.4441 - val_accuracy: 0.7000\n",
      "Epoch 471/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9981 - val_loss: 1.4239 - val_accuracy: 0.7100\n",
      "Epoch 472/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0467 - accuracy: 0.9975 - val_loss: 1.4206 - val_accuracy: 0.7075\n",
      "Epoch 473/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0463 - accuracy: 0.9978 - val_loss: 1.4086 - val_accuracy: 0.7063\n",
      "Epoch 474/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9983 - val_loss: 1.3920 - val_accuracy: 0.7075\n",
      "Epoch 475/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0467 - accuracy: 0.9972 - val_loss: 1.3727 - val_accuracy: 0.7088\n",
      "Epoch 476/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0445 - accuracy: 0.9981 - val_loss: 1.3727 - val_accuracy: 0.7063\n",
      "Epoch 477/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9973 - val_loss: 1.3797 - val_accuracy: 0.7037\n",
      "Epoch 478/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9978 - val_loss: 1.3777 - val_accuracy: 0.7100\n",
      "Epoch 479/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9973 - val_loss: 1.3566 - val_accuracy: 0.7163\n",
      "Epoch 480/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0468 - accuracy: 0.9976 - val_loss: 1.3522 - val_accuracy: 0.7063\n",
      "Epoch 481/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0478 - accuracy: 0.9978 - val_loss: 1.3811 - val_accuracy: 0.7088\n",
      "Epoch 482/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9983 - val_loss: 1.4063 - val_accuracy: 0.7113\n",
      "Epoch 483/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9976 - val_loss: 1.3970 - val_accuracy: 0.7113\n",
      "Epoch 484/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0468 - accuracy: 0.9977 - val_loss: 1.3985 - val_accuracy: 0.7100\n",
      "Epoch 485/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9977 - val_loss: 1.3933 - val_accuracy: 0.7088\n",
      "Epoch 486/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0463 - accuracy: 0.9980 - val_loss: 1.4133 - val_accuracy: 0.7075\n",
      "Epoch 487/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9982 - val_loss: 1.4243 - val_accuracy: 0.7088\n",
      "Epoch 488/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9982 - val_loss: 1.4325 - val_accuracy: 0.7138\n",
      "Epoch 489/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9977 - val_loss: 1.4282 - val_accuracy: 0.7088\n",
      "Epoch 490/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0451 - accuracy: 0.9985 - val_loss: 1.4031 - val_accuracy: 0.7113\n",
      "Epoch 491/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9978 - val_loss: 1.3903 - val_accuracy: 0.7050\n",
      "Epoch 492/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9979 - val_loss: 1.4277 - val_accuracy: 0.7138\n",
      "Epoch 493/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0445 - accuracy: 0.9984 - val_loss: 1.5122 - val_accuracy: 0.7075\n",
      "Epoch 494/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0468 - accuracy: 0.9976 - val_loss: 1.5023 - val_accuracy: 0.7063\n",
      "Epoch 495/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9980 - val_loss: 1.5000 - val_accuracy: 0.7088\n",
      "Epoch 496/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9978 - val_loss: 1.4834 - val_accuracy: 0.7050\n",
      "Epoch 497/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9982 - val_loss: 1.4636 - val_accuracy: 0.7012\n",
      "Epoch 498/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9979 - val_loss: 1.4447 - val_accuracy: 0.7025\n",
      "Epoch 499/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0477 - accuracy: 0.9976 - val_loss: 1.4346 - val_accuracy: 0.7012\n",
      "Epoch 500/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0470 - accuracy: 0.9972 - val_loss: 1.4486 - val_accuracy: 0.6962\n",
      "Epoch 501/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9972 - val_loss: 1.4619 - val_accuracy: 0.7000\n",
      "Epoch 502/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9983 - val_loss: 1.4556 - val_accuracy: 0.6925\n",
      "Epoch 503/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9974 - val_loss: 1.4590 - val_accuracy: 0.6900\n",
      "Epoch 504/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9980 - val_loss: 1.4379 - val_accuracy: 0.6937\n",
      "Epoch 505/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0471 - accuracy: 0.9972 - val_loss: 1.4318 - val_accuracy: 0.7000\n",
      "Epoch 506/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9978 - val_loss: 1.4398 - val_accuracy: 0.7025\n",
      "Epoch 507/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0466 - accuracy: 0.9976 - val_loss: 1.4550 - val_accuracy: 0.6987\n",
      "Epoch 508/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0469 - accuracy: 0.9974 - val_loss: 1.4469 - val_accuracy: 0.6962\n",
      "Epoch 509/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9982 - val_loss: 1.4296 - val_accuracy: 0.6975\n",
      "Epoch 510/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9984 - val_loss: 1.4087 - val_accuracy: 0.6987\n",
      "Epoch 511/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9978 - val_loss: 1.4071 - val_accuracy: 0.7025\n",
      "Epoch 512/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9980 - val_loss: 1.4200 - val_accuracy: 0.6987\n",
      "Epoch 513/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9980 - val_loss: 1.4235 - val_accuracy: 0.6962\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 514/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9980 - val_loss: 1.4400 - val_accuracy: 0.7012\n",
      "Epoch 515/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9978 - val_loss: 1.4284 - val_accuracy: 0.7037\n",
      "Epoch 516/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9979 - val_loss: 1.4064 - val_accuracy: 0.7050\n",
      "Epoch 517/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0445 - accuracy: 0.9982 - val_loss: 1.4199 - val_accuracy: 0.7050\n",
      "Epoch 518/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9979 - val_loss: 1.4204 - val_accuracy: 0.6987\n",
      "Epoch 519/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9984 - val_loss: 1.4415 - val_accuracy: 0.7037\n",
      "Epoch 520/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0454 - accuracy: 0.9978 - val_loss: 1.4379 - val_accuracy: 0.7063\n",
      "Epoch 521/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9980 - val_loss: 1.4255 - val_accuracy: 0.7075\n",
      "Epoch 522/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9984 - val_loss: 1.4144 - val_accuracy: 0.7063\n",
      "Epoch 523/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9982 - val_loss: 1.4278 - val_accuracy: 0.7050\n",
      "Epoch 524/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9987 - val_loss: 1.4231 - val_accuracy: 0.6975\n",
      "Epoch 525/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9982 - val_loss: 1.4110 - val_accuracy: 0.6975\n",
      "Epoch 526/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0462 - accuracy: 0.9973 - val_loss: 1.3855 - val_accuracy: 0.7050\n",
      "Epoch 527/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9976 - val_loss: 1.3811 - val_accuracy: 0.7037\n",
      "Epoch 528/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9980 - val_loss: 1.3848 - val_accuracy: 0.6987\n",
      "Epoch 529/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9977 - val_loss: 1.4019 - val_accuracy: 0.7063\n",
      "Epoch 530/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0446 - accuracy: 0.9981 - val_loss: 1.4142 - val_accuracy: 0.7063\n",
      "Epoch 531/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9976 - val_loss: 1.4196 - val_accuracy: 0.7050\n",
      "Epoch 532/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9977 - val_loss: 1.4007 - val_accuracy: 0.7125\n",
      "Epoch 533/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0461 - accuracy: 0.9974 - val_loss: 1.4122 - val_accuracy: 0.7100\n",
      "Epoch 534/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9980 - val_loss: 1.4092 - val_accuracy: 0.7150\n",
      "Epoch 535/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9986 - val_loss: 1.4026 - val_accuracy: 0.7188\n",
      "Epoch 536/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9976 - val_loss: 1.3906 - val_accuracy: 0.7125\n",
      "Epoch 537/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9980 - val_loss: 1.3949 - val_accuracy: 0.7163\n",
      "Epoch 538/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0451 - accuracy: 0.9978 - val_loss: 1.4268 - val_accuracy: 0.7050\n",
      "Epoch 539/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9977 - val_loss: 1.4347 - val_accuracy: 0.7063\n",
      "Epoch 540/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9980 - val_loss: 1.4567 - val_accuracy: 0.7088\n",
      "Epoch 541/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9984 - val_loss: 1.4465 - val_accuracy: 0.6975\n",
      "Epoch 542/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9978 - val_loss: 1.4119 - val_accuracy: 0.7075\n",
      "Epoch 543/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9979 - val_loss: 1.4120 - val_accuracy: 0.7063\n",
      "Epoch 544/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0448 - accuracy: 0.9979 - val_loss: 1.3927 - val_accuracy: 0.7037\n",
      "Epoch 545/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9978 - val_loss: 1.3913 - val_accuracy: 0.7012\n",
      "Epoch 546/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9982 - val_loss: 1.4075 - val_accuracy: 0.7050\n",
      "Epoch 547/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9983 - val_loss: 1.4061 - val_accuracy: 0.7000\n",
      "Epoch 548/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0461 - accuracy: 0.9971 - val_loss: 1.4005 - val_accuracy: 0.6875\n",
      "Epoch 549/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9984 - val_loss: 1.4321 - val_accuracy: 0.6837\n",
      "Epoch 550/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9976 - val_loss: 1.4498 - val_accuracy: 0.6900\n",
      "Epoch 551/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0451 - accuracy: 0.9978 - val_loss: 1.4546 - val_accuracy: 0.6887\n",
      "Epoch 552/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9980 - val_loss: 1.4663 - val_accuracy: 0.6950\n",
      "Epoch 553/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9984 - val_loss: 1.4709 - val_accuracy: 0.6925\n",
      "Epoch 554/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9977 - val_loss: 1.4790 - val_accuracy: 0.6875\n",
      "Epoch 555/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9983 - val_loss: 1.4720 - val_accuracy: 0.6925\n",
      "Epoch 556/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9974 - val_loss: 1.4803 - val_accuracy: 0.6925\n",
      "Epoch 557/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9976 - val_loss: 1.4677 - val_accuracy: 0.6937\n",
      "Epoch 558/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9983 - val_loss: 1.4704 - val_accuracy: 0.6900\n",
      "Epoch 559/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9974 - val_loss: 1.4622 - val_accuracy: 0.6950\n",
      "Epoch 560/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9980 - val_loss: 1.4574 - val_accuracy: 0.6975\n",
      "Epoch 561/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9981 - val_loss: 1.4597 - val_accuracy: 0.6950\n",
      "Epoch 562/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9980 - val_loss: 1.4539 - val_accuracy: 0.7088\n",
      "Epoch 563/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0449 - accuracy: 0.9980 - val_loss: 1.4262 - val_accuracy: 0.7100\n",
      "Epoch 564/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9978 - val_loss: 1.4023 - val_accuracy: 0.7063\n",
      "Epoch 565/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9978 - val_loss: 1.3718 - val_accuracy: 0.7050\n",
      "Epoch 566/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9976 - val_loss: 1.3641 - val_accuracy: 0.7000\n",
      "Epoch 567/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0451 - accuracy: 0.9981 - val_loss: 1.3312 - val_accuracy: 0.6975\n",
      "Epoch 568/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0461 - accuracy: 0.9978 - val_loss: 1.3265 - val_accuracy: 0.6925\n",
      "Epoch 569/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9978 - val_loss: 1.3301 - val_accuracy: 0.7000\n",
      "Epoch 570/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9981 - val_loss: 1.3463 - val_accuracy: 0.6987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 571/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9983 - val_loss: 1.3586 - val_accuracy: 0.6962\n",
      "Epoch 572/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9982 - val_loss: 1.3453 - val_accuracy: 0.6937\n",
      "Epoch 573/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9977 - val_loss: 1.3315 - val_accuracy: 0.6912\n",
      "Epoch 574/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9981 - val_loss: 1.3541 - val_accuracy: 0.6825\n",
      "Epoch 575/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9974 - val_loss: 1.2864 - val_accuracy: 0.6750\n",
      "Epoch 576/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9984 - val_loss: 1.2871 - val_accuracy: 0.6787\n",
      "Epoch 577/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9979 - val_loss: 1.3329 - val_accuracy: 0.6862\n",
      "Epoch 578/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9980 - val_loss: 1.3827 - val_accuracy: 0.6900\n",
      "Epoch 579/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9976 - val_loss: 1.4286 - val_accuracy: 0.6975\n",
      "Epoch 580/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0445 - accuracy: 0.9981 - val_loss: 1.4306 - val_accuracy: 0.6987\n",
      "Epoch 581/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9974 - val_loss: 1.5052 - val_accuracy: 0.6987\n",
      "Epoch 582/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9983 - val_loss: 1.4022 - val_accuracy: 0.6962\n",
      "Epoch 583/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9980 - val_loss: 1.3632 - val_accuracy: 0.6987\n",
      "Epoch 584/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9978 - val_loss: 1.3786 - val_accuracy: 0.6925\n",
      "Epoch 585/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9980 - val_loss: 1.4167 - val_accuracy: 0.7025\n",
      "Epoch 586/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0458 - accuracy: 0.9972 - val_loss: 1.3049 - val_accuracy: 0.6825\n",
      "Epoch 587/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9984 - val_loss: 1.3214 - val_accuracy: 0.6925\n",
      "Epoch 588/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9978 - val_loss: 1.3113 - val_accuracy: 0.6975\n",
      "Epoch 589/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9977 - val_loss: 1.3440 - val_accuracy: 0.6987\n",
      "Epoch 590/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0449 - accuracy: 0.9982 - val_loss: 1.3423 - val_accuracy: 0.6950\n",
      "Epoch 591/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9980 - val_loss: 1.4237 - val_accuracy: 0.5962\n",
      "Epoch 592/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9978 - val_loss: 1.4207 - val_accuracy: 0.6075\n",
      "Epoch 593/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0465 - accuracy: 0.9968 - val_loss: 1.3466 - val_accuracy: 0.6337\n",
      "Epoch 594/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9980 - val_loss: 1.3745 - val_accuracy: 0.6675\n",
      "Epoch 595/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9980 - val_loss: 4.5114 - val_accuracy: 0.6362\n",
      "Epoch 596/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0455 - accuracy: 0.9976 - val_loss: 3.9656 - val_accuracy: 0.6550\n",
      "Epoch 597/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9982 - val_loss: 3.3567 - val_accuracy: 0.6700\n",
      "Epoch 598/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9982 - val_loss: 2.8633 - val_accuracy: 0.6862\n",
      "Epoch 599/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9980 - val_loss: 2.5116 - val_accuracy: 0.6950\n",
      "Epoch 600/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9983 - val_loss: 2.1491 - val_accuracy: 0.6950\n",
      "Epoch 601/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9982 - val_loss: 1.9258 - val_accuracy: 0.7037\n",
      "Epoch 602/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9978 - val_loss: 1.8050 - val_accuracy: 0.7075\n",
      "Epoch 603/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9975 - val_loss: 1.7144 - val_accuracy: 0.7100\n",
      "Epoch 604/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0446 - accuracy: 0.9979 - val_loss: 1.6776 - val_accuracy: 0.7025\n",
      "Epoch 605/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9982 - val_loss: 1.6173 - val_accuracy: 0.7063\n",
      "Epoch 606/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0453 - accuracy: 0.9975 - val_loss: 1.5781 - val_accuracy: 0.7063\n",
      "Epoch 607/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0459 - accuracy: 0.9973 - val_loss: 1.5531 - val_accuracy: 0.7012\n",
      "Epoch 608/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9983 - val_loss: 1.5264 - val_accuracy: 0.7037\n",
      "Epoch 609/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0451 - accuracy: 0.9980 - val_loss: 1.5086 - val_accuracy: 0.7037\n",
      "Epoch 610/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9977 - val_loss: 1.4863 - val_accuracy: 0.7037\n",
      "Epoch 611/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9981 - val_loss: 1.4755 - val_accuracy: 0.7037\n",
      "Epoch 612/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9978 - val_loss: 1.4736 - val_accuracy: 0.6937\n",
      "Epoch 613/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9981 - val_loss: 1.4605 - val_accuracy: 0.6987\n",
      "Epoch 614/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9982 - val_loss: 1.4599 - val_accuracy: 0.7012\n",
      "Epoch 615/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0458 - accuracy: 0.9976 - val_loss: 1.4639 - val_accuracy: 0.7075\n",
      "Epoch 616/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0456 - accuracy: 0.9978 - val_loss: 1.4630 - val_accuracy: 0.7100\n",
      "Epoch 617/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9980 - val_loss: 1.4542 - val_accuracy: 0.7088\n",
      "Epoch 618/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0451 - accuracy: 0.9976 - val_loss: 1.4611 - val_accuracy: 0.7075\n",
      "Epoch 619/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9983 - val_loss: 1.4678 - val_accuracy: 0.7125\n",
      "Epoch 620/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9975 - val_loss: 1.4457 - val_accuracy: 0.7012\n",
      "Epoch 621/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0442 - accuracy: 0.9984 - val_loss: 1.4391 - val_accuracy: 0.7000\n",
      "Epoch 622/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9981 - val_loss: 1.4238 - val_accuracy: 0.7000\n",
      "Epoch 623/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9974 - val_loss: 1.4266 - val_accuracy: 0.6975\n",
      "Epoch 624/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9978 - val_loss: 1.4234 - val_accuracy: 0.6950\n",
      "Epoch 625/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9980 - val_loss: 1.4114 - val_accuracy: 0.6975\n",
      "Epoch 626/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0446 - accuracy: 0.9984 - val_loss: 1.4314 - val_accuracy: 0.6975\n",
      "Epoch 627/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9980 - val_loss: 1.4295 - val_accuracy: 0.7000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 628/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9975 - val_loss: 1.4254 - val_accuracy: 0.7050\n",
      "Epoch 629/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9982 - val_loss: 1.4367 - val_accuracy: 0.7100\n",
      "Epoch 630/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0457 - accuracy: 0.9978 - val_loss: 1.4486 - val_accuracy: 0.6975\n",
      "Epoch 631/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9986 - val_loss: 1.4238 - val_accuracy: 0.7025\n",
      "Epoch 632/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0448 - accuracy: 0.9978 - val_loss: 1.4069 - val_accuracy: 0.7063\n",
      "Epoch 633/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0455 - accuracy: 0.9980 - val_loss: 1.3966 - val_accuracy: 0.7100\n",
      "Epoch 634/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0464 - accuracy: 0.9968 - val_loss: 1.3728 - val_accuracy: 0.7088\n",
      "Epoch 635/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9976 - val_loss: 1.3669 - val_accuracy: 0.7063\n",
      "Epoch 636/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9983 - val_loss: 1.4109 - val_accuracy: 0.7050\n",
      "Epoch 637/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9982 - val_loss: 1.4342 - val_accuracy: 0.7113\n",
      "Epoch 638/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9983 - val_loss: 1.4281 - val_accuracy: 0.7088\n",
      "Epoch 639/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0445 - accuracy: 0.9976 - val_loss: 1.4148 - val_accuracy: 0.7088\n",
      "Epoch 640/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9978 - val_loss: 1.4159 - val_accuracy: 0.7037\n",
      "Epoch 641/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9980 - val_loss: 1.4306 - val_accuracy: 0.7050\n",
      "Epoch 642/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9984 - val_loss: 1.4354 - val_accuracy: 0.7050\n",
      "Epoch 643/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0436 - accuracy: 0.9984 - val_loss: 1.4313 - val_accuracy: 0.7025\n",
      "Epoch 644/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9982 - val_loss: 1.4294 - val_accuracy: 0.7075\n",
      "Epoch 645/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0447 - accuracy: 0.9975 - val_loss: 1.4131 - val_accuracy: 0.7100\n",
      "Epoch 646/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9978 - val_loss: 1.4223 - val_accuracy: 0.7063\n",
      "Epoch 647/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9984 - val_loss: 1.4494 - val_accuracy: 0.7138\n",
      "Epoch 648/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9988 - val_loss: 1.4576 - val_accuracy: 0.7100\n",
      "Epoch 649/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9984 - val_loss: 1.4293 - val_accuracy: 0.7125\n",
      "Epoch 650/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9982 - val_loss: 1.4183 - val_accuracy: 0.7100\n",
      "Epoch 651/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9981 - val_loss: 1.3936 - val_accuracy: 0.7163\n",
      "Epoch 652/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9984 - val_loss: 1.4100 - val_accuracy: 0.7075\n",
      "Epoch 653/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9982 - val_loss: 1.4147 - val_accuracy: 0.7150\n",
      "Epoch 654/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9976 - val_loss: 1.4190 - val_accuracy: 0.7225\n",
      "Epoch 655/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9984 - val_loss: 1.4019 - val_accuracy: 0.7188\n",
      "Epoch 656/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9980 - val_loss: 1.3881 - val_accuracy: 0.7212\n",
      "Epoch 657/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0448 - accuracy: 0.9974 - val_loss: 1.3788 - val_accuracy: 0.7138\n",
      "Epoch 658/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0421 - accuracy: 0.9986 - val_loss: 1.3817 - val_accuracy: 0.7138\n",
      "Epoch 659/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9984 - val_loss: 1.3840 - val_accuracy: 0.7037\n",
      "Epoch 660/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9977 - val_loss: 1.3882 - val_accuracy: 0.7037\n",
      "Epoch 661/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9980 - val_loss: 1.3883 - val_accuracy: 0.7025\n",
      "Epoch 662/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0441 - accuracy: 0.9978 - val_loss: 1.3889 - val_accuracy: 0.7012\n",
      "Epoch 663/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9980 - val_loss: 1.3741 - val_accuracy: 0.7025\n",
      "Epoch 664/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9983 - val_loss: 1.3767 - val_accuracy: 0.7012\n",
      "Epoch 665/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0436 - accuracy: 0.9981 - val_loss: 1.3968 - val_accuracy: 0.7037\n",
      "Epoch 666/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9982 - val_loss: 1.3976 - val_accuracy: 0.7000\n",
      "Epoch 667/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9982 - val_loss: 1.3950 - val_accuracy: 0.6987\n",
      "Epoch 668/1000\n",
      "19/19 [==============================] - 19s 999ms/step - loss: 0.0428 - accuracy: 0.9989 - val_loss: 1.4086 - val_accuracy: 0.6987\n",
      "Epoch 669/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0421 - accuracy: 0.9988 - val_loss: 1.4126 - val_accuracy: 0.7088\n",
      "Epoch 670/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9974 - val_loss: 1.3964 - val_accuracy: 0.7113\n",
      "Epoch 671/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9984 - val_loss: 1.3681 - val_accuracy: 0.7100\n",
      "Epoch 672/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9984 - val_loss: 1.3776 - val_accuracy: 0.7075\n",
      "Epoch 673/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9978 - val_loss: 1.3824 - val_accuracy: 0.7025\n",
      "Epoch 674/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 1.4016 - val_accuracy: 0.7100\n",
      "Epoch 675/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9982 - val_loss: 1.4019 - val_accuracy: 0.7037\n",
      "Epoch 676/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9974 - val_loss: 1.4099 - val_accuracy: 0.7025\n",
      "Epoch 677/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9984 - val_loss: 1.4248 - val_accuracy: 0.7012\n",
      "Epoch 678/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9979 - val_loss: 1.4184 - val_accuracy: 0.7025\n",
      "Epoch 679/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9977 - val_loss: 1.4117 - val_accuracy: 0.7050\n",
      "Epoch 680/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9982 - val_loss: 1.4057 - val_accuracy: 0.7012\n",
      "Epoch 681/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9979 - val_loss: 1.4113 - val_accuracy: 0.7012\n",
      "Epoch 682/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0435 - accuracy: 0.9980 - val_loss: 1.4425 - val_accuracy: 0.7063\n",
      "Epoch 683/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9982 - val_loss: 1.4326 - val_accuracy: 0.6975\n",
      "Epoch 684/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9975 - val_loss: 1.4108 - val_accuracy: 0.7012\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 685/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9983 - val_loss: 1.4290 - val_accuracy: 0.6987\n",
      "Epoch 686/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9988 - val_loss: 1.4411 - val_accuracy: 0.7050\n",
      "Epoch 687/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9978 - val_loss: 1.4641 - val_accuracy: 0.6987\n",
      "Epoch 688/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0433 - accuracy: 0.9978 - val_loss: 1.4506 - val_accuracy: 0.6937\n",
      "Epoch 689/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9980 - val_loss: 1.4746 - val_accuracy: 0.6937\n",
      "Epoch 690/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9985 - val_loss: 1.4768 - val_accuracy: 0.6887\n",
      "Epoch 691/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9979 - val_loss: 1.4274 - val_accuracy: 0.6912\n",
      "Epoch 692/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0426 - accuracy: 0.9984 - val_loss: 1.4053 - val_accuracy: 0.6937\n",
      "Epoch 693/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0435 - accuracy: 0.9983 - val_loss: 1.4000 - val_accuracy: 0.7012\n",
      "Epoch 694/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9980 - val_loss: 1.4099 - val_accuracy: 0.6950\n",
      "Epoch 695/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0454 - accuracy: 0.9976 - val_loss: 1.4588 - val_accuracy: 0.6987\n",
      "Epoch 696/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9980 - val_loss: 1.4751 - val_accuracy: 0.6937\n",
      "Epoch 697/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9979 - val_loss: 1.4767 - val_accuracy: 0.6950\n",
      "Epoch 698/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9981 - val_loss: 1.5049 - val_accuracy: 0.6925\n",
      "Epoch 699/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9984 - val_loss: 1.5085 - val_accuracy: 0.6912\n",
      "Epoch 700/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9980 - val_loss: 1.5347 - val_accuracy: 0.6887\n",
      "Epoch 701/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9980 - val_loss: 1.5378 - val_accuracy: 0.6887\n",
      "Epoch 702/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9982 - val_loss: 1.5283 - val_accuracy: 0.6925\n",
      "Epoch 703/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0435 - accuracy: 0.9984 - val_loss: 1.5140 - val_accuracy: 0.6912\n",
      "Epoch 704/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9977 - val_loss: 1.5072 - val_accuracy: 0.6937\n",
      "Epoch 705/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9976 - val_loss: 1.5011 - val_accuracy: 0.6962\n",
      "Epoch 706/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9978 - val_loss: 1.5004 - val_accuracy: 0.6987\n",
      "Epoch 707/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9980 - val_loss: 1.5146 - val_accuracy: 0.7012\n",
      "Epoch 708/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9981 - val_loss: 1.5233 - val_accuracy: 0.6937\n",
      "Epoch 709/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9984 - val_loss: 1.5005 - val_accuracy: 0.7000\n",
      "Epoch 710/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9982 - val_loss: 1.4831 - val_accuracy: 0.7000\n",
      "Epoch 711/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0453 - accuracy: 0.9981 - val_loss: 1.4567 - val_accuracy: 0.7025\n",
      "Epoch 712/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9981 - val_loss: 1.4505 - val_accuracy: 0.7012\n",
      "Epoch 713/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9977 - val_loss: 1.4604 - val_accuracy: 0.6950\n",
      "Epoch 714/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9986 - val_loss: 1.4627 - val_accuracy: 0.6962\n",
      "Epoch 715/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0447 - accuracy: 0.9976 - val_loss: 1.4651 - val_accuracy: 0.7025\n",
      "Epoch 716/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9980 - val_loss: 1.4693 - val_accuracy: 0.7050\n",
      "Epoch 717/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9974 - val_loss: 1.4596 - val_accuracy: 0.7025\n",
      "Epoch 718/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9984 - val_loss: 1.4649 - val_accuracy: 0.7012\n",
      "Epoch 719/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9986 - val_loss: 1.4687 - val_accuracy: 0.6975\n",
      "Epoch 720/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9984 - val_loss: 1.4591 - val_accuracy: 0.6962\n",
      "Epoch 721/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9978 - val_loss: 1.4822 - val_accuracy: 0.7050\n",
      "Epoch 722/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9976 - val_loss: 1.4821 - val_accuracy: 0.7000\n",
      "Epoch 723/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9980 - val_loss: 1.4812 - val_accuracy: 0.7012\n",
      "Epoch 724/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9975 - val_loss: 1.4707 - val_accuracy: 0.6975\n",
      "Epoch 725/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9978 - val_loss: 1.4641 - val_accuracy: 0.6912\n",
      "Epoch 726/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9978 - val_loss: 1.4364 - val_accuracy: 0.6900\n",
      "Epoch 727/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9986 - val_loss: 1.4458 - val_accuracy: 0.6950\n",
      "Epoch 728/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9976 - val_loss: 1.4509 - val_accuracy: 0.7075\n",
      "Epoch 729/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9983 - val_loss: 1.4532 - val_accuracy: 0.7050\n",
      "Epoch 730/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9979 - val_loss: 1.4891 - val_accuracy: 0.7000\n",
      "Epoch 731/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9978 - val_loss: 1.5012 - val_accuracy: 0.6900\n",
      "Epoch 732/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9974 - val_loss: 1.5019 - val_accuracy: 0.6837\n",
      "Epoch 733/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9981 - val_loss: 1.4846 - val_accuracy: 0.6875\n",
      "Epoch 734/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9981 - val_loss: 1.4851 - val_accuracy: 0.6850\n",
      "Epoch 735/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9980 - val_loss: 1.5296 - val_accuracy: 0.6862\n",
      "Epoch 736/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9986 - val_loss: 1.5404 - val_accuracy: 0.6937\n",
      "Epoch 737/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.5149 - val_accuracy: 0.6912\n",
      "Epoch 738/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0450 - accuracy: 0.9977 - val_loss: 1.5054 - val_accuracy: 0.6975\n",
      "Epoch 739/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9978 - val_loss: 1.5040 - val_accuracy: 0.6975\n",
      "Epoch 740/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9980 - val_loss: 1.5020 - val_accuracy: 0.6937\n",
      "Epoch 741/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9980 - val_loss: 1.6875 - val_accuracy: 0.6987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 742/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9982 - val_loss: 1.7025 - val_accuracy: 0.7000\n",
      "Epoch 743/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9986 - val_loss: 1.6557 - val_accuracy: 0.7037\n",
      "Epoch 744/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9978 - val_loss: 1.6398 - val_accuracy: 0.7125\n",
      "Epoch 745/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0427 - accuracy: 0.9982 - val_loss: 1.6079 - val_accuracy: 0.7125\n",
      "Epoch 746/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9982 - val_loss: 1.5701 - val_accuracy: 0.7188\n",
      "Epoch 747/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9980 - val_loss: 1.5442 - val_accuracy: 0.7200\n",
      "Epoch 748/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0432 - accuracy: 0.9984 - val_loss: 1.5483 - val_accuracy: 0.7138\n",
      "Epoch 749/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9983 - val_loss: 1.5255 - val_accuracy: 0.7138\n",
      "Epoch 750/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9982 - val_loss: 1.5040 - val_accuracy: 0.7025\n",
      "Epoch 751/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9980 - val_loss: 1.4821 - val_accuracy: 0.6987\n",
      "Epoch 752/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9981 - val_loss: 1.4641 - val_accuracy: 0.7075\n",
      "Epoch 753/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0422 - accuracy: 0.9986 - val_loss: 1.4618 - val_accuracy: 0.7000\n",
      "Epoch 754/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9985 - val_loss: 1.4653 - val_accuracy: 0.7050\n",
      "Epoch 755/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9972 - val_loss: 1.4518 - val_accuracy: 0.7175\n",
      "Epoch 756/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9986 - val_loss: 1.4550 - val_accuracy: 0.7088\n",
      "Epoch 757/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9982 - val_loss: 1.4402 - val_accuracy: 0.7025\n",
      "Epoch 758/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9984 - val_loss: 1.4606 - val_accuracy: 0.7012\n",
      "Epoch 759/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9988 - val_loss: 1.4177 - val_accuracy: 0.7025\n",
      "Epoch 760/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9978 - val_loss: 1.4042 - val_accuracy: 0.7050\n",
      "Epoch 761/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9986 - val_loss: 1.4081 - val_accuracy: 0.7012\n",
      "Epoch 762/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9980 - val_loss: 1.4062 - val_accuracy: 0.6962\n",
      "Epoch 763/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9976 - val_loss: 1.4065 - val_accuracy: 0.6962\n",
      "Epoch 764/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 1.4243 - val_accuracy: 0.6962\n",
      "Epoch 765/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9977 - val_loss: 1.4493 - val_accuracy: 0.7037\n",
      "Epoch 766/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9982 - val_loss: 1.4928 - val_accuracy: 0.7050\n",
      "Epoch 767/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0436 - accuracy: 0.9980 - val_loss: 1.4840 - val_accuracy: 0.7088\n",
      "Epoch 768/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0430 - accuracy: 0.9982 - val_loss: 1.4618 - val_accuracy: 0.7163\n",
      "Epoch 769/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9980 - val_loss: 1.4241 - val_accuracy: 0.7175\n",
      "Epoch 770/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9984 - val_loss: 1.4100 - val_accuracy: 0.7175\n",
      "Epoch 771/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9983 - val_loss: 1.4138 - val_accuracy: 0.7188\n",
      "Epoch 772/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9981 - val_loss: 1.3984 - val_accuracy: 0.7163\n",
      "Epoch 773/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9984 - val_loss: 1.3900 - val_accuracy: 0.7163\n",
      "Epoch 774/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9975 - val_loss: 1.3998 - val_accuracy: 0.7075\n",
      "Epoch 775/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9975 - val_loss: 1.4219 - val_accuracy: 0.7063\n",
      "Epoch 776/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9978 - val_loss: 1.4152 - val_accuracy: 0.7012\n",
      "Epoch 777/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9972 - val_loss: 1.4065 - val_accuracy: 0.7025\n",
      "Epoch 778/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9986 - val_loss: 1.4734 - val_accuracy: 0.7063\n",
      "Epoch 779/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0433 - accuracy: 0.9982 - val_loss: 1.4883 - val_accuracy: 0.7037\n",
      "Epoch 780/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9982 - val_loss: 1.5101 - val_accuracy: 0.7000\n",
      "Epoch 781/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0429 - accuracy: 0.9981 - val_loss: 1.5366 - val_accuracy: 0.7012\n",
      "Epoch 782/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9980 - val_loss: 1.5556 - val_accuracy: 0.7100\n",
      "Epoch 783/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9982 - val_loss: 1.5434 - val_accuracy: 0.7113\n",
      "Epoch 784/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9976 - val_loss: 1.5195 - val_accuracy: 0.7037\n",
      "Epoch 785/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9987 - val_loss: 1.5036 - val_accuracy: 0.7088\n",
      "Epoch 786/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9984 - val_loss: 1.5021 - val_accuracy: 0.7037\n",
      "Epoch 787/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.4730 - val_accuracy: 0.7037\n",
      "Epoch 788/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9978 - val_loss: 1.4069 - val_accuracy: 0.6950\n",
      "Epoch 789/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9984 - val_loss: 1.3645 - val_accuracy: 0.6912\n",
      "Epoch 790/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9985 - val_loss: 1.3884 - val_accuracy: 0.6925\n",
      "Epoch 791/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9985 - val_loss: 1.3600 - val_accuracy: 0.6875\n",
      "Epoch 792/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9983 - val_loss: 1.3859 - val_accuracy: 0.6937\n",
      "Epoch 793/1000\n",
      "19/19 [==============================] - 19s 986ms/step - loss: 0.0422 - accuracy: 0.9980 - val_loss: 1.3753 - val_accuracy: 0.7025\n",
      "Epoch 794/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9980 - val_loss: 1.3849 - val_accuracy: 0.6987\n",
      "Epoch 795/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0435 - accuracy: 0.9980 - val_loss: 1.4162 - val_accuracy: 0.6962\n",
      "Epoch 796/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9986 - val_loss: 1.4467 - val_accuracy: 0.6962\n",
      "Epoch 797/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9981 - val_loss: 1.4470 - val_accuracy: 0.6925\n",
      "Epoch 798/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9982 - val_loss: 1.4440 - val_accuracy: 0.6912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 799/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9981 - val_loss: 1.3886 - val_accuracy: 0.6962\n",
      "Epoch 800/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0452 - accuracy: 0.9973 - val_loss: 1.4020 - val_accuracy: 0.6912\n",
      "Epoch 801/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9983 - val_loss: 1.3740 - val_accuracy: 0.6925\n",
      "Epoch 802/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9983 - val_loss: 1.4206 - val_accuracy: 0.6987\n",
      "Epoch 803/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9974 - val_loss: 1.5415 - val_accuracy: 0.6975\n",
      "Epoch 804/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.6066 - val_accuracy: 0.7050\n",
      "Epoch 805/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0443 - accuracy: 0.9974 - val_loss: 1.5691 - val_accuracy: 0.7075\n",
      "Epoch 806/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9986 - val_loss: 1.5569 - val_accuracy: 0.7000\n",
      "Epoch 807/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9984 - val_loss: 1.5833 - val_accuracy: 0.7050\n",
      "Epoch 808/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0416 - accuracy: 0.9986 - val_loss: 1.5855 - val_accuracy: 0.7037\n",
      "Epoch 809/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9986 - val_loss: 1.5779 - val_accuracy: 0.6950\n",
      "Epoch 810/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9982 - val_loss: 1.5581 - val_accuracy: 0.7025\n",
      "Epoch 811/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9982 - val_loss: 1.5136 - val_accuracy: 0.7012\n",
      "Epoch 812/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9978 - val_loss: 1.5128 - val_accuracy: 0.6937\n",
      "Epoch 813/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9982 - val_loss: 1.5231 - val_accuracy: 0.7050\n",
      "Epoch 814/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0421 - accuracy: 0.9986 - val_loss: 1.5144 - val_accuracy: 0.7012\n",
      "Epoch 815/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9983 - val_loss: 1.5038 - val_accuracy: 0.6962\n",
      "Epoch 816/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.4905 - val_accuracy: 0.7025\n",
      "Epoch 817/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0429 - accuracy: 0.9979 - val_loss: 1.4901 - val_accuracy: 0.7025\n",
      "Epoch 818/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0411 - accuracy: 0.9987 - val_loss: 1.4860 - val_accuracy: 0.7050\n",
      "Epoch 819/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9985 - val_loss: 1.4707 - val_accuracy: 0.7063\n",
      "Epoch 820/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.4691 - val_accuracy: 0.7025\n",
      "Epoch 821/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9983 - val_loss: 1.4308 - val_accuracy: 0.6987\n",
      "Epoch 822/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9978 - val_loss: 1.3855 - val_accuracy: 0.6925\n",
      "Epoch 823/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0428 - accuracy: 0.9978 - val_loss: 1.3883 - val_accuracy: 0.6975\n",
      "Epoch 824/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9986 - val_loss: 1.3832 - val_accuracy: 0.6962\n",
      "Epoch 825/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9982 - val_loss: 1.3654 - val_accuracy: 0.6862\n",
      "Epoch 826/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9979 - val_loss: 1.3415 - val_accuracy: 0.6837\n",
      "Epoch 827/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9978 - val_loss: 1.3154 - val_accuracy: 0.6800\n",
      "Epoch 828/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9983 - val_loss: 1.3107 - val_accuracy: 0.6787\n",
      "Epoch 829/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9982 - val_loss: 1.3003 - val_accuracy: 0.6700\n",
      "Epoch 830/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9980 - val_loss: 1.3505 - val_accuracy: 0.6900\n",
      "Epoch 831/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9977 - val_loss: 1.3635 - val_accuracy: 0.6925\n",
      "Epoch 832/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0421 - accuracy: 0.9980 - val_loss: 2.1631 - val_accuracy: 0.6937\n",
      "Epoch 833/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9980 - val_loss: 2.4346 - val_accuracy: 0.6800\n",
      "Epoch 834/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9978 - val_loss: 2.2415 - val_accuracy: 0.6987\n",
      "Epoch 835/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 2.1051 - val_accuracy: 0.7063\n",
      "Epoch 836/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0424 - accuracy: 0.9983 - val_loss: 1.9871 - val_accuracy: 0.7138\n",
      "Epoch 837/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9986 - val_loss: 1.9915 - val_accuracy: 0.7063\n",
      "Epoch 838/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0435 - accuracy: 0.9975 - val_loss: 1.8895 - val_accuracy: 0.7100\n",
      "Epoch 839/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9978 - val_loss: 1.7735 - val_accuracy: 0.7138\n",
      "Epoch 840/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0449 - accuracy: 0.9975 - val_loss: 1.6503 - val_accuracy: 0.7150\n",
      "Epoch 841/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9974 - val_loss: 1.5827 - val_accuracy: 0.7150\n",
      "Epoch 842/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0442 - accuracy: 0.9976 - val_loss: 1.5691 - val_accuracy: 0.7150\n",
      "Epoch 843/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9975 - val_loss: 1.5434 - val_accuracy: 0.7138\n",
      "Epoch 844/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0434 - accuracy: 0.9977 - val_loss: 1.5348 - val_accuracy: 0.7163\n",
      "Epoch 845/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9986 - val_loss: 1.5506 - val_accuracy: 0.7150\n",
      "Epoch 846/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9983 - val_loss: 1.5883 - val_accuracy: 0.7125\n",
      "Epoch 847/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9980 - val_loss: 1.5693 - val_accuracy: 0.7100\n",
      "Epoch 848/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0440 - accuracy: 0.9974 - val_loss: 1.5650 - val_accuracy: 0.7175\n",
      "Epoch 849/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0456 - accuracy: 0.9981 - val_loss: 1.5402 - val_accuracy: 0.7175\n",
      "Epoch 850/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9987 - val_loss: 1.4831 - val_accuracy: 0.7113\n",
      "Epoch 851/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9977 - val_loss: 1.4755 - val_accuracy: 0.7075\n",
      "Epoch 852/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9982 - val_loss: 1.5017 - val_accuracy: 0.7063\n",
      "Epoch 853/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0435 - accuracy: 0.9977 - val_loss: 1.5335 - val_accuracy: 0.7037\n",
      "Epoch 854/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9981 - val_loss: 1.5330 - val_accuracy: 0.7075\n",
      "Epoch 855/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0439 - accuracy: 0.9977 - val_loss: 1.5072 - val_accuracy: 0.7138\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 856/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0428 - accuracy: 0.9979 - val_loss: 1.4718 - val_accuracy: 0.7088\n",
      "Epoch 857/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 1.4824 - val_accuracy: 0.7088\n",
      "Epoch 858/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9984 - val_loss: 1.4695 - val_accuracy: 0.7113\n",
      "Epoch 859/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0435 - accuracy: 0.9984 - val_loss: 1.4594 - val_accuracy: 0.7050\n",
      "Epoch 860/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0442 - accuracy: 0.9974 - val_loss: 1.4530 - val_accuracy: 0.6987\n",
      "Epoch 861/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9984 - val_loss: 1.4498 - val_accuracy: 0.6950\n",
      "Epoch 862/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9980 - val_loss: 1.4451 - val_accuracy: 0.6937\n",
      "Epoch 863/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9982 - val_loss: 1.4461 - val_accuracy: 0.6987\n",
      "Epoch 864/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 1.4436 - val_accuracy: 0.7075\n",
      "Epoch 865/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0441 - accuracy: 0.9976 - val_loss: 1.4192 - val_accuracy: 0.7037\n",
      "Epoch 866/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9988 - val_loss: 1.4072 - val_accuracy: 0.7125\n",
      "Epoch 867/1000\n",
      "19/19 [==============================] - 19s 999ms/step - loss: 0.0428 - accuracy: 0.9984 - val_loss: 1.4008 - val_accuracy: 0.7150\n",
      "Epoch 868/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0426 - accuracy: 0.9982 - val_loss: 1.3914 - val_accuracy: 0.7125\n",
      "Epoch 869/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9983 - val_loss: 1.4191 - val_accuracy: 0.7088\n",
      "Epoch 870/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9986 - val_loss: 1.4299 - val_accuracy: 0.7037\n",
      "Epoch 871/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0442 - accuracy: 0.9975 - val_loss: 1.4272 - val_accuracy: 0.7050\n",
      "Epoch 872/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9982 - val_loss: 1.4223 - val_accuracy: 0.7025\n",
      "Epoch 873/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9984 - val_loss: 1.4014 - val_accuracy: 0.7113\n",
      "Epoch 874/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0433 - accuracy: 0.9979 - val_loss: 1.4053 - val_accuracy: 0.7088\n",
      "Epoch 875/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9980 - val_loss: 1.4043 - val_accuracy: 0.7125\n",
      "Epoch 876/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9982 - val_loss: 1.2850 - val_accuracy: 0.6337\n",
      "Epoch 877/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9982 - val_loss: 1.6380 - val_accuracy: 0.5087\n",
      "Epoch 878/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0411 - accuracy: 0.9988 - val_loss: 1.5196 - val_accuracy: 0.5600\n",
      "Epoch 879/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9984 - val_loss: 1.3501 - val_accuracy: 0.6150\n",
      "Epoch 880/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0420 - accuracy: 0.9978 - val_loss: 1.2668 - val_accuracy: 0.7025\n",
      "Epoch 881/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9976 - val_loss: 1.7011 - val_accuracy: 0.7138\n",
      "Epoch 882/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0430 - accuracy: 0.9983 - val_loss: 1.6977 - val_accuracy: 0.7100\n",
      "Epoch 883/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9984 - val_loss: 1.6556 - val_accuracy: 0.7063\n",
      "Epoch 884/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0432 - accuracy: 0.9978 - val_loss: 1.5921 - val_accuracy: 0.7163\n",
      "Epoch 885/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0444 - accuracy: 0.9976 - val_loss: 1.5908 - val_accuracy: 0.7125\n",
      "Epoch 886/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0412 - accuracy: 0.9987 - val_loss: 1.5403 - val_accuracy: 0.7188\n",
      "Epoch 887/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0426 - accuracy: 0.9979 - val_loss: 1.4401 - val_accuracy: 0.7175\n",
      "Epoch 888/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9984 - val_loss: 1.4858 - val_accuracy: 0.7212\n",
      "Epoch 889/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0422 - accuracy: 0.9987 - val_loss: 1.5148 - val_accuracy: 0.7163\n",
      "Epoch 890/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9984 - val_loss: 1.5475 - val_accuracy: 0.7113\n",
      "Epoch 891/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9982 - val_loss: 1.5508 - val_accuracy: 0.7113\n",
      "Epoch 892/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9983 - val_loss: 1.5602 - val_accuracy: 0.7188\n",
      "Epoch 893/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9978 - val_loss: 1.6285 - val_accuracy: 0.7188\n",
      "Epoch 894/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9984 - val_loss: 1.6090 - val_accuracy: 0.7125\n",
      "Epoch 895/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9985 - val_loss: 1.6242 - val_accuracy: 0.7138\n",
      "Epoch 896/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9984 - val_loss: 1.6296 - val_accuracy: 0.7150\n",
      "Epoch 897/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9975 - val_loss: 1.5718 - val_accuracy: 0.7212\n",
      "Epoch 898/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0427 - accuracy: 0.9980 - val_loss: 1.5245 - val_accuracy: 0.7100\n",
      "Epoch 899/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0412 - accuracy: 0.9984 - val_loss: 1.5684 - val_accuracy: 0.7100\n",
      "Epoch 900/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9982 - val_loss: 1.6457 - val_accuracy: 0.7012\n",
      "Epoch 901/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0420 - accuracy: 0.9982 - val_loss: 1.5526 - val_accuracy: 0.6950\n",
      "Epoch 902/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9982 - val_loss: 1.5079 - val_accuracy: 0.7000\n",
      "Epoch 903/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0419 - accuracy: 0.9983 - val_loss: 1.4681 - val_accuracy: 0.7075\n",
      "Epoch 904/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0403 - accuracy: 0.9987 - val_loss: 1.4655 - val_accuracy: 0.7025\n",
      "Epoch 905/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9982 - val_loss: 1.4934 - val_accuracy: 0.7000\n",
      "Epoch 906/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0427 - accuracy: 0.9976 - val_loss: 1.5249 - val_accuracy: 0.6962\n",
      "Epoch 907/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9979 - val_loss: 1.5188 - val_accuracy: 0.6875\n",
      "Epoch 908/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9982 - val_loss: 1.5298 - val_accuracy: 0.6887\n",
      "Epoch 909/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9982 - val_loss: 1.6191 - val_accuracy: 0.6962\n",
      "Epoch 910/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9980 - val_loss: 1.6266 - val_accuracy: 0.7050\n",
      "Epoch 911/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9981 - val_loss: 1.5883 - val_accuracy: 0.7050\n",
      "Epoch 912/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0408 - accuracy: 0.9983 - val_loss: 1.6044 - val_accuracy: 0.7150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 913/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0420 - accuracy: 0.9984 - val_loss: 1.6293 - val_accuracy: 0.7063\n",
      "Epoch 914/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0412 - accuracy: 0.9986 - val_loss: 1.6539 - val_accuracy: 0.7025\n",
      "Epoch 915/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9984 - val_loss: 1.4529 - val_accuracy: 0.7012\n",
      "Epoch 916/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0424 - accuracy: 0.9982 - val_loss: 1.3166 - val_accuracy: 0.6600\n",
      "Epoch 917/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0408 - accuracy: 0.9984 - val_loss: 1.3250 - val_accuracy: 0.6675\n",
      "Epoch 918/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0413 - accuracy: 0.9982 - val_loss: 1.3345 - val_accuracy: 0.6825\n",
      "Epoch 919/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9983 - val_loss: 1.3538 - val_accuracy: 0.6812\n",
      "Epoch 920/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0404 - accuracy: 0.9985 - val_loss: 1.3375 - val_accuracy: 0.6787\n",
      "Epoch 921/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9978 - val_loss: 1.3492 - val_accuracy: 0.6825\n",
      "Epoch 922/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0410 - accuracy: 0.9985 - val_loss: 1.3538 - val_accuracy: 0.6850\n",
      "Epoch 923/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9979 - val_loss: 1.3168 - val_accuracy: 0.6737\n",
      "Epoch 924/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0438 - accuracy: 0.9974 - val_loss: 1.3332 - val_accuracy: 0.6750\n",
      "Epoch 925/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0413 - accuracy: 0.9984 - val_loss: 1.3273 - val_accuracy: 0.6762\n",
      "Epoch 926/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0425 - accuracy: 0.9983 - val_loss: 1.3485 - val_accuracy: 0.6887\n",
      "Epoch 927/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0427 - accuracy: 0.9984 - val_loss: 1.3982 - val_accuracy: 0.6962\n",
      "Epoch 928/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9982 - val_loss: 1.6375 - val_accuracy: 0.6937\n",
      "Epoch 929/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0423 - accuracy: 0.9980 - val_loss: 1.4757 - val_accuracy: 0.6925\n",
      "Epoch 930/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0413 - accuracy: 0.9986 - val_loss: 1.4546 - val_accuracy: 0.6900\n",
      "Epoch 931/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9984 - val_loss: 1.6014 - val_accuracy: 0.7037\n",
      "Epoch 932/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9979 - val_loss: 1.7157 - val_accuracy: 0.7012\n",
      "Epoch 933/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9982 - val_loss: 1.8972 - val_accuracy: 0.7063\n",
      "Epoch 934/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0416 - accuracy: 0.9985 - val_loss: 1.8180 - val_accuracy: 0.7063\n",
      "Epoch 935/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9982 - val_loss: 1.7160 - val_accuracy: 0.7012\n",
      "Epoch 936/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0418 - accuracy: 0.9988 - val_loss: 1.6594 - val_accuracy: 0.7025\n",
      "Epoch 937/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0413 - accuracy: 0.9988 - val_loss: 1.6727 - val_accuracy: 0.7075\n",
      "Epoch 938/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9982 - val_loss: 1.7623 - val_accuracy: 0.7037\n",
      "Epoch 939/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9984 - val_loss: 1.7537 - val_accuracy: 0.7075\n",
      "Epoch 940/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0410 - accuracy: 0.9986 - val_loss: 1.6825 - val_accuracy: 0.7063\n",
      "Epoch 941/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0408 - accuracy: 0.9988 - val_loss: 1.6390 - val_accuracy: 0.7063\n",
      "Epoch 942/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0412 - accuracy: 0.9982 - val_loss: 1.5947 - val_accuracy: 0.7050\n",
      "Epoch 943/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0410 - accuracy: 0.9985 - val_loss: 1.5463 - val_accuracy: 0.7063\n",
      "Epoch 944/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9980 - val_loss: 1.5059 - val_accuracy: 0.7125\n",
      "Epoch 945/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0409 - accuracy: 0.9985 - val_loss: 1.4971 - val_accuracy: 0.7075\n",
      "Epoch 946/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0406 - accuracy: 0.9986 - val_loss: 1.4955 - val_accuracy: 0.7063\n",
      "Epoch 947/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0409 - accuracy: 0.9986 - val_loss: 1.4819 - val_accuracy: 0.7088\n",
      "Epoch 948/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0410 - accuracy: 0.9985 - val_loss: 1.4674 - val_accuracy: 0.7100\n",
      "Epoch 949/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0406 - accuracy: 0.9987 - val_loss: 1.4567 - val_accuracy: 0.7075\n",
      "Epoch 950/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0422 - accuracy: 0.9979 - val_loss: 1.5826 - val_accuracy: 0.7075\n",
      "Epoch 951/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0412 - accuracy: 0.9986 - val_loss: 1.9053 - val_accuracy: 0.6950\n",
      "Epoch 952/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9986 - val_loss: 1.8166 - val_accuracy: 0.7000\n",
      "Epoch 953/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9980 - val_loss: 1.7378 - val_accuracy: 0.7025\n",
      "Epoch 954/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0408 - accuracy: 0.9983 - val_loss: 1.7126 - val_accuracy: 0.6987\n",
      "Epoch 955/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0411 - accuracy: 0.9984 - val_loss: 1.6482 - val_accuracy: 0.7063\n",
      "Epoch 956/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9977 - val_loss: 1.6107 - val_accuracy: 0.7063\n",
      "Epoch 957/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0428 - accuracy: 0.9976 - val_loss: 1.5646 - val_accuracy: 0.7125\n",
      "Epoch 958/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0432 - accuracy: 0.9969 - val_loss: 1.5262 - val_accuracy: 0.7088\n",
      "Epoch 959/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0416 - accuracy: 0.9984 - val_loss: 1.5234 - val_accuracy: 0.7088\n",
      "Epoch 960/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9981 - val_loss: 1.5235 - val_accuracy: 0.7088\n",
      "Epoch 961/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9978 - val_loss: 1.4541 - val_accuracy: 0.7113\n",
      "Epoch 962/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0437 - accuracy: 0.9973 - val_loss: 1.4335 - val_accuracy: 0.7188\n",
      "Epoch 963/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0429 - accuracy: 0.9975 - val_loss: 1.4004 - val_accuracy: 0.7138\n",
      "Epoch 964/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0413 - accuracy: 0.9983 - val_loss: 1.4124 - val_accuracy: 0.7175\n",
      "Epoch 965/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9976 - val_loss: 1.4112 - val_accuracy: 0.7125\n",
      "Epoch 966/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9985 - val_loss: 1.4314 - val_accuracy: 0.7138\n",
      "Epoch 967/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9985 - val_loss: 1.4491 - val_accuracy: 0.7150\n",
      "Epoch 968/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0410 - accuracy: 0.9986 - val_loss: 1.4370 - val_accuracy: 0.7200\n",
      "Epoch 969/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9979 - val_loss: 1.4090 - val_accuracy: 0.7250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 970/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0405 - accuracy: 0.9988 - val_loss: 1.4138 - val_accuracy: 0.7212\n",
      "Epoch 971/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9983 - val_loss: 1.4346 - val_accuracy: 0.7188\n",
      "Epoch 972/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0416 - accuracy: 0.9983 - val_loss: 1.4430 - val_accuracy: 0.7150\n",
      "Epoch 973/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9982 - val_loss: 1.4509 - val_accuracy: 0.7100\n",
      "Epoch 974/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0421 - accuracy: 0.9979 - val_loss: 1.4410 - val_accuracy: 0.7175\n",
      "Epoch 975/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0408 - accuracy: 0.9986 - val_loss: 1.4569 - val_accuracy: 0.7138\n",
      "Epoch 976/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0418 - accuracy: 0.9983 - val_loss: 1.4309 - val_accuracy: 0.7138\n",
      "Epoch 977/1000\n",
      "19/19 [==============================] - 21s 1s/step - loss: 0.0408 - accuracy: 0.9984 - val_loss: 1.4292 - val_accuracy: 0.7150\n",
      "Epoch 978/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0405 - accuracy: 0.9986 - val_loss: 1.3186 - val_accuracy: 0.6975\n",
      "Epoch 979/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9983 - val_loss: 1.3331 - val_accuracy: 0.6850\n",
      "Epoch 980/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9982 - val_loss: 1.3399 - val_accuracy: 0.6887\n",
      "Epoch 981/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9983 - val_loss: 1.3389 - val_accuracy: 0.7000\n",
      "Epoch 982/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9976 - val_loss: 1.3374 - val_accuracy: 0.7037\n",
      "Epoch 983/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9981 - val_loss: 1.3315 - val_accuracy: 0.7050\n",
      "Epoch 984/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9985 - val_loss: 1.3389 - val_accuracy: 0.7063\n",
      "Epoch 985/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0423 - accuracy: 0.9976 - val_loss: 1.3363 - val_accuracy: 0.7125\n",
      "Epoch 986/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9980 - val_loss: 1.3573 - val_accuracy: 0.7113\n",
      "Epoch 987/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0417 - accuracy: 0.9980 - val_loss: 1.3825 - val_accuracy: 0.7113\n",
      "Epoch 988/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0414 - accuracy: 0.9987 - val_loss: 1.4127 - val_accuracy: 0.7237\n",
      "Epoch 989/1000\n",
      "19/19 [==============================] - 19s 1s/step - loss: 0.0419 - accuracy: 0.9982 - val_loss: 1.4145 - val_accuracy: 0.7262\n",
      "Epoch 990/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0431 - accuracy: 0.9978 - val_loss: 1.4205 - val_accuracy: 0.7262\n",
      "Epoch 991/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9988 - val_loss: 1.4264 - val_accuracy: 0.7275\n",
      "Epoch 992/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0419 - accuracy: 0.9982 - val_loss: 1.3206 - val_accuracy: 0.7025\n",
      "Epoch 993/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0415 - accuracy: 0.9982 - val_loss: 1.3492 - val_accuracy: 0.7063\n",
      "Epoch 994/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0414 - accuracy: 0.9981 - val_loss: 1.3675 - val_accuracy: 0.7100\n",
      "Epoch 995/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0412 - accuracy: 0.9987 - val_loss: 1.3775 - val_accuracy: 0.7088\n",
      "Epoch 996/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0410 - accuracy: 0.9979 - val_loss: 1.3995 - val_accuracy: 0.7037\n",
      "Epoch 997/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0416 - accuracy: 0.9983 - val_loss: 1.4091 - val_accuracy: 0.7075\n",
      "Epoch 998/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0422 - accuracy: 0.9981 - val_loss: 1.4319 - val_accuracy: 0.7000\n",
      "Epoch 999/1000\n",
      "19/19 [==============================] - 20s 1s/step - loss: 0.0400 - accuracy: 0.9986 - val_loss: 1.4498 - val_accuracy: 0.7050\n",
      "Epoch 1000/1000\n",
      "19/19 [==============================] - 19s 984ms/step - loss: 0.0411 - accuracy: 0.9984 - val_loss: 1.4615 - val_accuracy: 0.7000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0xb01cb43d0>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history = model1.fit([fine_tune_data_train_4P_red, fine_tune_data_train], fine_tune_labels_train, epochs=EPOCHS, batch_size=800, validation_data=([fine_tune_data_test_4P_red, fine_tune_data_test], fine_tune_labels_test), validation_batch_size=800, callbacks=[model_checkpoint_callback])\n",
    "# history = model1.fit(fine_tune_data, fine_tune_labels, epochs=EPOCHS, batch_size=1600, callbacks=[model_checkpoint_callback])\n",
    "model1.save_weights(\"./Models/EfficientNet-kP-PCA-LSTM/effnet-kp-model-weights\")\n",
    "model1.load_weights('./Models/EfficientNet-kP-PCA-LSTM/effnet-kp-model-weights')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9116a6a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_train_loss(history, model_name):  \n",
    "    plt.figure(figsize=(8,6))\n",
    "    plt.plot(history.history['loss'])\n",
    "    plt.plot(history.history['accuracy'])\n",
    "    plt.plot(history.history['val_loss'])\n",
    "    plt.plot(history.history['val_accuracy'])\n",
    "    plt.title(model_name+' loss')\n",
    "    plt.ylabel('loss')\n",
    "    plt.xlabel('epochs')\n",
    "    plt.legend(['train loss', 'train accuracy', 'val loss', 'val accuracy'], loc='upper left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c55b5870",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAIhCAYAAACcznj/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAADFs0lEQVR4nOzdd3hT5d8G8PskTfcudECBQtmjzIIM2UNGBZElyBD9qYCvAwHFgYKoiChLwcVQBBVElL333ntDC2V2QfdKc94/Duc0s01K27T0/lwXF01ykjxJ0+TO91mCKIoiiIiIiIjsTGXvBhARERERAQymRERERFRCMJgSERERUYnAYEpEREREJQKDKRERERGVCAymRERERFQiMJgSERERUYnAYEpEREREJQKDKRERERGVCAymVGYsXrwYgiCY/Tdu3DhERUVBEAQsXry42No0YsQIi23S/zdixIhia5M1rH2udu7cmefj0r9+VlYWXn/9dQQFBUGtVqNRo0YAgISEBAwaNAj+/v4QBAF9+vQBAAiCgE8//bRI2v24li1bhlmzZpm9TH7s06ZNM7lMfo0ePXrU5vs8f/48Pv30U0RFRVl9nREjRsDd3T3f49q3b2/we3NxcUHDhg0xa9Ys6HQ65biC/E6Ki/7f/86dO00uF0UR1atXhyAIaN++faHed0GfF2tfr/Z47yIqKg72bgBRcVu0aBFq165tcF6FChUQEBCAAwcOIDQ0tNja8vHHH+P1119XTh8/fhxjxozBF198gQ4dOijnly9fvtjaVBSMH49M/7meP38+fvzxR8ydOxdNmzZVAtNnn32GVatWYeHChQgNDYWvry8A4MCBAwgODrapHUFBQcXyO162bBnOnj2Lt99+2+Ix06ZNw6uvvqo8nsd1/vx5TJ48Ge3bt0dISEih3Ka+atWqYenSpQCAmJgY/PDDD3jnnXdw9+5dfPXVVwAK9jspbh4eHliwYIFJ+Ny1axeuXbsGDw8P+zSMiAAwmFIZVL9+fTRr1szsZU899VSxtiU0NNQgJGVkZAAAatSoUextKUrWPJ6zZ8/CxcUFb7zxhsn5oaGhGDJkiMH5BXl+nJycSsTz2rlzZ+zcuROff/45vvnmG3s3xyouLi4Gz1337t1Ru3ZtfPfdd5g6dSo0Gk2JeG7zM3DgQCxduhTff/89PD09lfMXLFiAli1bIikpyY6tIyJ25RM9Yq477NNPP4UgCDh37hxeeOEFeHl5ISAgACNHjkRiYqLB9UVRxLx589CoUSO4uLjAx8cH/fr1w/Xr1x+rXXIbjMldk/pdtyEhIejVqxc2btyIJk2awMXFBbVr18bChQtNrn/v3j289tprCA4OhqOjI6pWrYrJkydDq9UaHHfnzh0MGDAAHh4e8PLywsCBA3Hv3r3HekzGBEHAL7/8gvT0dINufkEQsHXrVly4cMGkG9Zc9+jt27fx6quvolKlSnB0dESFChXQr18/3L9/H4DlLs8rV65g8ODB8Pf3h5OTE+rUqYPvv//e4Bh5WMIff/yBDz/8EBUqVICnpyc6d+6MS5cuKce1b98e69atw40bNwy6v/XVqlULL7/8Mr7//nvcuHEj3+fn6NGjePbZZ+Hr6wtnZ2c0btwYy5cvVy5fvHgx+vfvDwDo0KGD2aES1tq3bx/KlSuHXr16ITU11eJxGo0GTZs2RVpaGmJjYwGY/51Y+zrLzMzElClTUKdOHTg7O8PPzw8dOnTA/v37lWMK42/shRdeAAD88ccfynmJiYlYuXIlRo4cafY6CQkJGD16NCpWrAhHR0dUq1YNH374ITIzMw2OS0pKwv/+9z/4+fnB3d0dzzzzDC5fvmz2Nq15zT2uvXv3olOnTvDw8ICrqytatWqFdevWGRyTlpaGcePGoWrVqnB2doavry+aNWtm8Pxcv34dgwYNQoUKFeDk5ISAgAB06tQJJ0+eLNT2EgGsmFIZlJOTY/Kh6OCQ95/C888/j4EDB+Lll1/GmTNnMHHiRAAwCHyvvfYaFi9ejDfffBNfffUVEhISMGXKFLRq1QqnTp1CQEBA4T8YM06dOoV3330X77//PgICAvDLL7/g5ZdfRvXq1dG2bVsAUlho3rw5VCoVJk2ahNDQUBw4cABTp05FVFQUFi1aBABIT09H586dcefOHXz55ZeoWbMm1q1bh4EDB9rUJp1OZ/KcA7nP+4EDB/DZZ59hx44d2L59OwCgatWqOHDgAEaPHo3ExESlG7lu3bpm7+P27dsIDw9HdnY2PvjgA4SFhSE+Ph6bNm3CgwcPLD7/58+fR6tWrVC5cmV88803CAwMxKZNm/Dmm28iLi4On3zyicHxH3zwAVq3bo1ffvkFSUlJeO+99xAREYELFy5ArVZj3rx5ePXVV3Ht2jWsWrXK4nPy6aefYsmSJfj444/x22+/WTxux44deOaZZ9CiRQv88MMP8PLywp9//omBAwciLS0NI0aMQM+ePfHFF1/ggw8+wPfff48mTZoAgM1DFpYvX45hw4Zh5MiRmDt3LtRqdZ7HX7t2DQ4ODvDx8TF7ubWvM61Wi+7du2PPnj14++230bFjR2i1Whw8eBA3b95Eq1atABTO35inpyf69euHhQsX4rXXXgMghVSVSoWBAweajA3OyMhAhw4dcO3aNUyePBlhYWHYs2cPvvzyS5w8eVIJeqIook+fPti/fz8mTZqE8PBw7Nu3D927dzdpg62vuYLYtWsXunTpgrCwMCxYsABOTk6YN28eIiIi8Mcffyh/w2PHjsWSJUswdepUNG7cGKmpqTh79izi4+OV2+rRowdycnIwffp0VK5cGXFxcdi/fz8ePnz42O0kMiESlRGLFi0SAZj9l52dLUZGRooAxEWLFinX+eSTT0QA4vTp0w1ua/To0aKzs7Oo0+lEURTFAwcOiADEb775xuC46Oho0cXFRZwwYYJVbdyxY4cIQFyxYoVJGyw9nsjISOW8KlWqiM7OzuKNGzeU89LT00VfX1/xtddeU8577bXXRHd3d4PjRFEUZ8yYIQIQz507J4qiKM6fP18EIP73338Gx/3vf/8zea7yejyW/kVHRyvHDh8+XHRzczO5jXbt2on16tUzOR+A+MknnyinR44cKWo0GvH8+fMW22Pud9ytWzcxODhYTExMNDj2jTfeEJ2dncWEhASDx9KjRw+D45YvXy4CEA8cOKCc17NnT7FKlSpm2wBAHDNmjCiKovjhhx+KKpVKPHXqlCiKub/TI0eOKMfXrl1bbNy4sZidnW1wO7169RKDgoLEnJwcURRFccWKFSIAcceOHRYfvzH953zatGmiWq0Wv/rqK5Pj5N9Bdna2mJ2dLd65c0d8//33RQBi//79DR6b/u/E2tfZb7/9JgIQf/75Z4ttfdy/Mf3nVv5dnj17VhRFUQwPDxdHjBghiqIo1qtXT2zXrp1yvR9++EEEIC5fvtzg9r766isRgLh582ZRFEVxw4YNIgBx9uzZBsd9/vnnJs+Lta85c69Xc8wd99RTT4n+/v5icnKycp5WqxXr168vBgcHK+9d9evXF/v06WPxtuPi4kQA4qxZs/JsA1FhYVc+lTm//fYbjhw5YvAvv4rps88+a3A6LCwMGRkZiImJAQCsXbsWgiDgxRdfhFarVf4FBgaiYcOGSvezKIoGl5urIj6uRo0aoXLlysppZ2dn1KxZ06DLeO3atejQoQMqVKhg0Ba5urNr1y4AUrXOw8PD5PEPHjzYpjZ99dVXJs/5kSNHCrWKvGHDBnTo0AF16tSx+joZGRnYtm0bnnvuObi6uho8Fz169EBGRgYOHjxocB1zrwUAVnXJG5swYQJ8fX3x3nvvmb386tWruHjxojK+1rh9d+/eNRhGYE5+rzlRFPHaa6/hk08+wbJlyzBhwgSzt3Pu3DloNBpoNBpUqFAB33zzDYYMGYKff/7Z4n1b+zrbsGEDnJ2dLXaly7dlzd+YNdq1a4fQ0FAsXLgQZ86cwZEjRyze9/bt2+Hm5oZ+/foZnC+vlLFt2zYA0t8KAJOx0MZ/KwV5zdkqNTUVhw4dQr9+/QxWXVCr1Rg6dChu3bqlvG6aN2+ODRs24P3338fOnTuRnp5ucFu+vr4IDQ3F119/jW+//RYnTpwwWImBqLCxK5/KnDp16lic/GSJn5+fwWknJycAUN7E79+/D1EULQatatWqAQB+/fVXvPTSSwaXiaJoU1tsbSsgtVf/A+f+/ftYs2YNNBqN2duIi4sDAMTHx5t9TIGBgTa1qVq1ajY/57aKjY21eUZ4fHw8tFot5s6di7lz55o9Rn4uZPm9Fmzh6emJjz76CG+//bYSbPTJY2PHjRuHcePGWdU+Y/m95rKysvDXX3+hXr16ZrudZaGhofjzzz8hCAKcnZ1RtWpVuLq65nnf1r7OYmNjUaFCBahUlmsl1v6NWUMQBLz00kuYM2cOMjIyULNmTTz99NNmj42Pj0dgYKDJOGF/f384ODgoXd7x8fFwcHAweX0Y/60U5DVnqwcPHkAURQQFBZlcVqFCBaUdADBnzhwEBwfjr7/+wldffQVnZ2d069YNX3/9NWrUqAFBELBt2zZMmTIF06dPx7vvvgtfX18MGTIEn3/+OVcxoELHYEpUCMqVKwdBELBnzx4lqOiTz4uIiMCRI0dsum1nZ2cA0uQQ/dt+nA+vcuXKISwsDJ9//rnZy+UPLz8/Pxw+fNjk8sKe/FQYypcvj1u3btl0HR8fH6WKNGbMGLPHVK1atTCaZ9GoUaMwe/ZsvPfeexg1apTBZeXKlQMATJw4EX379jV7/Vq1auV5+/m95pycnLBjxw5069YNnTt3xsaNG82OGXV2drb5y4W1r7Py5ctj79690Ol0FsOptX9j1hoxYgQmTZqEH374wWL7AOlv4NChQxBF0SCcxsTEQKvVKr8jPz8/aLVaxMfHG4RT47+V4njN+fj4QKVS4e7duyaX3blzB0Dua8vNzQ2TJ0/G5MmTcf/+faV6GhERgYsXLwIAqlSpggULFgAALl++jOXLl+PTTz9FVlYWfvjhh8dqK5ExBlOiQtCrVy9MmzYNt2/fxoABAywe5+fnZ7aimRd5TcrTp08jPDxcOX/NmjUFaisgtXf9+vUIDQ21OHEFkGZ4L1++HKtXrzbowl62bFmB77uodO/eHUuWLMGlS5fyDWsyV1dXdOjQASdOnEBYWBgcHR0LpS3GFeq8ODo6YurUqRgyZIgSFmS1atVCjRo1cOrUKXzxxRf53idgWrm15jXXuHFj7Nq1C507d0b79u2xZcsW+Pv7W9X+vFj7OuvevTv++OMPLF682GKXurV/Y9aqWLEixo8fj4sXL2L48OEWj+vUqROWL1+Of//9F88995xyvjxhrVOnTgCkv5Xp06dj6dKlePPNN5XjjP9Wiuo1p8/NzQ0tWrTAP//8gxkzZsDFxQWANAnx999/R3BwMGrWrGlyvYCAAIwYMQKnTp3CrFmzkJaWZlIVr1mzJj766COsXLkSx48fL/S2EzGYEhWC1q1b49VXX8VLL72Eo0ePom3btnBzc8Pdu3exd+9eNGjQwKQaZq0ePXrA19cXL7/8MqZMmQIHBwcsXrwY0dHRBW7vlClTsGXLFrRq1QpvvvkmatWqhYyMDERFRWH9+vX44YcfEBwcjGHDhmHmzJkYNmwYPv/8c9SoUQPr16/Hpk2bbLq/K1eumB03FxwcXGgLsk+ZMgUbNmxA27Zt8cEHH6BBgwZ4+PAhNm7ciLFjx5psqiCbPXs22rRpg6effhqjRo1CSEgIkpOTcfXqVaxZs0ZZJcAWDRo0wD///IP58+ejadOmUKlUeVYbX3jhBcyYMQMbNmwwuezHH39E9+7d0a1bN4wYMQIVK1ZEQkICLly4gOPHj2PFihUApPV5AeCnn36Ch4eH0t1u7RehOnXqYM+ePejcuTPatm2LrVu3PvbvxtrX2QsvvIBFixbh9ddfx6VLl9ChQwfodDocOnQIderUwaBBg4rkb8zc7lvGhg0bhu+//x7Dhw9HVFQUGjRogL179+KLL75Ajx490LlzZwBA165d0bZtW0yYMAGpqalo1qwZ9u3bhyVLlpjcZlG85ox9+eWX6NKlCzp06IBx48bB0dER8+bNw9mzZ/HHH38o1d8WLVqgV69eCAsLg4+PDy5cuIAlS5agZcuWcHV1xenTp/HGG2+gf//+qFGjBhwdHbF9+3acPn0a77///mO3k8iE3aZdERUzczOe9eU1Kz82NtbsbenPiBdFUVy4cKHYokUL0c3NTXRxcRFDQ0PFYcOGiUePHrWqjeZm5YuiKB4+fFhs1aqV6ObmJlasWFH85JNPxF9++cXsrPyePXua3G67du0MZhqLoijGxsaKb775pli1alVRo9GIvr6+YtOmTcUPP/xQTElJUY67deuW+Pzzz4vu7u6ih4eH+Pzzz4v79+8vlFn5H374oXLs487KF0VphvbIkSPFwMBAUaPRiBUqVBAHDBgg3r9/XxRFy7OcIyMjxZEjR4oVK1YUNRqNWL58ebFVq1bi1KlTTR6L8e/G3G0mJCSI/fr1E729vUVBEAxWVYDerHx9mzdvVp4X49foqVOnxAEDBoj+/v6iRqMRAwMDxY4dO4o//PCDwXGzZs0Sq1atKqrVaqt+P+ae81u3bom1a9cWQ0JCxGvXromiaPl3YMzc78Ta11l6ero4adIksUaNGqKjo6Po5+cnduzYUdy/f7/B7RX0byy/v3+Z8ax8URTF+Ph48fXXXxeDgoJEBwcHsUqVKuLEiRPFjIwMg+MePnwojhw5UvT29hZdXV3FLl26iBcvXjT7vFjzmnucWfmiKIp79uwRO3bsqDxXTz31lLhmzRqDY95//32xWbNmoo+Pj+jk5CRWq1ZNfOedd8S4uDhRFEXx/v374ogRI8TatWuLbm5uoru7uxgWFibOnDlT1Gq1ebaLqCAEUSzkmRdERERERAXA5aKIiIiIqERgMCUiIiKiEoHBlIiIiIhKBAZTIiIiIioRGEyJiIiIqERgMCUiIiKiEqFUL7Cv0+lw584deHh4mOxjTERERET2J4oikpOTUaFCBYvbDstKdTC9c+cOKlWqZO9mEBEREVE+oqOj891RrlQHUw8PDwDSA/X09LRza4iIiIjIWFJSEipVqqTktryU6mAqd997enoymBIRERGVYNYMu+TkJyIiIiIqERhMiYiIiKhEYDAlIiIiohKhVI8xtYYoitBqtcjJybF3U6iUU6vVcHBw4NJkREREReSJDqZZWVm4e/cu0tLS7N0UekK4uroiKCgIjo6O9m4KERHRE+eJDaY6nQ6RkZFQq9WoUKECHB0dWemiAhNFEVlZWYiNjUVkZCRq1KiR7yLBREREZJsnNphmZWVBp9OhUqVKcHV1tXdz6Ang4uICjUaDGzduICsrC87OzvZuEhER0RPliS/5sKpFhYmvJyIioqLDT1kiIiIiKhEYTImIiIioRGAwLQNCQkIwa9Ysu98GERERUV6e2MlPpVn79u3RqFGjQguCR44cgZubW6HcFhEREVFRYTAtpURRRE5ODhwc8v8Vli9fvhhaRERERPR4ylRXviiKSMvSFvs/URStbuOIESOwa9cuzJ49G4IgQBAEREVFYefOnRAEAZs2bUKzZs3g5OSEPXv24Nq1a+jduzcCAgLg7u6O8PBwbN261eA2jbvhBUHAL7/8gueeew6urq6oUaMGVq9ebdNzefPmTfTu3Rvu7u7w9PTEgAEDcP/+feXyU6dOoUOHDvDw8ICnpyeaNm2Ko0ePAgBu3LiBiIgI+Pj4wM3NDfXq1cP69ettun8iIiJ68pSpiml6dg7qTtpU7Pd7fko3uDpa91TPnj0bly9fRv369TFlyhQAUsUzKioKADBhwgTMmDED1apVg7e3N27duoUePXpg6tSpcHZ2xq+//oqIiAhcunQJlStXtng/kydPxvTp0/H1119j7ty5GDJkCG7cuAFfX9982yiKIvr06QM3Nzfs2rULWq0Wo0ePxsCBA7Fz504AwJAhQ9C4cWPMnz8farUaJ0+ehEajAQCMGTMGWVlZ2L17N9zc3HD+/Hm4u7tb9fwQERHRk6tMBdPSwMvLC46OjnB1dUVgYKDJ5VOmTEGXLl2U035+fmjYsKFyeurUqVi1ahVWr16NN954w+L9jBgxAi+88AIA4IsvvsDcuXNx+PBhPPPMM/m2cevWrTh9+jQiIyNRqVIlAMCSJUtQr149HDlyBOHh4bh58ybGjx+P2rVrAwBq1KihXP/mzZt4/vnn0aBBAwBAtWrV8r1PIiIievKVqWDqolHj/JRudrnfwtKsWTOD06mpqZg8eTLWrl2LO3fuQKvVIj09HTdv3szzdsLCwpSf3dzc4OHhgZiYGKvacOHCBVSqVEkJpQBQt25deHt748KFCwgPD8fYsWPxyiuvYMmSJejcuTP69++P0NBQAMCbb76JUaNGYfPmzejcuTOef/55g/YQ0ZMlW5eN8/HnUc+vHhxUZepjh4hsVKbGmAqCAFdHh2L/JwhCoT0G49n148ePx8qVK/H5559jz549OHnyJBo0aICsrKw8b0fuVtd/bnQ6nVVtEEXR7GPSP//TTz/FuXPn0LNnT2zfvh1169bFqlWrAACvvPIKrl+/jqFDh+LMmTNo1qwZ5s6da9V9E1HpM3n/ZLy4/kXMOjbL3k0hohKuTAXT0sLR0RE5OTlWHbtnzx6MGDECzz33HBo0aIDAwEBlPGpRqVu3Lm7evIno6GjlvPPnzyMxMRF16tRRzqtZsybeeecdbN68GX379sWiRYuUyypVqoTXX38d//zzD9599138/PPPRdpmIrKf/679BwD49fyvdm4JEZV0DKYlUEhICA4dOoSoqCjExcXlWcmsXr06/vnnH5w8eRKnTp3C4MGDra58FlTnzp0RFhaGIUOG4Pjx4zh8+DCGDRuGdu3aoVmzZkhPT8cbb7yBnTt34saNG9i3bx+OHDmihNa3334bmzZtQmRkJI4fP47t27cbBFoiIiIqmxhMS6Bx48ZBrVajbt26KF++fJ7jRWfOnAkfHx+0atUKERER6NatG5o0aVKk7RMEAf/++y98fHzQtm1bdO7cGdWqVcNff/0FAFCr1YiPj8ewYcNQs2ZNDBgwAN27d8fkyZMBADk5ORgzZgzq1KmDZ555BrVq1cK8efOKtM1ERERU8gmiLYtsljBJSUnw8vJCYmIiPD09DS7LyMhAZGQkqlatCmdnZzu1kJ40fF0R2a7Brw2Un88MP2PHlhCRPeSV14yxYkpEREREJQKDKRERERGVCAymRERERFQiMJgSERERUYnAYEpEREREJQKDKRERERGVCAymRERERFQiMJgSERERUYnAYEpEREREJQKDaRkQEhKCWbNm2bsZRERERHlysHcDyFT79u3RqFGjQguTR44cgZubW6HcFhEREVFRYTAtpURRRE5ODhwc8v8Vli9fvhhaVLxsefxERERUOpStrnxRBLJSi/+fKFrdxBEjRmDXrl2YPXs2BEGAIAiIiorCzp07IQgCNm3ahGbNmsHJyQl79uzBtWvX0Lt3bwQEBMDd3R3h4eHYunWrwW0ad+ULgoBffvkFzz33HFxdXVGjRg2sXr06z3b9/vvvaNasGTw8PBAYGIjBgwcjJibG4Jhz586hZ8+e8PT0hIeHB55++mlcu3ZNuXzhwoWoV68enJycEBQUhDfeeAMAEBUVBUEQcPLkSeXYhw8fQhAE7Ny5EwAe6/FnZmZiwoQJqFSpEpycnFCjRg0sWLAAoiiievXqmDFjhsHxZ8+ehUqlMmg7ERERFb2yVW7KTgO+qFD89/vBHcDRuq702bNn4/Lly6hfvz6mTJkCQKp4RkVFAQAmTJiAGTNmoFq1avD29satW7fQo0cPTJ06Fc7Ozvj1118RERGBS5cuoXLlyhbvZ/LkyZg+fTq+/vprzJ07F0OGDMGNGzfg6+tr9visrCx89tlnqFWrFmJiYvDOO+9gxIgRWL9+PQDg9u3baNu2Ldq3b4/t27fD09MT+/btg1arBQDMnz8fY8eOxbRp09C9e3ckJiZi37591j6DioI8/mHDhuHAgQOYM2cOGjZsiMjISMTFxUEQBIwcORKLFi3CuHHjlPtYuHAhnn76aYSGhtrcPiIypRbUyBFz7N0MIioFylYwLQW8vLzg6OgIV1dXBAYGmlw+ZcoUdOnSRTnt5+eHhg0bKqenTp2KVatWYfXq1UpF0pwRI0bghRdeAAB88cUXmDt3Lg4fPoxnnnnG7PEjR45Ufq5WrRrmzJmD5s2bIyUlBe7u7vj+++/h5eWFP//8ExqNBgBQs2ZNg3a9++67eOutt5TzwsPD83s6TNj6+C9fvozly5djy5Yt6Ny5s9J+2UsvvYRJkybh8OHDaN68ObKzs/H777/j66+/trltRGSeIAiA9R1HRFSGla1gqnGVqpf2uN9C0qxZM4PTqampmDx5MtauXYs7d+5Aq9UiPT0dN2/ezPN2wsLClJ/d3Nzg4eFh0jWv78SJE/j0009x8uRJJCQkQKfTAQBu3ryJunXr4uTJk3j66aeVUKovJiYGd+7cQadOnWx5qGbZ+vhPnjwJtVqNdu3amb29oKAg9OzZEwsXLkTz5s2xdu1aZGRkoH///o/dViKSqMrYqDEiKriyFUwFweou9ZLKeHb9+PHjsWnTJsyYMQPVq1eHi4sL+vXrh6ysrDxvxzhACoKghE1jqamp6Nq1K7p27Yrff/8d5cuXx82bN9GtWzflflxcXCzeV16XAYBKJX1oiXpjcbOzs80ea+vjz+++AeCVV17B0KFDMXPmTCxatAgDBw6Eq2vhfZkgKuvUKjVg/u2FiMgAv8aWQI6OjsjJsW481p49ezBixAg899xzaNCgAQIDA5XxqIXl4sWLiIuLw7Rp0/D000+jdu3aJtXVsLAw7Nmzx2yg9PDwQEhICLZt22b29uVVA+7evaucpz8RKi/5Pf4GDRpAp9Nh165dFm+jR48ecHNzw/z587FhwwaDYQtE9PgECPZuAhGVEgymJVBISAgOHTqEqKgoxMXFWaxkAkD16tXxzz//4OTJkzh16hQGDx6c5/EFUblyZTg6OmLu3Lm4fv06Vq9ejc8++8zgmDfeeANJSUkYNGgQjh49iitXrmDJkiW4dOkSAODTTz/FN998gzlz5uDKlSs4fvw45s6dC0Cqaj711FOYNm0azp8/j927d+Ojjz6yqm35Pf6QkBAMHz4cI0eOxL///ovIyEjs3LkTy5cvV45Rq9UYMWIEJk6ciOrVq6Nly5aP+5QRkR61oLZ3E4iolGAwLYHGjRsHtVqNunXrKt3mlsycORM+Pj5o1aoVIiIi0K1bNzRp0qRQ21O+fHksXrwYK1asQN26dTFt2jSTJZb8/Pywfft2pKSkoF27dmjatCl+/vlnZcjA8OHDMWvWLMybNw/16tVDr169cOXKFeX6CxcuRHZ2Npo1a4a33noLU6dOtapt1jz++fPno1+/fhg9ejRq166N//3vf0hNTTU45uWXX0ZWVharpURFQBBYMSUi6wiiaMMimyVMUlISvLy8kJiYCE9PT4PLMjIyEBkZiapVq8LZ2dlOLaTSYt++fWjfvj1u3bqFgIAAi8fxdUVku6f/fBoPMx8CAM4MP2PfxhBRscsrrxkrW5OfiIxkZmYiOjoaH3/8MQYMGJBnKCWiglEJ7JwjIuvw3YLKtD/++AO1atVCYmIipk+fbu/mED2RGEyJyFp8t6AybcSIEcjJycGxY8dQsWJFezeH6InEdUyJyFp8tyAioiLFyU9EZC0GUyIiKlLsyicia/HdgoiIihSDKRFZi+8WRERUpBhMichafLcgIqIixZ2fiMhaDKZERFSkOPmJiKzFYPqECgkJwaxZsyxePmLECPTp06fY2kNEZReXiyIia/HdgoiIihQrpkRkLQZTIiIqUhxjSkTWKlPBVBRFpGWnFfs/URStbuOPP/6IihUrQqfTGZz/7LPPYvjw4QCAa9euoXfv3ggICIC7uzvCw8OxdevWx3puMjMz8eabb8Lf3x/Ozs5o06YNjhw5olz+4MEDDBkyBOXLl4eLiwtq1KiBRYsWAQCysrLwxhtvICgoCM7OzggJCcGXX375WO0hoicHZ+UTkbUc7N2A4pSuTUeLZS2K/X4PDT4EV42rVcf2798fb775Jnbs2IFOnToBkELhpk2bsGbNGgBASkoKevTogalTp8LZ2Rm//vorIiIicOnSJVSuXLlAbZwwYQJWrlyJX3/9FVWqVMH06dPRrVs3XL16Fb6+vvj4449x/vx5bNiwAeXKlcPVq1eRnp4OAJgzZw5Wr16N5cuXo3LlyoiOjkZ0dHSB2kFETx525RORtcpUMC0NfH198cwzz2DZsmVKMF2xYgV8fX2V0w0bNkTDhg2V60ydOhWrVq3C6tWr8cYbb9h8n6mpqZg/fz4WL16M7t27AwB+/vlnbNmyBQsWLMD48eNx8+ZNNG7cGM2aNQMgTa6S3bx5EzVq1ECbNm0gCAKqVKlS0IdPRE8gTn4iImuVqWDq4uCCQ4MP2eV+bTFkyBC8+uqrmDdvHpycnLB06VIMGjQIarU0Tis1NRWTJ0/G2rVrcefOHWi1WqSnp+PmzZsFat+1a9eQnZ2N1q1bK+dpNBo0b94cFy5cAACMGjUKzz//PI4fP46uXbuiT58+aNWqFQBphn+XLl1Qq1YtPPPMM+jVqxe6du1aoLYQ0ZNHpWIwJSLrlKlgKgiC1V3q9hQREQGdTod169YhPDwce/bswbfffqtcPn78eGzatAkzZsxA9erV4eLign79+iErK6tA9yePgTXubhNFUTmve/fuuHHjBtatW4etW7eiU6dOGDNmDGbMmIEmTZogMjISGzZswNatWzFgwAB07twZf//9dwGfASJ6krBiSkTW4rtFCeTi4oK+ffti6dKl+OOPP1CzZk00bdpUuXzPnj0YMWIEnnvuOTRo0ACBgYGIiooq8P1Vr14djo6O2Lt3r3JednY2jh49ijp16ijnlS9fHiNGjMDvv/+OWbNm4aefflIu8/T0xMCBA/Hzzz/jr7/+wsqVK5GQkFDgNhHRk4OTn4jIWmWqYlqaDBkyBBERETh37hxefPFFg8uqV6+Of/75BxERERAEAR9//LHJLH5buLm5YdSoURg/fjx8fX1RuXJlTJ8+HWlpaXj55ZcBAJMmTULTpk1Rr149ZGZmYu3atUponTlzJoKCgtCoUSOoVCqsWLECgYGB8Pb2LnCbiOjJwWBKRNYqMe8WX375JQRBwNtvv23vppQIHTt2hK+vLy5duoTBgwcbXDZz5kz4+PigVatWiIiIQLdu3dCkSZPHur9p06bh+eefx9ChQ9GkSRNcvXoVmzZtgo+PDwDA0dEREydORFhYGNq2bQu1Wo0///wTAODu7o6vvvoKzZo1Q3h4OKKiorB+/XqOKyMiAIbB1Jbl84io7BHEEvAuceTIEQwYMACenp7o0KFDnltp6ktKSoKXlxcSExPh6elpcFlGRgYiIyNRtWpVODs7F0GrqSzi64rIdq9sfgWH7koTT08OPQm1igvuE5UleeU1Y3YvaaWkpGDIkCH4+eefleocERE9OfR3ftKJBR92RERPPrsH0zFjxqBnz57o3LlzvsdmZmYiKSnJ4B8REZVs+it+5Ig5dmwJEZV0dp389Oeff+L48eMGW1/m5csvv8TkyZOLuFVERFSY9JeLYsWUiPJit4ppdHQ03nrrLfz+++9Wj9WbOHEiEhMTlX/c9pKIqOTTn/zEYEpEebFbxfTYsWOIiYkxWJ8zJycHu3fvxnfffYfMzExlpyOZk5MTnJycirupRET0GPSDKbvyiSgvdgumnTp1wpkzZwzOe+mll1C7dm289957JqGUiIhKJ1ZMichadgumHh4eqF+/vsF5bm5u8PPzMzmfiIhKLwGc/ERE1rH7rHwiIio7SsDS2URUgpWoLUl37txp7yYQEVEhE5EbRlkxJaK8sGL6hAoJCbF6By0iouLCMaZElBcGUyIiKlL63fesmBJRXhhMqcTIzs62dxOIqIixYkpEeSlTwVQURejS0or9ny2D/X/88UdUrFgROp3hm/ezzz6L4cOHAwCuXbuG3r17IyAgAO7u7ggPD8fWrVttei6OHDmCLl26oFy5cvDy8kK7du1w/Phxg2MePnyIV199FQEBAXB2dkb9+vWxdu1a5fJ9+/ahXbt2cHV1hY+PD7p164YHDx4AMD+UoFGjRvj000+V04Ig4IcffkDv3r3h5uaGqVOnIicnBy+//DKqVq0KFxcX1KpVC7NnzzZp/8KFC1GvXj04OTkhKCgIb7zxBgBg5MiR6NWrl8GxWq0WgYGBWLhwoU3PEREVDv0xpgymRJSXEjX5qaiJ6em41KRp/gcWslrHj0FwdbXq2P79++PNN9/Ejh070KlTJwDAgwcPsGnTJqxZswYAkJKSgh49emDq1KlwdnbGr7/+ioiICFy6dAmVK1e26n6Sk5MxfPhwzJkzBwDwzTffoEePHrhy5Qo8PDyg0+nQvXt3JCcn4/fff0doaCjOnz+vrC978uRJdOrUCSNHjsScOXPg4OCAHTt2ICfHtm66Tz75BF9++SVmzpwJtVoNnU6H4OBgLF++HOXKlcP+/fvx6quvIigoCAMGDAAAzJ8/H2PHjsW0adPQvXt3JCYmYt++fQCAV155BW3btsXdu3cRFBQEAFi/fj1SUlKU6xOR/TCYElFeylQwLQ18fX3xzDPPYNmyZUowXbFiBXx9fZXTDRs2RMOGDZXrTJ06FatWrcLq1auVymF+OnbsaHD6xx9/hI+PD3bt2oVevXph69atOHz4MC5cuICaNWsCAKpVq6YcP336dDRr1gzz5s1TzqtXr57Nj3fw4MEYOXKkwXmTJ09Wfq5atSr279+P5cuXK8Fy6tSpePfdd/HWW28px4WHhwMAWrVqhVq1amHJkiWYMGECAGDRokXo378/3N3dbW4fET0+zsonImuVqWAquLig1vFjdrlfWwwZMgSvvvoq5s2bBycnJyxduhSDBg1SqpWpqamYPHky1q5dizt37kCr1SI9PR03b960+j5iYmIwadIkbN++Hffv30dOTg7S0tKU2zh58iSCg4OVUGrs5MmT6N+/v02Py5xmzZqZnPfDDz/gl19+wY0bN5Ceno6srCw0atRIafedO3eUkG7OK6+8gp9++gkTJkxATEwM1q1bh23btj12W4no8bFiSkR5KVvBVBCs7lK3p4iICOh0Oqxbtw7h4eHYs2cPvv32W+Xy8ePHY9OmTZgxYwaqV68OFxcX9OvXD1lZWVbfx4gRIxAbG4tZs2ahSpUqcHJyQsuWLZXbcMknTOd3uUqlMhlba25yk5ubm8Hp5cuX45133sE333yDli1bwsPDA19//TUOHTpk1f0CwLBhw/D+++/jwIEDOHDgAEJCQvD000/nez0iKiJ6bwWsmBJRXspUMC0tXFxc0LdvXyxduhRXr15FzZo10bRp7tjYPXv2YMSIEXjuuecASGNOo6KibLqPPXv2YN68eejRowcAIDo6GnFxccrlYWFhuHXrFi5fvmy2ahoWFoZt27YZdLvrK1++PO7evaucTkpKQmRkpFXtatWqFUaPHq2cd+3aNeVnDw8PhISEYNu2bejQoYPZ2/Dz80OfPn2waNEiHDhwAC+99FK+90tExYM7PxFRXsrUrPzSZMiQIVi3bh0WLlyIF1980eCy6tWr459//sHJkydx6tQpDB482GQWf36qV6+OJUuW4MKFCzh06BCGDBliUI1s164d2rZti+effx5btmxBZGQkNmzYgI0bNwIAJk6ciCNHjmD06NE4ffo0Ll68iPnz5yvhtmPHjliyZAn27NmDs2fPYvjw4cpQhPzadfToUWzatAmXL1/Gxx9/jCNHjhgc8+mnn+Kbb77BnDlzcOXKFRw/fhxz5841OOaVV17Br7/+igsXLiirGRCRfXCMKRFZi8G0hOrYsSN8fX1x6dIlDB482OCymTNnwsfHB61atUJERAS6deuGJk2a2HT7CxcuxIMHD9C4cWMMHToUb775Jvz9/Q2OWblyJcLDw/HCCy+gbt26mDBhgjLrvmbNmti8eTNOnTqF5s2bo2XLlvjvv//g4CAV4SdOnIi2bduiV69e6NGjB/r06YPQ0NB82/X666+jb9++GDhwIFq0aIH4+HiD6ikADB8+HLNmzcK8efNQr1499OrVC1euXDE4pnPnzggKCkK3bt1QoUIFm54bIio6HGNKRHkRxFLcr5KUlAQvLy8kJibC09PT4LKMjAxERkaiatWqcHZ2tlMLyV7S0tJQoUIFLFy4EH379i202+Xrish2Y7aNwe5buwEAi7otQrNA00mPRPTkyiuvGeMYU3qi6HQ63Lt3D9988w28vLzw7LPP2rtJRKSHFVMiyguDKT1Rbt68iapVqyI4OBiLFy9WhhYQkf3od8zpwGBKRJbxU5ueKCEhIZz1S1SC2TpRk4jKFk5+IiKiIsVZ+URkrSc+mLJ6RoWJryeix8MxpkSUlyc2mGo0GgDS7GyiwiK/nuTXFxHljxVTIrLWEzvGVK1Ww9vbGzExMQAAV1dXCIJg51ZRaSWKItLS0hATEwNvb2+rNgsgIlPsdSCivDyxwRQAAgMDAUAJp0SPy9vbW3ldEZGV9LIoK6ZElJcnOpgKgoCgoCD4+/sjOzvb3s2hUk6j0bBSSvSYOMaUiPLyRAdTmVqtZqAgIrITjjElIms9sZOfiIio5GHFlIjywmBKRERFymDnJwZTIsoDgykRERUbBlMiyguDKRERFSmOMSUiazGYEhFRsWHFlIjywmBKRETFhsGUiPLCYEpEREWKXflEZC0GUyIiKjasmBJRXhhMiYioaOltScpgSkR5YTAlIqJiw658IsoLgykRERUp/TGmrJgSUV4YTImIqNgwmBJRXhhMiYioSHFWPhFZi8GUiIiKDSumRJQXBlMiIipSoqhXMdWxYkpEljGYEhFRsdHv1iciMsZgSkRERYpjTInIWgymRERUbHQ6jjElIssYTImIqEgZjDFlxZSI8sBgSkRExYaz8okoLwymRERUbBhMiSgvDKZERFRsGEyJKC8MpkREVKQ4K5+IrMVgSkRExYYVUyLKC4MpEREVKc7KJyJrMZgSEVGx0Q+pRETGGEyJiKhIcYwpEVmLwZSIiIoNx5gSUV4YTImIqEixYkpE1mIwJSKiYsOKKRHlhcGUiIiKlt58JwZTIsoLgykRERUbBlMiyguDKRERFSmOMSUiazGYEhFRsWHFlIjywmBKRERFijs/EZG1GEyJiKjYcOcnIsoLgykRERUpjjElImsxmBIRUbHhGFMiyguDKRERFSlWTInIWgymRERUbHQ6VkyJyDIGUyIiKlL6E550YDAlIssYTImIqNhwjCkR5YXBlIiIig3HmBJRXhhMiYio2HCMKRHlhcGUiIiKFGflE5G1GEyJiKjY6IdUIiJjDKZERFSk9Gfl5+hYMSUiyxhMiYio2HBWPhHlhcGUiIiKFMeYEpG1GEyJiKjYsGJKRHlhMCUioiKlXzFlMCWivDCYEhFRsWEwJaK8MJgSEVGRMpiVzzGmRJQHBlMiIio2rJgSUV4YTImIqNiwYkpEeWEwJSKiYqPfrU9EZIzBlIiIihTHmBKRtRzs3YDSJCY5A+lZOfBzd4K7E586IiJbcYwpEeWFFVMbjF9xGu2+3okNZ+7auylERKUGd34iImsxmNpAJUj/c4QUEVHBsWpKRJYwmNpAEKRkysH7RETWE42+zjOYEpEldg2m8+fPR1hYGDw9PeHp6YmWLVtiw4YN9mxSnpSKKXMpEVGBMZgSkSV2DabBwcGYNm0ajh49iqNHj6Jjx47o3bs3zp07Z89m5UFKpjoGUyIiqxn3MnGcKRFZYtep5REREQanP//8c8yfPx8HDx5EvXr17NQqy3LHmDKZEhEVFCumRGRJiVnzKCcnBytWrEBqaipatmxp9pjMzExkZmYqp5OSkoqreQCAR0NMWTElInoMrJgSkSV2n/x05swZuLu7w8nJCa+//jpWrVqFunXrmj32yy+/hJeXl/KvUqVKxdpWARxkSkT0uDiBlIgssXswrVWrFk6ePImDBw9i1KhRGD58OM6fP2/22IkTJyIxMVH5Fx0dXaxtVT16tviWSkRkPePhT6yYEpEldu/Kd3R0RPXq1QEAzZo1w5EjRzB79mz8+OOPJsc6OTnBycmpuJuokCumOvblExEVGMeYEpEldq+YGhNF0WAcaUkicIF9IiKbmczK17FiSkTm2bVi+sEHH6B79+6oVKkSkpOT8eeff2Lnzp3YuHGjPZtlkbzAPgumREQFx5VNiMgSuwbT+/fvY+jQobh79y68vLwQFhaGjRs3okuXLvZslkW5C+zzTZWIqKA4xpSILLFrMF2wYIE9795mj3IpJ+UTEdnAZEtSHceYEpF5JW6MaUmmetSVz24oIqKCY8WUiCxhMLUFF9gnIrKZ8fAnzsonIksYTG2gVEwZTImICozBlIgsYTC1gTzGVMdkSkRkNS6wT0TWYjC1gVwxJSKigmPFlIgsYTC1gZxLufMTEVHBMZgSkSUMpjYQlFn5RERUUFpRa+8mEFEJxWBqA6ViyjGmRERW45akRGQtBlMb5O78ZN92EBGVZlodK6ZEZB6DqQ0EyMtFMZkSEVnLeFY+u/KJyBIGUxsoFVP7NoOIqFRjVz4RWcJgagN58hPHmBIRWU+umKoFNQCuY0pEljGYFgBzKRGR7RxUDgA4xpSILGMwtcGljH/hUmkR7mddsndTiIhKDXlcvhJMOcaUiCxgMLVBvPYyHNwv4aH2pr2bQkRU6sjBlGNMicgSBlMbuKvLAwBStDF2bgkRUekhjzF1ENiVT0R5YzC1gZvaHwCQoou1c0uIiEofpWLKyU9EZAGDqQ3kimlqDoMpEZHVHk0Y5eQnIsoPg6kN3B9VTFNz2JVPRGQrBlMiyg+DqQ28HAIBABniQ6Rkpdi5NUREpYPxGFN25RORJQymNnBSu0Gn9QAARCVF2bcxRESlDGflE1F+GExtoBIE6DKlcabXE6/buTVERKWDsvOTStr5ieuYEpElDKY2EADosqRxppGJkfZtDBFRKcMxpkSUHwZTGwh6FVMGUyIi6yg7P3GMKRHlg8HUBoIA6LLYlU9EVBAalQYAx5gSkWUMpjaQxphKXfnRSdHI1mXbuUVERCWfMiufXflElA8GUxsIAEStJ9RwglbU4lbyLXs3iYio1FCCKSc/EZEFDKY2UKkEACq4qYIAsDufiMgWakGalc+ufCKyhMG0ANyECgA4AYqIyBbsyiei/DCY2kAlCAAANxWDKRGRtZRZ+SrOyieivDGY2uBRLoWrIHXlRyVG2a8xRESlDCumRJQfBlMbqB4FUzfkjjGVKwFERGSeyax8Tn4iIgsYTG0gQEqmzgiASlAhJTsFcelxdm4VEVHpoCywz8lPRGQBg6kN5K58QdCgontFAEBUUpT9GkREVAoYV0w5xpSILGEwtYHwKJmKIpRgejvltj2bRERUasjBlJuTEJElDKY2eFQwhU4UEewRDIDBlIgoPyaz8tmVT0QWMJjaQJ78JCK3Ysrdn4iIrMOufCLKD4OpDXK78lkxJSKyljLGlJOfiCgfDKY2UCqmIhDs/iiYJjOYEhFZg2NMiSg/DKa2eFQx1YmiEkxj0mOQoc2wZ6uIiEq2R8s9syufiPLDYGoD/Yqpl5MX3DRuAIA7qXfs2CoiotKBk5+IKD8MpjaQF9jXidJ4U7lqyglQRESWcecnIrIWg6kN5Iqp3C8V5C5tTXov9Z59GkREVIoowVTHYEpE5jGY2kDe+Un3aLxUgGsAAAZTIqK8mMzK5xhTIrKAwdQG+stFAUCgWyAA4H7afbu1iYiotGDFlIzdSbmDrJwsezeDShAGUxvk7vwk/S8HU1ZMiYgs485PZM6p2FPotrIbXlz/or2bQiUIg6kNVHLF9NFpuSufFVMiovxx8hPpW3NtDQDgQsIFO7eEShIGUxsIynJRhl3591LvKecREZEheYypWlADYFc+SVQCIwiZ4qvCBkrF1GjyU2ZOJh5mPrRTq4iISgcusE/6GEzJHL4qbJA7K19Kpo5qR/g5+wHgOFMiovxwjCnpE5SZG0S5GExtIBhVTAEgwI3jTImIrKFRaQCwYkoSeWgHkT4GUxvkzsrPTaaBrpyZT0SUF+NZ+dm6bHs2h0oIduWTOQV6Vfz6669Yt26dcnrChAnw9vZGq1atcOPGjUJrXEmjTH7SO49LRhERWUdZYJ9d+YTcXkgAnEBMigIF0y+++AIuLi4AgAMHDuC7777D9OnTUa5cObzzzjuF2sCSRGW0wD7Arnwiovwos/JVauW0TtTZs0lUAuh35bOKTjKHglwpOjoa1atXBwD8+++/6NevH1599VW0bt0a7du3L8z2lSjydzv9L3bsyiciso7clQ9IS0Y5qh3t2BqyN/2KaWZOJl8PBKCAFVN3d3fEx8cDADZv3ozOnTsDAJydnZGenl54rSthBKMF9gF25RMR5cd4jCnAtUzJUGZOpr2bQCVEgSqmXbp0wSuvvILGjRvj8uXL6NmzJwDg3LlzCAkJKcz2lSjGy0UBucH0ftp96EQdB3MTEVmgETTKz5yZT/pfTrJysuzYEipJCpSivv/+e7Rs2RKxsbFYuXIl/PyktTyPHTuGF154oVAbWJIYL7APAOVdywOQxsdwkX0iIlPGY0wBVkzJcFwpK6YkK1DF1NvbG999953J+ZMnT37sBpVkuWNMc5OpRqWBr7MvEjISEJsWC19nX/s0joiohFMJKggQIEJkxZSQnZMbTFkxJVmBKqYbN27E3r17ldPff/89GjVqhMGDB+PBgweF1riSRvXo2TJe1MLf1R8AEJMWU7wNIiIqBUS9d015nCkrpqQVc18DrJiSrEDBdPz48UhKSgIAnDlzBu+++y569OiB69evY+zYsYXawJJE3j5NZ7TeWnkXqTs/Nj222NtERFSaMJiSTL9iymBKsgJ15UdGRqJu3boAgJUrV6JXr1744osvcPz4cfTo0aNQG1iSKAvsG5VM5Yop1zIlIjLj0XumAEFZu5Jd+aRfMWVXPskKVDF1dHREWloaAGDr1q3o2rUrAMDX11eppD6J5OWidBaCaWwaK6ZERHmRK6bc/YlYMSVzClQxbdOmDcaOHYvWrVvj8OHD+OuvvwAAly9fRnBwcKE2sCRRKRVTo678RzPzGUyJiCwThNyKqX61jMomzsoncwpUMf3uu+/g4OCAv//+G/Pnz0fFihUBABs2bMAzzzxTqA0sSeQxpiZd+S7syiciskR/8pO8ZBTHmJL+ayBDm2HHllBJUqCKaeXKlbF27VqT82fOnPnYDSrJlIqp0bx8pSufk5+IiCwSIECjkhbZZ1c+6VdM07RpdmwJlSQFCqYAkJOTg3///RcXLlyAIAioU6cOevfuDbVanf+VSytl5yfDs+Wu/Pj0eGh1WoNt94iIyjqDiiknP9EjBsE0m8GUJAVKUFevXkWPHj1w+/Zt1KpVC6Io4vLly6hUqRLWrVuH0NDQwm5niZC785NhMvV19oWD4ACtqEVcepyyTSkREeUSBEHpytcPJVQ26Xfls2JKsgKNMX3zzTcRGhqK6OhoHD9+HCdOnMDNmzdRtWpVvPnmm4XdxhIjd+cnw/NVggrlXMsB4AQoIiJj+l/mlVn5rJiWeayYkjkFqpju2rULBw8ehK9v7vabfn5+mDZtGlq3bl1ojStpVI8GmRrv/ARIE6Dupd5DTDp3fyIissRB4HJRJOEYUzKnQBVTJycnJCcnm5yfkpICR0fHx25USSVXTI13fgJyx5lyW1IiIkPmxphyVj7pvwZSs1Pt2BIqSQoUTHv16oVXX30Vhw4dgiiKEEURBw8exOuvv45nn322sNtYYlja+QngIvtERPkRkDvGlOuYEseYkjkFCqZz5sxBaGgoWrZsCWdnZzg7O6NVq1aoXr06Zs2aVchNLDlyd34yTaZyMGXFlIjIMnmMKSumpBN1ys/p2el2bAmVJAUaY+rt7Y3//vsPV69exYULFyCKIurWrYvq1asXdvtKFEuTnwCgvAu78omI8iIIueuYclY+6U+KY8WUZFYH07Fjx+Z5+c6dO5Wfv/322wI3qCSTl4syR9mWlIvsExEpjJfXc1RL8xD090mnskmH3IopZ+WTzOpgeuLECauOE/IIb6WdoCywb1oyDXANAMCKKRGRJQIEOKqkYJqVk2Xn1pC96Xfls2JKMquD6Y4dO4qyHaVC7gL7ppfJFdOkrCRkaDPg7OBcnE0jIiqRjLdw1qilrvwsHYNpWWfQlc+KKT1SoMlPZZ25iqmHxgPOaimMcmY+EZEpAbljTFkxJf0vLenadIMKKpVdDKY2UCqmZi4TBCF3Zj4X2SciAmB5jCkrpqQfREWIyNBm2LE1VFIwmNogdx1Tc9FUbwIUK6ZERCYEIXeMKSc/kfFnKceZEsBgapO8xpgC0rakAHA/7X5xNYmIqEQzHmOqzMrnclFlnv6sfIDjTEnCYGqDvGblA9z9iYgoPxxjSjLjMaWsmBLAYGoTldyVb+FyuSufY0yJiCSWKqYcY0omXfmsmBLsHEy//PJLhIeHw8PDA/7+/ujTpw8uXbpkzybl49GWpLq8K6Zcy5SIyJQgCLnBlBXTMk+umHpoPACwYkoSuwbTXbt2YcyYMTh48CC2bNkCrVaLrl27IjU11Z7NskipmFoomcrbkrIrn4joEaP3S2VLUk5+KvPkarqboxsAVkxJYvUC+0Vh48aNBqcXLVoEf39/HDt2DG3btrVTqyxTP0qmlsaYyrs/xabHQhTFJ3oXLCIiWxns/MSu/DJPrpi6a9wBsGJKkhI1xjQxMREA4Ovra/byzMxMJCUlGfwrTnIw1Vroyi/nWg6AtFBwcnZysbWLiKiksjjGlF35ZZ4cTOXNaZZeWGrP5lAJUWKCqSiKGDt2LNq0aYP69eubPebLL7+El5eX8q9SpUrF2kY5mOZYCKYuDi7wcJTGyrA7n4jIkABB2ZKUy0WRPPlJ7sq/mHCR3flUcoLpG2+8gdOnT+OPP/6weMzEiRORmJio/IuOji7GFuoFU0uDTJHbnc8JUEREZiqmKlZMSSKvY9o7tLdyXlJW8faEUslj1zGmsv/7v//D6tWrsXv3bgQHB1s8zsnJCU5OTsXYMkMOKinHi6I0M1+lMh1DWt6lPK4+vMpgSkRkRH9WPiumJHflt6zQEl5OXkjMTERqdsmc/EzFx64VU1EU8cYbb+Cff/7B9u3bUbVqVXs2J19qvSBqaZypvJZpXHpcsbSJiKgkM16rkhVTAgxfFypBpUyASslOsVeTqISwa8V0zJgxWLZsGf777z94eHjg3r17AAAvLy+4uLjYs2lmOegFU0vjTP1c/AAwmBIRGdMfY8pZ+WWb/hAPFVRw00jjTFOzWDEt6+xaMZ0/fz4SExPRvn17BAUFKf/++usvezbLIsOKqc7sMeWcpZn58enxxdImIqKSzHiMKbckJcBwO1JBEFgxJYVdK6bGXTwlnX4wtZBLla782HTOyiciMuakluYJMJiWbSZd+Y5SMOUYUyoxs/JLA7VgRcXURaqYsiufiMi0AOHiIA3TStem26M5VELIM/IBKZjKXflltWJ6I+kGll5Yyh3RUEJm5ZcWKpUAlQDoxPzHmLIrn4jIkCAISgBJ06Zxh7wyzKArH09uV/6qK6twIuYEJrWcBAeV5cj1/OrnkZmTiaTMJIxqNKoYW1jysGJqI3nJKIu7Pz2qmCZnJyNDm1Fs7SIiKg3kYKoTdayalmEWZ+VnPVnBdNL+SVh1dRU2RG7I87jMnEwAwJ7be4qjWSUag6mN8tv9yUPjoSyHEp/BqikRkUyAAGcHZwiQ3ke5N3rZpV8x1e/Kf1LHmN5LvWfVcU9axbggGExtlF8wFQSB40yJiB4xnpWvElTKOFNuP1l26Y8xFQRBmfz0pAYza3sHuFwWg6nN5GBqqSsf4AQoIiJz5PGkT3p1jPJn0JWPJ3/yk9XBVMu/CQZTGznkUzEF9IJpGoMpEZVt5pYFZDAlS+uYPkkVQ61Oq/xsdTDl3wSDqa1yK6YWFjKFXjDNYDAlIpLJY0tdNa4AOMa0LDOelf8kVkz11+rlRD/rMZjayKaKKbvyiaiMMx5jCgCuDo+CKceYllny60KAAEEQ4OHoAeDJqhhm5OSuzMMvYdZjMLWRWp1/MJXXMmUwJSLKJVdM2ZVP8hAPlSDFkCe9YpqUmWTHlpQuDKY2knd/sqZiykX2iaisM1sxfdSVz2Badsld+fKEOGWMaXZqqduu3BL9tcyTsvIOpvJWvQCQrSvbuz8xmNqIs/KJiAro0SZPHhqp2/ZJqo6RbeQvLCoYVkyfpI0X5EXzgfxf6/LwFoDVVQZTG8k7P1k7xvRJ+eZHRFQQ5t4DPZ08AeRfRaInl1wxlbvyXRxclJ+flC8s+l35+e1opb+ua2JWYpG1qTRgMLWRLRXTbF0233iJiB6Rx5h6Oj4KpmW8MlSWGXflC4LezPwnZFtS/clPqdmpyNHlGCwhpS9Hl6P8nJjJYEo2cHg0+UmXRzB1VDsqb7zsziciMqQEU35xL7OMJz8Bud35T8oMdv2KqQgRPVf1RP81/c0GT/3AymBKNlEJ+VdMAY4zJSIyplRM2ZVf5sld1yq9GOLm8CiYPiHLiOlXTAHgdsptXH14FYvPLTY5Vj+YPsx8WMQtK9kYTG2Uu46p5QX2AQZTIiLAwhhTduWXecZd+cCTt1qDfsVU382kmwanRVGEVmTFVMZgaiNrxpgCXMuUiMiYHELYlU/muvKftB3B9JeL0vcg84HB6Rwxx+C0cXAtaxhMbeRgxQL7ANcyJSICzK9jyq58Uiqm0KuYOpSNimlCeoLBaeMJUcdjjhdZm0oDBlMbqa1YLgrIDaax6bFF3iYiotJADiHeTt4ApHUen5QZ2MUlMTMR/139D9k5pXsRdnmMqX5Xvjz5qajWMT0Xfw53Uu4UyW2bYzzGVJaQkXcwvfrw6hMzzrYgGExt5GBlV355l/IA2JVPRGWbuYqpm8ZN+fIemRhZ3E0q1YZuGIqP9n2E5ZeX27spj8VsV34RVkxvJd/CoLWD0G1lN1yIv1Dot2+OpcfxMPOhwfJQ+sHUxcEFQNkuajGY2khlxZakAMeYEhEZ06+OhXqFAgCuJ163V3NKnbj0OCXI74jeYefWPB5lgX2YWS6qCKqF+p/FW25sKfTbN8fSRgEiRIOZ9/LEJwECi1pgMLWZrRXTsvyth4jI0u531byrAQCuJV4rzuaUardTbis/O6oc7diSxydX0vW/rLhopGphqrbwK6b6E4xuJhfP5CI5YPu7+ptcph885YqpWqXmij5gMLWZWp78lJP3clFyME3MTLQ4AJqIqKwKdg8GANxLuWfnlpQemdrcvdeLc6xkUTC7wH4RrmOq33VeXLPe5YppVa+qJpfdSr6l/CwHU41Kw95WMJjaTFnHNO+CKbycvOCgcgDAmflEVHaZG2MKAOVdpS/vMekxxdmcUi0zJzeY3k65bXF7y9JA6co3t1xUUQRTo4qppUq+LK9Jebtv7cZz/z2Hs3Fn87wNeYxpDe8aynnNA5sDAKKTo5Xz5N+jg+DAFX3AYGoztZUL7AuCwJn5RESP6C8LBHCCaEHo975l5GRg/539dmzN41Fm5eu9Ltw17gAsj80siKycLGy7sc1g0frU7NQ8J1htjNqIln+0xNILS81ePmbbGFx9eBUzjs4we3lSVhIO3DmA5KxkAECTgCYY23QsZneYjYblGwIAbqWYVkwdVLnBNCat7H5hc7B3A0oba8eYAtIb773Ue3zjJSIyolRMy/AHsK30K6YAsP3mdrQNbmun1jwec135Ps4+AEyXU3occ47Pwa/nfzU5PzY9Fu6OUhDOysnCqiur4OXshcsJl/HzmZ8BANMOT8OQOkOU68SnxxsESmcHZ7P3+fKml3Ex4aJy2k3jhpfqvwQgd1cng4qpmDvGtJJHJQDAjaQbtj/YJwSDqY2Uiml+ffngzHwiIpn+JBcgt2Kark1HanaqMiObLDMOppcSLtmpJY/PXFe+n7P0mRmfUXjd2GuurzF7flx6nDL287fzv2H28dl53k5CRgKe/fdZg00hKntUNnusfigFYPDaDvaQxlbrB1N5/KuDygHVvKRJgdcTr0MURZO/m7KAXfk2snZLUoAz84mILI3lc9W4Kl2391PvF2eTSi05mNbyqQUAuPLwil3Hmcalx+Hfq/+aBGZrKDs/6QUvuZiTmJmIbF3hbCDgpHYye77+GM4j947keztbb2w12anM2udefp0DUCqid1PuKteXH6uD4IAqnlUgQEBSVlKhBvTShMHURhq19JRl5zMrH+AYKiKivMgf0lFJUfZtSCkhB8Bq3tXg6uCKzJxMRCVG2a09b+14Cx/v+xjfn/ze5uvKk+L01zH1cvKCWlADAB5kPDB7PVtZCqb6BSN5XGdezIVEa1fc0a+Y+rv6w1HlCK2oxb1UaUUK/TGmzg7OqOBeAQDs+ru1JwZTGzk6SE9Zljb/YKp05acxmBJR2aSsVwnTLskQrxAADKbWkoOQi4MLavlKVdOLDy7mdZUidTr2NABg0dlFWHZhmU1LWJmrmKoElTLOND49HssvLcec43OUYwvC0jjQ6UemK8+nNWNakzKTTM6zNpjKuzkB0mOs6FERQG53vjzGVF7Jp4pnFYPLyxoGUxs5OUjf5jKtCKbsyicisqyqpzTGr6xWhmwl773upHZSuvNLyjjTLw9/iZGbRlq9nai5YArkjjO9kXQDnx38DD+f+RkH7hwocLsc1YYbEcgVWSB31zE5mA6sNdDi7Rh34wOmY34tkbdalck9BUow1RkGU3nsanFtBFDSMJjayMmGiqk865Rd+URPLq1Oiz8u/oHopLJZ3ciPPMbUXMVU3v3JeLIImSdX6JzUTqjjVwcAcCGhePZ9t8btlNvYdnObVccqs/KNYkjTgKYAYDAZ6bfzv+W77qglxl35rSq0Qh1f6bmTK7zyeNN6fvVMrp+dI43/lCumA2sNxITwCQCALF3+FVO1oFYCp0wOpvIi+/LGCc5qqbpb2VMKpmV1Zj6DqY0cH40xzbJijKn+QrmP0xVBRCXXruhd+OLQF+ixqgd3ebORvKbjpQeXrK60lWVyhc5R7ah05V9KuFTg0GZs6YWlWH5puVXHWnqt74zeadX15XVM9WflA0D/mv0BGK7zuf/Ofuy7s8+q2zVmHEzVglqZGX839S5SslKUJcvq+tU1ub5cKZX/Dw8MVz7brfl7d3FwMakKy7ueyY8xTSttKCBvMCBXTNmVT1Zx0khPWaY2J58jc7sktKLWYHFfInpyRCZFKj9vj95ux5aUcGZWvQl0C0QFtwrQiTqcij1V/G0qJRIyEhCfHm9QMa3uXR0A8DDzId7d9e5j38e1h9cw7fA0fHbws3x3NJLbpK99cHsAwLm4c1bdn6Wu/FDvUPg6+5ocb83MeXNMgqlKjUC3QABS8Htt62sQISLYPVh5TvXJn91yMPVy8lKGB1gTTM2NcTXuype/lMld/nLF9GZS/jtUPYkYTG2kVEyt6MrXqDXwcZIGcnOcKdGTSX9Zmz239tixJSWTpS1JZY0DGgMATsacLIbWFL10bTpGbR2FLn93KZTxnw8yHqDvf33Rd3VfPMx8CEAKW05qJ1R0lybRbLmxpUBLNunT74JfcXlFvsfL3d/+rv74O+JvfPjUhwCAe2n3lO5vaxh35QuCgCb+TZTTzQKaAQBOxJzAxqiN+PrI1zZVEo2DqUpQIcgtCID0vMkTuD5v8znUKrXJ9eWqphxQPR094aiSgqk1z7n+xCeZfjAVRdGkYlrRvSJUggpp2rQyuWQUg6mN5Fn51kx+Ajgzn+hJpz9b9+i9o3ZsSclmbowpADQuLwXT4/ePF2dzisza62ux9/Ze3Eu9h6kHpwIA9t/ej+9OfKcspG6LPy7+gfiMeCRkJChVQzlszes8TznucZ+/w3cPKz9bU51Mzpa22/R09EQt31oIcA2As9oZOlGHsTvH5rs2rbkF9mWvNHgFDoIDPB09MbrRaADAlQdXMH7XePx2/jfMPDbT6sdlXJF1EByUiqzchd82uC2aBDQxuS4gVS0/2POBUlzydPRUnv+CVkwrelSEAAGp2al4kPkAadmPgumjiqmj2lEJz2/veLtQd8IqDRhMbWTLrHxAby3TDAZTotJOFEUcuHPAIIDqz9a9k3pH+ZCx1qorq7D9ZtkdAhAeFA4AOBZzzOzM59JGPyCejD2Jf6/+i9e2voYfT/9YoKEe+pOb5OdHDkbVvKqhd2hvAHismetanRan404rp6OTo/OtSirdz4+qfIIgKGM3d97aifG7x+fZDW2pKx8A6pWrh78i/sKynssQ4hkCAEjJTlEuPxFzwuoubuMvA2qVGh6OHgbn6Y8t7VO9DwApwAJSJVnePaqie0VUcK+Q25VvxeQnF7VpxdRJ7QR/V38A0gQo44opkDsJ7FTsKay5Zn73qicVg6mNbFnHFMidmR+bxq58ImtEJ0WX2G7djVEb8eqWV/HSppdw7eE1AEByVrLBMbasyXk/9T4m7Z+Et3a8hc1RmwuzqcXqbNxZRKyKyDNgW6qYVvOqhure1aHVaa2eOFNSXX5wGWuvrwWQO/n1430fK5fbEh6P3DuCzw58ZvY50e+eblWhFQBpglBBXUq4hHRtOtw17mgR2AKAVKnLqyIofwFzc8hdPL62b23l5xMxJ5TnwhxlVr6ZiikA1PSpiSqeVeDr7GuwxBMgrXSjPzkqLzmiYTBVCSqDnZgAKLP0AeCjpz7Cku5LMLHFRADA0fu5X0KnPT0NKkGlBFNzXfnGgdlcVz6Q251/Ju6MScUUAD5t+alSNS1rK/swmNrI1mCqdOWXsRcWUUGIooj/bfkfhm4Yivmn5iNHl4OLCRdLzKoWkYm5E53OxUuTPIyrfPLaiNbQ70l5b/d7Ni1QXpK8s/MdRCVF4a0db5lcZk1lq0OlDgCAfbcLNvO6pPjnyj8ApFnVf0f8jZo+NQ0u33d7n1XPR2p2KsZsG4Pll3NnyMu9bwAQ4Bag/NwiSAqSlx5csvg5I4oivjn6Db4/+b3Z+5dDbXhgOPrXkmbFX35wOc8x00ow1dvV6IMWH+CLNl+ge0h36fTeDyxOapNn5Vv6wiJTq9Rmd2a6nHA5z+vJjLc2dVA55FkxdVI7oZF/I1T1qmpwzIh6I9DIv5FyDGC+K984CFta4F+uiE47PA3/XfsPgGHFVKPWYFDtQQAMt08tCxhMbSRPfrJmVj7AbUmJbHEv9R5up9wGAMw7OQ+NljRC/zX9MfnAZDu3TKI/1uv6QymAysFUnul7O/m21beXmJG7WodW1GLLjS2F0cwCiUmLwU+nf8KMIzNs/iC0pkfIXJetrGWFlgCAfXf2IV2bbtN929OdlDvKOEUA2Ht7LwBgVKNR8HPxwy9df8GohqMwrtk4uDq44k7qHZOgFpUYhde2vIbRW0crAXFT1CaT52Fd33X4pesvmNtxrsHkID8XP6XiZ6kieyv5FhafW4wfTv2AjVEbTS7feWsnAKB1hdboWqUrwsqHAQDOxluenZ+qNezKBwAPRw9EhEbgk1afKOFv2IZhOBN7xuT6eY0xNRbgmhvE5QnFb+9826rPVeOufJWgMgim3k7eBrcvkyuaMv01TuXJT+aCqfF5liqmLzd4WakwywvsGy/EL4+F5RhTypO8XJQ165gC3P2J7OOvi39h4NqBpW4dvPPx582e/8+Vf0rEAvb6+3evvrYaWp1Wmfwk78RzN/Wu1beXmGW4jNzmG/bpzr+feh/DNgzD3BNz8ev5X9F+eXurl/0BTKtE+vKblQ8Ajco3QpBbEBIzEzF43WCkZacpS0iV1GEda66tQbeV3dB3dV+kZKXg13O/Kguih5WTgp2Psw9GNxqN4fWGo11wOwDAgbsHkJSVhMVnF2Prja149t9nsf/Ofuy5vQcT90xEYmaiyWz+zpU7w8XBBS2CWqB9pfYmIV/uzt98Y7PZPeb1dxAy7l6/9vAaTseehoPggI6VO0IQBGWcZV7LRhkvcaTPTeOG7zt9j1o+taATdXhvz3smPQv5deXrC/UOVX7uWa2n8vO3R7/N97rGr021oDboyq/oXtHslyZ/V3+DIQQGwTSP5aKMK7SWKqYuDi74rftvStAGDKvPQNEG05K8bjCDqY1sWS4KYFc+2cfUQ1NxPv48hm0YplQmkrOSbVrG5XElZCRg8oHJylhMfZcSLpld2/dk7EkA0odPj6o9DC47HlN4s7ZTslLw58U/lQqXtR5k5n7ox6bHYkPkBiUIyDvx3Eu9Z/Xtyc9Bw/INIUDA6djTWH5pOTqv6IyXN72M+6n3cS/1ns0TqsxJy07D7lu7TSpIObocjN42WqlUywatG4SP9n5UaB+KeXXZatQafNLyEwDA1YdX0WJZCzT8rSFeXP8iXt70Mh5mPCzQfcalx2HVlVUmYeFxLT67GB/s/QCA9DuMTIzEHxf/ACB1rRtX2wAos77nnZyHLiu64Jtj3+Cdne8YBPeEjAR8vO9j5W9mSqspODzkML5tn3cA61KlCwBpcfsOyztg8dnFmLB7gjIzXv8L6pF7RwzeB+Tw2TigsTInooZ3DQDSjHRLzHXl62vs3xhTWk9R7n/VlVUGlyuTn/LpygeAsU3HollAMwyoOcDgfWF95HqD4TXmyNVImVpQG1R5vZ28zV5PJaiUYKgSVMrELiA3mGpFrcnfk/FrTd7dyRwXBxcMrjPY4LQ+eS30vJaMOhlzMs/fkzk5uhwMWTcE/7ft/2x6vyouDKY2crJxuSh25ZM9xaXHYeaxmZhyYApa/dEK43aNQ1p2Gn449YPZwFiYPtjzAf6+/Df+t/l/BuefjTuL/mv643+b/2cw3i0pKwm/X/gdgDTm8Ku2X2FLvy14rvpzAIBDdw8Z3I7xuFNbluJZcHYBPj/0OUZtHYXLDy5j963dOHT3EERRxPab2w26Z/XJIdTfRZpRu/f2XmVmboNyDQDYWDF9FEyre1dHY39p2aTPDn6G+2n3cfjeYYzaNgo9/umBIeuHPHY4nXNiDsZsG4PpR6YbnD/7xGxcfnAZ7hp3/PrMr3iz8ZvKZf9d+w8Tdk14rPu1pmIKAK0rtsYHLT4wOT9Ll4VD9wx/94fvHsbPp3/Oc+xxVk4WnvvvOUzaPwn/Xf3PtkZbIL9evzn2jcH5++7sw+2U23BxcMHcjnPNVuDk8YlA7k4/MkeVI77v9D0AYEf0DuXxVveubnbnIGN1/eri6YpPA5AqhN8c+wYbIjdg2uFpAAwrpunadINx0HLo0e/OVnYtzIg3+BvdfWs3ll5Yiti0WOUxWAqmcrs6Ve4EwHTstTLGNJ/HBgDezt5Y9MwifNzyYzQo3wD/9fkPLQJbIEfMwcubXsb8k/MthjOTYKpSG1RpPZ08Ld7v520+R/PA5pgQPsGgnfqTz4wnQBl/+c/vS9ELtV9QftYPzEBuYSshIwE5uhycizuHrw5/hcTMRNxNuYuWy1pi6IaheGnjSzYtxL8hagOuJV7DsZhjJvdZEjjkfwjpk5eLsnVWfmp2KtKy00rki4CeLMazxBefW6z8vD16O747+R2WnF+CeSfnYceAHcqbX2GTtxA0Hsay5toaiBBxIeECtkdvR3x6PDpU6oCZx2ZCq9PCTeOG1hVaA5DGbfaq1gurrq7Cmutr4Kh2xKetPsWRe0fw7s53ER4Yjmltp+G3c79h3sl5GNVoFF5p8Eq+bdNfs3HUllGISZeCaIBrAO6n3UeDcg2wtMdSkw9NOZj2rNYTi84twq5buwBIVZcqnlUASME0R5djdrFumVanhQhRWTDd08kTnSp3MqkKX3lwBYBURfzr0l/oV7MfPtr7EVpXbI2w8mEYv2s8WldsjfHNxud5f6IoYumFpQCAZReX4a0mb8FV44qYtBgsPrsYAPBO03fQJKAJmgQ0Qf+a/fHF4S+wIXIDDt87jJSsFLg7ulu8fWtYE0AG1hqIiu4VkZiZiP+u/qcEtIVnF6Jj5Y7QqDRIzEzEy5tfBiDN6O9UpRMSMxPxMPMhqnhWwdm4sxAgIC49Tnl+99/Zj341+z1W+/ff2Y+3d7yNIXWGmFwmLx9W2aOyxTGFtXxqoWOljtgevR0v1nkR/wv7H07GnIS7xh0atQaNyjcyOL6yR2XU9K1p9raMCYKA7zp9hx3RO/D2jreV828kS0ML5NeR7HridWVLU3k8sVydA3IDUWZOJlKzU+Hu6I4bSTcwZtsYAMAvZ35R2pvfZ1qXKl2w7eY2k33fbenKN1bNqxo+b/M5Rm4aiZvJNzHv1DwsOrcIKyJWKH+HMnNd+fq8HL0s3k/LCi2V8c/6jIOp/nOgH0Tr+NbBhPC8v9h5OXlhbse5OBV7SpkQJSvvUh6ejp5IykrCmbgzGLdrHO6n3cedlDu4n3ZfWUIrJj0Gt5JvoZKnaaXenBWXpA0UhtUdBk9Hy8HcXhhMbWTrrHxXB1e4OLggXZuOuPQ4VNZULsrmEeU7s3vJ+SUApErW9ujtyt7Uhcm4ihCXHqdUYQ7ePaicL3+Irru+DpceSOPqPmzxoUEI0n+zXnllJTydPLHo7CIA0pi6WmdrYe6JuQCkCmZ+wTQ1O9VgLKscSgHgfprU9Xkm7gwO3TuEp4KeUi5Ly05Tgk6riq2w6NwiZZxWOZdyqOheER6OHkjOSsbxmOMIDww3e/9p2Wl4adNLiE6OVsbneTl6oUe1Hvj66NcApIC2MWqjwXCH9ZHrodVpsT16O7ZHb8dTQU8hKikKUUlR2Hd7H0bWH4k+1fuYDYDGQxauPryKsPJh2HpjK0SICCsfhgG1BiiXezt7Y3rb6Tgdexq3U27jbPxZg+dCn7xzjkwURaUNSVlJuBh/0ez1zFEJKrQNbgsAiAiNwM2kmxi4dqA0LGX9MCRnJxsEnKP3j6JTlU54fcvrOBd/Dq0rtjY7POPyA+tmcBsTRRF7b+9FLd9aWHJ+CdK16fjlzC8ApMkx4YHh+OfKPzh2/xgAoIJ7BYu3JQgCZnaYiaikKFT1rApBENCxckeDY9pUbIO9t/einl89/Nb9N6XL2BoqQYVOlTvh74i/0W+NFMIdBAdodVplwlV9v/o4G39W6f7W6rRKV67+F1QXBxe4adyQmp2KuPQ4uDu6G/ye49LjlGXRzI0x1Sfv+25c0cxrHVNrBLgFYOWzK/HzmZ/x0+mfkK5Nx9YbW9G+Unt8sv8T1C9XH282ftNsVz4gjQM+HXcafWv0tfm+1So1nNROyMzJRLo2HT7IHScqjzv1dfbF8ojllm7CQPtK7dG+Unuz99O6QmtsiNqAlVdWKu9P5tbD/WjfR/i1+68m5+tEHf67+h+CPYIRHhiOmLQY5QuwPJa4pGFXvo3kYKrVicjR5V86FwRB+UBmdz4VB/mDpo5vHfzR8488j917y7YxlsbOx5/H0gtLTbqRohKjzLYpOjna7HJKx2OOIzU7FQGuAQaTGwDpzXlpj6XKaTmUyuRQCsCq7u4Vl1ZAK2oR4hmCkfVHKud3rNRRWVQbAL46/JXB49p1axdyxBxU8qiEpv5NDY71d/WHg8oBbSq2ASAtk2OpLfNPzcf5+PNIzkpWPmh8nX1RzqUcZrWfhW4h3TC60Wh80eYL+Dn7oU/1PlALalxMuIg5J+Yot6Mf8KOSojBp/ySsuLwCoihiwZkFeH7189h3ex90og4Lzy40aIO8s488pleemGNMnsRzKsbyPvZ/XvzT4PSs47OUZYbe3fku3t75tsXr5qeyZ2Wlq/Ns/FmTqtuRe0dwP/U+zsafhQjR4pjhG0k3sODMAryx7Q2TsY760rLTDMbcbY/ejtHbRuOFtS+YzKafED5BGb6hFaXwk1cwBaTwWM2rmsUwNiF8AkY3Go2fu/5sUyjVV8u3Fn7q8hMAaXzj2bizSNemw9PRE11DugKQquZx6XHot7qfMuHOuOfEeHyj8cREOdzm1ZUP5O77Hpsea/A3oczKf4wY4uzgjP9r/H9KVfLIvSP44tAXOBV7CksvLMWk/ZPMrmMKAL90+wXr+65XxobbSq6MG6+eIFdMHVSFU/eTv5j/e/Vfk8vGNBqDl+q9BCD3PVRfclYyxu4ci0n7J+G93e8ByF0arEG5BspKIiUNK6Y2kseYAlLV1MXRcveZrLxLeUQnR3NmPhULebKKn4sf6perj1ntZ+Fu6l1U8qiEN7a/YXDsydiTBhUuayRlJWFX9C48E/IMBq4dCEAKZvIEDAAmExLkN8w5x6VgVce3DmLSYpCclWywe8qklpPMdu2FlQ9Dbd/auJggVd+6VumKQbUHYeSmkQbH6Q9jSMxMxKf7P8XFhIsYUGsAulTpgvf3vK8EjKF1h6JXtV7IzMlEeGA4OlXuhGxdNtK16ei4vCOuPryKCwkXUNevLrQ6LX49J1UjelTtAY1ag0qelZTHKY8lfy3sNey5tQf3Uu/hj4t/4OUGUpdzVk4Wvjv5HZr4N8GyC8tMHl+IVwgAoFOVTuhURRqT1za4LXYM2AFBEHAj6QZOxJwwuZ5KUBmMs/zx9I/wd/XHrOOzAACvb31dqZIBwFNBT+Hg3YOYdXwWjtw/oqwbqr/AuPHzviFqg8GuQMaMJ00tPLsQC88uxJnhZwzCc0ENqzsMUUlRBktpjWo4Cj+d/gmXHlwyWOsTkAJDp8qdlNnncve5/JwcvncY3UK6mXRB30i6geEbhktjnXv8joruFZXXq35VXdamYhuTVS8quOUdTPNT1asqRjUc9Vi3AUgTjzQqDZKzkzF0w1AAUnvbBbfDt8e+RWJmIjos72BwHf2ufEB6/7iZfFPp6tffgQrI7SL3cfZBXrycvODt5I2HmQ9xM/mmskSSPPa4IF35xuRVCeThQ7JNUZtMjpUDo4uDi9lJatZycXDBw8yHJl9A5fczjUpT4NvWJw+5MKdDpQ6o6VMTi85JX9ZTslIMvii8uf1NZYOA2PRYiKKI/belYGqpB6QkYMXURo56wdTatUw5M5+Kk9zdLM827VSlE16s+yKaBzVXurEG1hoIB8EBCRkJStXOWu/tfg8f7P0A/7f9/5Tzrj68anCMcVU0JTsF5+LPYWPURqgFNT5o8QE299uMPYP2YFLLSQCkyos8gcOcl+u/DAeVA+r71cf0ttMRHhiOZ0OfNThGf0mazw99jq03t+JWyi18e+xbdP+nuxJKnw19Fv1q9oOrxhXvN39fmaChUWng6eipdKtN2jcJk/ZNQrPfm+Fc/Dk4qZ2URa+reuYuwC2H8lDvUKV68+/Vf5WK67rr67Do7CL83/b/Q5YuC9W8qsFDk7uWYjWvamYfs/yFoVtINwDSh+r8zvOVD/NdA3ZhYbeFGFx7MAQIiEmLUWaLy+RQ6qhyNJjYpL+YvaWqUcPyDQFI3fWWJldYuwOPNbOvzfF29sa37b/FF22+UBauH91otPJa+em0VB2s41sHg2sPxvedvse4ZuPQsVJHzGo/CwNrDTS4vXRtuhJYRFGETtThq8NfodeqXojPiEe2Lht/XvwTff/rq7yO5TDj4+SDANcAvNv0XTioHBDiGWJQdZLX/7Q3Zwdnk56HntV6opp3NXSu3NnsdeTKpkz+snU/7T50ok75Umi88HxF94r5tke+bf2Kt/x6KmhXvr5Q71C0D26vnA50C1R2TTJWGEEYyB3CYFIxfTT5qaAVb2M1fGoYnP5fA2kyaZ/qfVDTpyYEQVDGiepXTDNzMk2WWjsddxqbbkiv/XaVzPeSlASsmNpIo1ZBrRKQoxORkW3dOFN5tmNJXJaBnjzGwVTm4uCC9X3XY931dXiuxnM4FXsKFxMu4uDdg1aPNToff17pLtWvTsw7OQ/tgtspO6gYr8OYmp2qdMF3C+mmzFB2VDvi+RrPQxRFNA1omueH1DNVn0Hb4LZQq9TKRJ+xTcfi0N1DyMjJQGJmIpKzkqETdUjJTsGWKKnCFuwerIQnFwcXjGk0Bi/WeTHPD6gxjcZgV/QuXHpwSRn76q5xx9Q2U5WhOQNqDcDtlNtoG9xWGRcJSCF16sGpiEqKwomYE6jjV8ek2tmhUgfcTL6pVAG9nCxPwACAwbUHo365+nBxcEFNn5r49Zlf4enkCW9nb4QHhiM8MBzXE6/j4N2DStU41CsUOWKOMhawtm9tNCjfAOOajcOMozMM2mtuZx35Oo4qR6XaZTyxJCsnS1lc39/F36CyaKmLs6AiQiMQERqhnH690evKwvAA0KtaLwyrN0w5PbvjbACGwzvk6umk/ZOw5voanLh/QumG17fqam53/+wOsxFWPgy7b+1GlypdDBZnFwQBU1tPxUf7PkKnyp0MZt7b27hm41DepTzUKjVq+dRSgvy37b/Fx/s+VnYbeqvJW2hYvqFJ9VA+Pf3IdGUlBye1EwJdAw16RMwtTm+sikcVnI49jXG7xqGGTw1U86qmzMp/nK58fZ+0+gQ7l+8EIC131blKZ3yy/xOT4wqri91SV7785Vh/q9bH4aZxQ5/qfRCZGIkfOv8AN40bBtcZbPA3665xR1JWkjIZCgCuPrgKraiFt5M3krOSkSPmYNWVVdCJOrQNbqt86SyJGEwLwEWjRkqmFhnZ1lVM5fXP8lrPjKiwyBNmzK3PV8G9Av4XJn3j7lKlCy4mXMSCMwvQO7S3VZWL1ddWW7xs4NqBmN52OrpX7a5U6fyc/RCfEY/bybeVEKY/rhOQKhj6E2/yYm45lTXPrYFO1OGpZU9BhIiU7BTsvbUXWlGLUK9QfNrqU6U7c07HOVZ1YVX1qoolPZbgnR3v4FbKLdT3q4+ZHWYaVMdaV2yN1hVbm1zX3dEdrSu2xo7oHRi+cTic1c7IyMkAIK1xWcO7BobXG44cMQdx6XF5VollgiAYfJCYC0CtKrRSus5bBrXET12lSuK8k/Nw+N5hTGwu7f3dp3ofnI07i+TsZExrMw3ezt4W71ej1qCOXx2cij2FU7GnTILp3dS7ECHCxcEFjQMaG3SdGi+5Vdhridbzq4eK7hWVoQSWumVdNa6Y2X4mEjMT0a5SO+y/sx8ZORnKOFuZk9oJU9tMxfhd45Xznqv+nDJBydIkmRZBLbD5+c2FUvkrTF5OXnizyZsm5wuCgHebvYvridcR7BGMl+u/bLbtxhVUQOpWHtlgJA7czd1hKq/VIGQNyjfAmutrAABTD07Fwm4LH2tWvjnlXMphSfcl+O7kd3ij8Ruo61cXd1Pv4odTPxgcV1j356IxH0zl5eKC3M1XbAvis9afGZw2/iLp5ugGpBpWTM/ESbtt1fWri3Px55CYmYgNkRsASF/QSjIG0wJw1qiQkglkWNmVL79hlrZdeChXSlYKrideLzFddXmxVDE1NqTOEPxy5hdEJUVh9bXV+PbYt+hTvQ/eafqO2eOzddnKG1uXKl3Mbp+55cYW1PKphZi0GKgEFcIDw7ExaiMO3zsMnahDFc8qeY6ZKgi5ciHPkk3KTMKO6B0ApNmuYeXD0C2kGwQIaB7Y3Orbre1bG6ufW43snGybl3nrFtJNaYMcSgFg2tPTDD5Ufuv+m023m5chdYYgNj0WsWmxeLfZu8r5oxuNxmiMVk57OXnh63ZfW327Dcs3xKnYU1hxaQUiqkUgTZsGlaCCi4OLsv1qRfeKJq83W7ZmLaingp7CyisrAVgOpgDQuUpu9/XEFhPx7bFv0bVKV9T1q6tsdzu97XR0rNzRIJjK3ab5KWmhND8+zj5Y1tN0rLM+c130r4e9jqeCnsIHLT7AF4e+QHXv6lbdX/+a/SFAwOeHPsfRe0cRmxb72LPyzWnk3wi/dP1FOW2uamm8XFRByV35xmvSyquiWBpKUBTknaz0g+nhe9KSeE38m+BG0g0kZiYiTZtmMEmzpGIwLQBnjfTCTs+ysmLq/qhimnLL5okmZH9p2Wl4Yd0LiEqKwqJui9AssJnV173+8Dp8nX3zrEoZi0+Px6WES2ga2NRgvTx991LvYUPkBkSERph8e5bX2vRyzrt72E3jpkwS+WjfRwCkiSs7o3diVodZJmPJjt0/hoSMBPg6++Krtl+he3R3JKQnICI0Ar+c+QU/n/kZW25sUQJrywotlXV85QH4libZFAZPR0/Epsdi3+19ykzjDpU7QCWoMKPdjHyubZ5GpSnQJIZnQp7B7ZTbOBV7Crtv7QYgdXVb6jIvDI5qx3zXTCyIgbUGYtnFZTgZexL77uzDh3s/hAABLSu0VMbRVXCvgBZBLfDXpb+U691ONQymBR1jmpdXGryiVInNVfjM6Vujr0H1M8gtCM4Ozsrs5/ebv48FZxbgm/bfWL0u5JOopo/hGqpTWk3B08FSdX9QrUHwd/VHfb/6Vt2Wg8oBg2oPwuprq3Em7gz239mvbPBhad3XwmDuC2VhBVO53VtubMHRe0eVyVRyxTS/FRoKkzzhSe7K14k6JZi2CGphUEQYVGsQAtzyH35hTwymBSAHU2vHmFb0kL55pman4mHmw3xnMZL9ZOuyceTuEQS5B6GqV1WIoohP9n+ijNM7fO+w1cH0UsIlDFw7EDliDr56+it0C+kGtUqN/Xf2w8/Zz2zl8H7qffRb00+pes7qMEuZmKPvvd3v4XjMcSw4uwALui4wuC15kl1+FVNAGpdnvHf29cTrWHd9Her41cHx+8fxTtN34KBywOUEaS3IpgFNoVFpDGbhD64zGD+f+dngdt4Pfx/rI9cbnFe/nHUfZAXh5eSF2PRYTD00FQDQIrCFspxPcVOr1Hg17FXk6HLQ5e8uiE2PRYugFnZpy+Oq7FkZjf0b48i9I/hw74fKqg/6r5uK7hXRuXJnTG87HRN2S+F46sGpBrejv995YQn2CMaaPtKGDQWdbGI8HGNInSFmF9Eva3ycffBfn/+w/NJyuDq4GoxDFwTB7PtSfsIDw3Em7gyO3j+KzVHSl8de1XoVVpNNmFvKypqhB9aQg6nxEmVy5d4eFdNVV1bB38UfsemxSMxMhKuDK+qVqwcnh9wCR6/Qonu+CwuDaQG4KMHUuoqpk9oJ/q7+iEmTdmdgMLW/jVEbsSVqC15v+Loy61EURfzf9v/Dvtv74CA44PvO3+NcnDSTXGa8g4pO1EGAgHRtOpKykhDoFoiTMSex9vpabLmxRVlS5b097ykLdF9LvAZvJ29s67/N5MN0U9QmJZQC0o4yxh8AFxMuKgskJ2YmYsbRGfi5qxQK99/Zr2w/aM1SKC2CWqB5YHMcvncYFdwqoGPljvj9wu/48fSPyjHtK7VHeGC4MqM2xDPE5HbKuZRD2+C2SnVwSJ0hCPEKMfhg8NB4PPbuO3mp41tHWR1AgIDPWn9WaOPJCkqtUmPlsyuxPnK9xdnQpUEtn1o4cu+IEkqNVXSvCEEQ0L1qd/xw6gdcT7xusIyVh8YDXz79ZZG0TaMunGV5yFQ1r2p4v/n7hXZ7TQOaYuHZhdgUtQnp2nSoBbXFjSgKg9lgWsgVU2PykD15ImhxkB/n8ZjjeG3ra8r5zQKbQaPS4G5K7jbJdX2Lr10FxWBaAM4a6cPO2mAKSN35MWkxiE6ORoPy9qniPAkSMxPx0d6PUM61HCY9NalAwyIO3DmgjCM7FXsK//T+BxnaDPx8+mdlCR2tqMVrW3L/wLtW6YrNNzYrS6YAwImYE/hgzwdIyU6BAAEPMh+gimcVk0XAZfKEIEAaBzp0w1B82/5bg7Fce25LC5M7qKQdW/RnWQJSF37/NdJOTWHlw3A27iwO3j2Im0k3UdmzMjZGSiH6+RrPW7WMi4PKAQu6LQAgBfOj948q+9XL5DAiV43lNTeNyXt969NfYmh6u+n5Lsb9OJoENFEmWLSp2KZQJx88Dh9nn1JfgdOvPBuvnQoYjkc0/h3vHLATPs4+dv+SQPbXyL+R8kUeyN2YoqiY25WqsIKppWFWgLSebXEuXq+/TJ4+eak0/TXUS8NQQr5TFIAyxtSWYOqRO86UDImiiKkHpyJiVYTZRcT1fXvsW+y8tRN/X/4bYb+FmazTZk5mTibG7xqPqQen4sO9H+LVLa8ql91Pu482f7RBpxWd8OclaQebZgHNDKqCg2oNwsQW0ozmWym30H9Nf0w9OBUjNo7ArZRbeJj5EA8ypXGd+qG0vl99fNfxO5wZfgaDaw82adf5+POYf3K+stvMZwc+U8bLDagpzVI33slDv2t8YvOJygxzuaorX1+/m91agiCYHQOalJUEURSVaqS5iqkl1bxz1+eUF8EuKu0rtYe/iz9Uggov1X+pSO+rrOka0hX9a/ZHDZ8a+LHLj/i6reHkKf1gKn+BAaSlhPxc/BhKCYA0Dlx/7GpRd3cXZVe+/pDprf22ollA7hCvsc3GFs59WEl/ya4dA3bg67Zf49iLx5Rl7OTPA3k95JKOFdMCsHWMKZDbrcolo0zturVLmTQx5cAUrOptfstArU6LbTe3GZw3/cj0PGeXHrl3xGR3IEDaGaVvjb74eN/Hyg4kgLQw86iGo1DZszL23N6Dqp5VlUkR7hp3pGSn4GLCRaVy6uHogfp+9aESVMq6nh0qdcCsDrMMPozfbfYuavjUQJuKbeDh6IGnlkmBcvet3dh2c5tBZdRZ7ax0AxnvKiIvcfNS/ZdQv1x9tAuWlr85E3sGKVkpysD7gq6n6O7ojg19N+Be6j18efhLXH5wGclZybj28BoSMhLgrHZWdm6xRpuKbfBZ68/wVNBTRR5OyrmUw+Z+m5GRk1GkldmyyEHloGyEAEirVOjTn+iRoc1dhaB/zf5F3zgqVfrV7IfPD30OQCoaFCVzk5+Mq/0F1bd6X5yNO4uX6r2EALcAfNjiQ3x15Cu81eStIh1Lb84rDV6BSlBhUO1BKOdSDs9Ufcbg8klPTULrCq1Nzi+pGEwLgBXTwqW/1d3Vh1eRkpUCd0d3g2PStel4b/d7SMxMhJvGDWObjsVnBz/DmbgzuJd6D4FugYhLj0NcepwSnLJysjBu1ziD21EJKnzV9it0q9INGTkZ+HjfxwCkb5SvNHjFYFyQ8Yeqn4ufQYBsVaEVpj09TRkznJyVDHeNu9muEke1o8H4yj97/olB6wYplVZ97zd/XxnMrn9/OlGnVIi7h3QHIO2xLh8nL27uofF4rGAW7BGMYI9gNA9sjssPLhssv9TYv7FNk0xUgsrqxfsLg1qlhpuKobSoGb++9DcImN52Or478R361+pfpGOKqXQaVHsQdt3ahb239xZ5Bc/cOFDjL1UFFeIVgoXdFiqnq/tUV8b6Fzc/Fz+MDx9v8XJvZ288X/P5YmzR42EwLQCXAo4xBbiWqTnyum+yCwkX0CygGTZFbUJdv7r45cwvyk4sGpUGH7b4EBGhEfjnyj84F38OR+4dwdYbW7E9ejsAaZmTDG0GOlbuqIyPHFhrIFpXaI1yLuWUMb4uDi6Y12kebibfxAu1X8i3oidPAKroXhFLui9RlkKS6e8Ikx/j8Z+vhb2GNxrn7mMvd8nrd+XfTr6NlOwUOKocUd1HWj9QrgikZqcqC5obt6ug5G3u4jPileffeJtDKpsEQYCHoweSs5JNdv7pXKWzwbqhRMbmdJyDg3cOonmQ9esKF4TZYJpdOMGUig6DaQE42zgrH8itmN5PvY+snKxC20f3SSDv3CI7fv840rXpGL/b9BvgZ60/U8JRTZ+aOBd/zmRv8MsPpGWNFp9bDEBaJNvcDigAlHX5rPF/jf8PTfyboFPlTjYvuG7My8kLbho3JXg2CWhicLm8MLR8ebYuG/9c/QeA9M1cXltTf79meWvIwgqmctBee30ttDotyruUR49qPQrltqn0+7z159h8YzOromQzjUpj03tvQXk5eWFEvRHQ6rTKpE55y14quRhMC8DW5aIAaWtGFwcXpGvTcSvlFqp5Vcv/SiWMKIo4H38eVb2qPnYwk29PEARlh5hnQ5/F6mur8dPpn0zGAfk6+2Jw7cF4JiR3jIzxriM9q/VEVk6WyY5EXUO6PnZb5Tbo79X9OARBQCWPSspYVePFrN0cc4NpanYqXlz/ojL5qLF/49zjNLnHyRVTa/autoank1Qx1eqkvcQjQiMKtNg8PZk6VO6ADpU72LsZRHmSd0GTg6nxey2VPAymBeBUgDGmgiCgmlc1nIs/h6sPrpa6YJqhzcCUA1Ow5voa9KzWE5+2/BQH7hxAY//G2HZzGxoHNLb6MWXmZGLinok4dv8Yfun6C+6l3QMAvFjnRWy5scVg72EHlQNeC3sNr4W9ZjJ2U3+AebOAZviyzZcQBAE6UYdvjn6DtdfXYmjdoTZN1ilOXo654/JM9j5+VDFNy07D7lu7lVDaq1ovjGo4Kvc4vWAqr19a3qVwu/IBqTI7tO7QQrldIqLi9m/vf3Hs/jE8G/qsvZtC+WAwLQAXZUtS22b31fKthXPx53DpwaVCq+IVh8ycTAxeP1hZXH7d9XW4k3LHYGmnGj41sDJipVVrpK24tEKpas4/NR86UQcfJx/U9q2NRc8swt+X/8ax+8fg4+SD+Z3nW6zONvZvjOF1h2Pt9bWYED5BuW+VoML48PF5DgYvCV5u8DIO3TuE9pXam1wmT/7SilrMOCJtpzmi3giDPdCB3DGmado07LgpTVAqrAWr6/rVha+zLyq6V8SE8AlFup0mEVFRCvUOLZLdx6jwMZgWgLuTFExTM7U2XU/uQpC3diwNEjMTMfv4bFx5cAVqQa3sZGS83uiVB1dwPuE86vnVM7mNaw+vYVPUJrSp2Ab+rv6YfXy2cpkcUGv61IQgCKjnVw/1WprehjmCIGBc+DiMCx+X/8ElUMsKLbHy2ZVm1/JzcXCBg+AArahVZtubC7D6C0g/yHwAT0fPQptQ4O/qj50DdpaKBZmJiOjJwGBaAJ4u0ji7pIxsm65Xy0faz/zSg0uF3iZbxKXHYeHZhYhLi0NcRhyeCnoKI+qNMJiQ9SDjAeadnKcsOg9IFbvuVbvjpY0vITk7GS/UfgENyzfEX5f+womYE1h9dbVJML2UcAnDNgxDmjYN80/Nt9imktrdXtQsjXdSCSq83fRtzDgqVUsD3QINxpbKXBxcIEBQ1mJtHti8UMeBMpQSEVFxYjAtgIIG05q+Ugi5m3oXiZmJBmv/FYccXQ6+PfYtfjv/m8H5R+4dQXJWMtKy09C7em80KNcAUw5MwdabW5VjWgS2wEv1X4KXkxd+6/4bdt7aiRfrvAhnB2d4OXlh1NZRWB+5HuOajVP2rr6YcBETdk9AmtZwkfgA1wBMCJ+Ad3fldksX51qXpcXwesPRr2Y/LDizAC0rtDS7nJUgCAYbBLQIalGcTSQiIipUDKYF4OksBa/EdNuCqaejJyq4VcCd1Du4lHCpyNZwi0+Px1dHvsKVB1cwqNYgDKwt7Zd76N4hg1DaO7Q3IpMicTr2tLK00tabW9GlShcllLYIaoFxzcYZVDSr+1RX1tEEgJZBLeHv4o+Y9Bjsvr0bnSp3wrrr6/DB3g+gE3Uo51IOy3osw5m4MwhyC1LWEe1xswfWR65Hx0odDW6Pcrlp3CwudWUOAz4REZVmDKYF4OUiPW1J6baNMQWAsPJhuJN6B0fuHyn0YHrk3hFsjtqMLTe2ID4jHgAw9dBU1ParjYblG2LPrT0ApPUp53aci6YBTbHt5ja8veNt5TYSMhKU7UGb+DfBj51/zHdvYbVKjS4hXbD0wlIcunsIFd0rYurBqdCJOrQNbosPWnyAIPcgBLkbjqUcHz4eLYJaoEdVro1ZGN5p+g6cHZzt3QwiIqICYzAtALlimpyRDZ1OhEpl/Ti8lhVaYmPURuy/sx9jGo0plPasvb4Wv5//HefizynnlXcpD0e1I26n3Mbhu4fhpHbCissrAEiL1Mv7vzco18B8O4Na4qeuP1ndhoblG2LphaXYELkBa6+tRUp2CkK9QjG7w2w4qMy/zMq5lEPfGn2tvg8yb06HOTh87zCXcyIiolKPwbQA5DGmOhFIzdLCw9n6ySYtg1oCAM7GnUVSVpLBWpHWyMrJgggRTmonXEq4hI/2faQs0i4LKx+Gqa2nYnPUZnx38jssu7gMP53+CZk5mXi64tPoUCl3UWx/V3/M7jAbe27vQXZONv679h8A2Lyvrrym6MPMh9Jpv/qY22muxVBKhYcLnRMR0ZOCqaEAnBxUcFSrkJWjQ1KGbcE0yD0IVb2qIjIxEofvHrZpT+msnCwMXDsQyVnJGFp3KL4/+b3BYvRD6w7F203eVmbXh3iFAJBm4QNAHd86+KLNFyaTaDpW7oiOlTvinyv/KMFUP7xaI9g9WFneCADea/4e170kIiIim5hO86V8CYKQOzPfxglQANCqQisAwP47+62+ztUHV5VtKe+n3ceMozOUUPpinRdxfOhxTAifYLDkk/5OTGHlwvB7j9/h7ext8T56VuuJvjX6Ymb7mQa3Yw1BEDCywUgA0gLvjfwb2XR9IiIiIlZMC8jTxQFxKZk2z8wHpO78pReWYt/tfdCJOrPLAOk7HXsaIzaOQLbO8L5eC3sNoxuNtnj96t7VMbrhaCRkJOD1hq/nGzad1E6Y3GqybQ9Gz6thryLYPRidqnQq8G0QERFR2cVgWkDyBKiCVEzDA8PhpnHDndQ72HJjC7qFdDN7XEJGAjZGbsScE3OQrcuGr7MvBtUehJZBLZGclYw2FdvkuQC6IAgY1WiUxcsLm5PaCc/VeK7Y7o+IiIieLAymBZS7yL7tS0a5alwxtO5Q/HDqB/x27jeTYKoTdVh+aTk+P/S5cl6wezD+6PlHnl3xRERERKUZx5gWkKezvJap7RVTABhUaxA0Kg1Ox53G2bizAIBsXTb+vvw3ev/b2yCUdqrcCct6LmMoJSIioicaK6YF5FXAbUllfi5+eCbkGay5vgZfH/kaU9tMxUd7P8LxmOMAAA+NB14JewXP13i+2LcuJSIiIrIHBtMCyp2Vb3tXvuzlBi9j281tOB5zHD3+kXY/ctO4YXTD0Xi+5vNw07gVSluJiIiISgN25ReQPPmpILPyZaHeoVjQbQFCPEMAABXdK2JOhzkYVm8YQykRERGVOayYFpCny6MxpgXsypfVL1cfK59diejkaFTzqpbnLHsiIiKiJxmDaQEVRsVU5qh2RKh36GPfDhEREVFpxq78AvJzkxarf5CaZeeWEBERET0Z7BpMd+/ejYiICFSoUAGCIODff/+1Z3Ns4usuBdN4BlMiIiKiQmHXYJqamoqGDRviu+++s2czCsTPzQkA8CAtCzk60c6tISIiIir97DrGtHv37ujevbs9m1BgPq4aCAIgilI4LefuZO8mFQ9RlP7h0f+izvDnnExApwPEHCAnC0h/AKidALUDoM0CdFogM1n6362c9L+ju/R/dhoAAVCppdvKSgUcnHNv37QxpmflaIGMRMAjANBmSrcpPPr+JagBlYOF6wqAgyOQozdmWFABuhxAmyGd7+QhXV+nffS482iLSXvzuTwrVXruXHwftfHR8yk/r8rzrMv9HcjnO7pKt6HNAlQq6foqB+m5y06Tfs5Oe3SfAiA8eryCYPi/3A4xB1BppN+ZoHr0+9QBjo9WihBzHj3XWbn3BUFqvygCao10vC7HsJ3yY3B0l+4zJ1s6RuP86LWjlR5LThagdpR+XzlZ0vOv00rHuvjk/j6Mn1f959TBUXr88vOl/9wBufcvt1dQSfehzZSOFVS5z41OC7iW03sMj66vzcxtf3ZG7vNSnMz+XRTbndvpbu31mMva4wXK3mMua48XQN0+0udGCVKqJj9lZmYiMzNTOZ2UlGS3tjioVfB20eBBWjbiU+wQTDOTgeR7QEYSoE2XPhgzk4C0eCArRQqDGYlAyj3pPPmFL+qkD92k29J1HByl8OjkJR0vf+Bmp0kfuoIKgPgo2BiHMSIiIiq1JvW2dwtMlKpg+uWXX2Ly5Mn2bobCz93pUTDNBOBRuDcuikDiLeBBFBB3GYi9JIXJrFQg4Trw8Cbs9u3OKvKyV48qZKIohWBRJ1WfIAKp8YCDkxSkBbVUoVNpcgOws6cUngUht+ppzf06uQOpcYDGRaryZaVKFThAqn4BhlVCuZqWnQ5oXA3Py9FKFUFnT6mdok5qq6B6dBuWHrf+WVYcp3aUqoVpCVLlTVDnPm6levfoPo3Pz0yWfpafX50ut9KocZG+YDi6Ssco1UP5MepVEwHp+RLUjyqUjyqK8n1lPJR+P4B0Wu0otVWnffT7dX70HGfnXsf4H0QgK036X6WRvqlrMx9Vs9XS70qlflRt1UmvDwen3PtNi5faqHbUe171nku5GyMn89HtGj13+r9z+TWRky3dl9pB+kInCIbVaojSlzeVJvcxiI+ur9ZIz7N+G4ubXZeYs9N92+0xl7XHC5S9x1zWHm/JU6qC6cSJEzF27FjldFJSEipVqmS39vi5OeIqCnEC1MObwMV1QPxV4MoW4OGN/K/j4psbVLyrAB6BUhBMTwBc/QBnb6n7U6UCnDylD9IcLeBZQQot2kzpQzXjoRTKnL2k/508HgUZnXT7GtfcsCR3cRr8/CgAyN3waicpsKg10nlERERE+ShVwdTJyQlOTiVnLKfcfS9VTAtIlwNc3gQcWwRc2yFVm/SpHYHg5oCrL+AeII2drNAECAyTgqcSEEsgdal6eREREZGdMTk8Bl+3x1gyKjMF2D8XODgfyEzMPb9CYyDkaSCgHlAn4lGlsoQGTyIiIqJCZNdgmpKSgqtXryqnIyMjcfLkSfj6+qJy5cp2bJl1/AqylmlOthRGTyyRxo7Kwv8H1HsOqNyyxM2QIyIiIioOdg2mR48eRYcOHZTT8vjR4cOHY/HixXZqlfX8CtKVv24scPw36WcnT6DteKD5/6TxnkRERERlmF2Dafv27SHadY22xyNvSxqfYkXF9NoOYNVrQMp96XTrt4CnxkhjRomIiIiIY0wfh5+1Y0yvbAX+elFabxQA2owFOn9SxK0jIiIiKl0YTB9DgKe0buO9xAyIogjB3CSly5uB1f8nhVK/6sDA3wH/OsXcUiIiIqKSj8H0MVTwdoEgAOnZOYhLyUJ5D6OlrE79KXXfA4BvNeD1vRxLSkRERGQBp38/BkcHFYIeVU1vJqQZXpj+ANj4vvRzwxeAkZsZSomIiIjywGD6mIJ9pS0sbz3QC6Y6HfDni1I4LVcLePY7wL28nVpIREREVDowmD6mYB+pCnrrwaOJTboc4Pe+wI29gIML0Gced0AiIiIisgKD6WOSJ0DFJj9ay/TyJuD6DunnlqOB4GZ2ahkRERFR6cJg+pjKP1pkPzYlU+rC3/21dEGVNkD7iXZsGREREVHpwmD6mOSZ+LHJmcCl9cCd44CjB9BvIaDW2Ll1RERERKUHg+ljkoNpYlIysHu6dGb4y9zRiYiIiMhGnJXzmORg2jvlLyD1FODsDbR43b6NIiIiIiqFWDF9TP4eTvBECoZjjXRGxCzAM8iubSIiIiIqjRhMH5OHswbDXPbBTchEhm9toG4fezeJiIiIqFRiMC0EvdX7AQCRVQYAgmDn1hARERGVTgymjyvhOmporyBHFHDco729W0NERERUajGYPq4zKwEA+3X1cCXF2c6NISIiIiq9OCv/caQlAPvnAABW5bTBw4Q0OzeIiIiIqPRixfRxXNsOZCYhzTMU/+raIJrBlIhKGFGrtXcTSgRRFJG8YweS1q9HdkyMvZtTKHKSkpB93/bHImZlQRTFImhR6aHLykL6mbPQPngAURSRk5IKAMi8cgXa2Fg7t86QmJOj/Jxx/jzuTZmC9JMn871e+qlTuPnaa7j+bG/c//rrImxh4WLF9HFE7QEAaKt1hi5GhegHaRBFEcITOAEqOyYGgkYDBx8fg8eYffs2Hvz5F7Qx9+HVpw9cW7QAAOiSkqBLT4fg7AwHH5/ib+/du0j49TfoUlMgiiIC3nsPKnd3ZF66BE1gINTe3gW+7ZykJKg9PQuvsUVM/hBSOTnlfZwoQszIgMrFpZhaVvqIOh1S9x+AS6NGULu7KednRUfDwc8PgosLMi9fhmOVKhAzMiA4OiJmxjfQZWTAvX07QCfCuU5tpJ8+jfQTJ+BYLRS6lGR4Pfcc1B4eULm6Flpbk7dtw8PlK5CyaxcAIPCzKfDs3sOg3fnJSUyENiYGjtWrF/r7Wvb9GAgOajj4+Zneb0oqsm5EQRsbC7WbG1zDw5XLRFFE9u3b0AQFQRsXBzE9HXDQICsqCi4N6kPt5QVRFJFx7jxSdu1EzoOHSD91ChlnzuTegVqNgIkT4fviEIP7FbOzkXrwEFRurnCqVg2iKNrl/Ss/OcnJiJ05Cw///htiVha8ej+LgI8+Qk5iEqJfew0enTvD/523Da4j6nS4/8WXSFqzRnoPK+cHr5694P/ehNz38/sxSD99Cq5NmkDt61tsn2VZ0dFIXLMGgkaD9FOnIGZkSp8fDg7IvHZNeoy9eiFg4vtIPXAAGRcuwqlmDThVrw7HSpWktt+5g/hfFkDl4QHfoS9C0GiQtGEjxBwtvHr3gcrFGQ+WLkP23bt4uGIFdCkpAADB1RUqjQY5SUlwKFcO2thYCM7OqLZ2DRyDg00+01P27kPSunXwH/eu2dduXgw+O+/dQ+revfDo0gXZ9+5BUzFY+dsUs7KQuHYdNMEVkXHmDGK+ngEAcG4YhsxLlyFmZODBH38CoggHf3+4NGoEl7AG8HvlFQCA9sEDPFy+ArEzZyr3nXn5Mjw6dYZTaDWovbwK+JsqHoJYir82JSUlwcvLC4mJifC0R1CY0wRIuAbtwD9Q8zcROhE4/EEn+HsWzlhTUadD+okTcKpRI98gJIoiMs6cgaZCBTiUK2f1fWgfPEBWZBQ0QYHQpaXBKTQUAKDLzET27TtQOTvh9th3Tb6defbsCa8+fXDn/feREx+vnK+pVAm65GTkPHyonCc4OcG9bVt4dO2CnAcP4d72aYhaLZyqV7fcrvh4aOPj4VStGjLOnkVOcjKyIqPg9WwEVB4eENRqk+tkRUcjfsECpB0+gqzr1w0uc23RApqgICT++y9Urq7wefFFuDQMQ+aVq8g4dxbamFg4N2gAn8GDIWgc4BAQgOxbt3D3o4+REx8Pv1dfhdrbC8mbtyDxv//g1bcvAt5/DxkXLiJh0SK4t28nvbH4+uDep5MhqFSo/OtiqJzNvxZ0aWlI2rQZDuXKwa1Na6QdOoz4RQshOGig9vRETmIiyr8xBs5160qBMTsbKkdHAEDcDz/gwV/Lob17FyoPD2gCA+E/YQLcn26j3L4oikhcuRJidjbif1kAbWwsHKtWhfvTbZB16zYginBr0xqOlavApXEjiGlpuDPxA6QeOgTfoUPh0aE9XBo1Mvmd5CQkQO3jA5WbG8TsbEClgjYmBjHffIu0I0fg3qY1Aj7+GGoPD0CttvmDTdRqITiYfl/WPngA6HQ2fxDIz0VWZCRUTk7QJjxA6r69cKxSBR7dugE5OXj499/Ivn8fmoAA6e8nIADJ27bBtXFjOIaG4u6kSci6chVefXoj/exZpO7eA5W7O9zbPg2H8v5Q+3gjdtZsqNzdlQ87qNWAXpXDGip3d3h06wrnunXhXLs2XBo3RsrOnci+cxcenToi7dhx5CQ+hKBW/397dx4fRX0/fvw1OzN7ZrO5SEIgQFAOBaQKnmAtitSKba1VqSJgrfW2qK0X+vtqsRZqv9rWqrTetdovVsVq1VrxwlpQNIAipyj3kZB7s/fOfH5/TLIQE45IQhJ9Px/kkTD7mdnPzmd39v15fz4zQ+OCd0isW4dKJrHCYfScEL5hw7GjUSL//e8+n8t9yCHo2dlkffNE8i+9FM3VevAstW0bn3/3e9iRCO5DDqHX9J9h9umDZph4Bg9q0bapigqsunpcPi+RRe/hHTrEKet2tzp2qVSKHXfeSd3fnwHbRvP5MEtK8AwahMvjxjtsODvvvx+7vj6zTtHNN5E9cSJoGpW/+x31zz7XtNNcYNsttm/270dq46b92ueBE07AO/IIrNpazKIiGl6bT2LVqszjmmmSe/55pLZtQ6UtUlu3oufnoRJJguPHk/fjCwGI/Oc/6Pn5eAYObNWxs+rqaPzPu+i5uSQ//wzQMPuUUPf885jFvfEOG0bjW2/h7t+fnEnnkli9mm233ErWmBMovv32zP5TSmE3NmI3NrL9lluJLFy419cW+sEPSO/cSc5ZP8AsLaVi1mxiS5a0Kmf26YPm82IWFRP94ANUMgmA+9BD6PWzn5E9YQLRJUuJr1hB6Pvfy9THTiapmzuX6kceJV1RgX/0aIzevdGDQVI7duAZPAi7IUzs449xl5VR+PPrMAoKWn2+07W1bDj7HFJbt+67wTQNdgtZXMEg/R5/jLpnnqFu7tN7XK3wxhvRs4Nsv+XWfT9HZseYBI4/jsSatRTecD3Zp59O3d+fYcdttwHgOewwimfc3KLT9EV1zz1H9cOPYMfjuDwekhs2AGD06oVVV+ccQ3fjHT4cPSeHyLvv7rVq7gEDMtvaXf+/PkHDv16l4V//wqqtBUDPz2/xHd1MLyig9P778I0cudfn6ijtidckMP2y6rfC7w4HzQU3buDk+5byeVWEx358NOOGFLZrU3YsRmThQpIbNhJbtpRUZSW9rrqK7bf+P9IVFWhuN1nf+haaYRBfvRojL48+v7uH8JtvEf3wQ4KnnEzDv14l/O9/o3k8eA491MkaJJNkfesk8i68EM0wnB6oaZLcvJnavz5J47vvktq8ucUHPThhAjln/5Ctv7geu6Fhv1+DKxRCRaOtPmh74x02jPxLLyHyn3eJLV9OurISdBeB446n8a23dn3J74lhQDpN4KRvkt65k8TKVa2KBE44YZ8H8M7i8vsJnnoq8TVrSKxeDYDvG98geOqp1D0/j+S6zwAwS0pIbdvWan09J4fShx5i2y9+QXLjRsA5oLU1zOQdeQRlTz+NnUhg1dZS+9RTVD/08IHVPxDAfcghlM55gNS27Wy56irSFRW7FWgdFOy+3OX3k3XyybgHllHw05+imWaLYlY4TGzpUqIffIhnyBDiK1dS+9RTBE4ci5GbS3L9BpJbt5Levj1zGbbQ97/v7M9Vq9D8fsxevdACfryDh1B8+224vF7sSITauXNJbd9BcMKp1Dz+FxrffLNVNT2HHwa2yrRNZzOKijAKC0msXr1fnxPNNNv1edqT0A9+QP3zz+/x8dzzzyNdVe1kitxujIICGt9+GzsS2eM6nsMOo+CnFxP8zneouv8Bqu67b4+voffsWYQmTgSckYwNk37kfNb35QtBSEfI/+nF5P/kJ2z6ycXEV6zY81N7vah4fJ/b8wwahOZ2t9hWYMwYguNPwaqvd7K277yDSiT2r4Jf+EwFTjyRkrt+Q+2TT1F1//2tihdceSV2YyM1f/nL/m3fMCi68Uayxo2j5tFHMlm3vdUncNxxu46hmobZty8qnXY+l19C0YybyZs6lfiaNdTPm0fNX57Y9aCmUXDllZglJU4QHo/TuGABeihEbMmSFgmPPWkraMudOoX4ipXEysszywquuJzcyZOpfvAh0lVVeA8bilVXh9XYuNcgd3ea283gRQtxBXaNQiTWr2fLVVdjVVVh7da5+rL0vDw8gwbh7tcPPSeEd9gwgqeeSvSDD6maMweVTBJburTNdQNjxpB/6SU0/PMl6p55ps36D5j7f3gPP/yA67kvEpgeDB/NhecvhZKj4JK3uGbuUv6xbBvXjh/M9PGD9nszqcpKNp4/mdSWLZ1WVc/gwaQrK/frQ90mTaPX9J8BEPvoYxJr16IZBiqdxjdyJL2uuxZ3377E16yl5q9PYBYWkTd1CvG1a6l/bh4qlcSORDNDip1G0zB6F5Peth3/McdQ/P9uxTNoENWPPErl//4vZr9SCq+9joaXXyI8//XMaoFvnkjw5JOpvOu32NEvzBM2DCdw3ORkYIyiItIVFZilpU5Q38Q7bBhWbW2bAWZ7BE76JkZe/l4DCdh1sEpXV2UC3MKbbqT6kUewdla1KGuWlpJ14omgaSTXf05k4SIAPIMOJVW5M5OZ0vPyMIqLSG3c1CIo2Z8Ayd2/PwVXXM7OBx5oM1vlys7GO+xwvEOGoufmUvf00we8r9riP/ZYkps2tf2lqWmgafiOOIL4ypWZzBDsOeD/IrOkBM000TweXNlB0tt3OFm0UIiCq64itmwZnkGHkn3GGejBINHycsySEvTcPIz8POdzk0qBpjnDlJ+vx8jLxaqvJ7J4MbEPy6l/4YUWz+kKhbDr69H8flQigWYY5E25gMSn60DXKbjsUsJvvknNY06G3ijsRda4k8n61kn4jjwSTdOwGhuJf/wxFb+5C5VMknP22ey8/37UF9/vbeyzPnf/L7GPPqbmb3+DL7wPWuy3po7iFwVO+ib9/vxnADZdcgmRd5wpUIU33ACahkomSVdU4AplUz3nTwDknHMORTNuxm5spPJ/7yb2ySckP2vqyPXvh6YbqGSSwuuvJ2vsGGIrVuAbOZLYRx+RrqgktW0bwQmnoodCGHl5gDNPr3mkxU4mafjnS857pbLSmau5dSt2QwP5l11K1tixNLzyCspWpDZvQiVT2MkE4X+9ekABs3vAAFIVFahYjMAJx6Pn5BJ+/XVUMokrENhrZ2B3nsMPo+iGGwgcdxwANU88wc57/4hmmhT/z//Dqq8n9vFyUlu3Ev3wQ7AsfCNH0nvWLDwDyzLbia9dS/2857EjEayGBlxeL6Ef/IDU1i1UP/Rwm1m5LzJLSnCXlbWZqfeNHkXOmWdS89TfWmSiQ2edRf28eS3Klj70IP6jjmoR5O3OTiaJLv4ATXcROP54rMZGtlxxJdHFiwHwH38cve+4A6NXLxrfeYfk+g0k1q6l4aWXyP7ed4l/9DHJjRspffDP+EeN2uPzgDP6tvH8yaR37nSO9Tt2ZN77gbFjKbjiCjaef37mNXrKykhu2IhZUkLi00+Jr1zZYnv5l12KHW7E5ffj+8ZIjMJC7EgU7/BhWNXVmH36sGX6NTS+8QZ6bi7e4cPp/atfYRbtX5Kr4dVX2XrNtQBoPh8ls2YRGDsGPSsLcKZoNLz0T4ITJhAtL6du7tPEli0ja/wp9L333jZHTDqaBKYHwz+uhGVPwpjpcOpMHn13PTNfWskpQwt55MI9p/abKcsisug9Ku+528n0mWarg75n0CDyL/4JnsGDCc+fj+Z2k1y/nvoXXtxVZuhQEqtX4wqFKLr5JlQqRXLdOjSvDxWPteyNNtFME/egQ8mbfAGBE8eS2ryZxnf+gysQYOc99wAQPHU8Jb/9LfEVK9BDob0Ou+8vKxwmsXo1dc//g+Cp4wm/+m/nS1jXKb79NryHHU5y/Xoa334b74jhmCUlNC5YgHfIUPyjR2H06kV89RqS6z9H83ox+/QhVl6eGYbNnTJlj/PB7EgEze/PDD8q2ya1aRNGYWFmXl+6tpbEmrVOtjYWI3fy+c7jPh/Vjz5K1pgxmWEPZdukt29H8/kyX3wAsY8/Rs/Lp+pPcwi/+u9WWd/gd04jvvwTPEOH0Pu224ivXk3jW2+TNW5ci6H4+hdeYNuNN2X+n33GGfiOGIHmduM55JDM8JFSinUnfatVBso9cCA5P/wheRdOa3PaQzNl28Q//tjJjjbNI2yen7f12mtbBN8AZt++9P71nahEEnfZABJr1xJfsZLcCyZj5OZihcNUP/gQ1Q89lFlHc7tbBIG7aw66mnlHHoFnQBlmnxKUZZOuqCC2fDn5P70YT1kZW6ZfQ3rHDmefnP4dsk46ieTGTdQ8/niLToWekwNKYdXXY5aWUvKb2fiOPBKUQnO5SG7YwI5f/xqrppbCG67Hf/TRpDZvxqqrI/HZ52Sf9m0aXn6Z6Ifl5JxzNv5Ro7ATCTRdbzUUqVIpZw5v01SLA9U8gtLw8itknzGRrHHjsMNhXMEg2LYzD7iNL1Vl2+36grGTSTaeP5n4J58ATqfL5fXiP/ZY7GiU4PhT8A4fjrtvXwAS69aRWLsW//HHs/Oee6h75llnQ5pG4S9+Tt60aSQ3bmT92eegYrHMSIDvyCPpe+8fSFVUsuHsswHo+8D9BE8+uVWdklu2omJRPINad+6TW7bgCgS6zZzP+KpVVD/8CFZjmJyzz8bdpw/pnTuJr1pN41tvAeAdMYLs00/HXTaA1MaNeEeOxKqpwaqvxzNwIOBMm0LT0ICqhx8mvX0HeT++kMi7/6Xi17/OPJ+7f3+Kb78NNA3/scfu9zQZq74eOxLB6N27XVNr7Hicuqefpm7e8+jBIH1+dw/pnTsJv/EmiXXrcA8YQM4Pz8Ldrx/gfA7StbXOvODsbIz8/Myx1WpsJPzv16h/4YVMIPlFQ1etbP/UH9sm+uGH6KEcvEMGt3q87rl5bL/lFgInnuhMCautpezFF/AObl32i6zGCHYkgllUSMP8+VT86k78Rx9N71/ficvtZsP5k9ucGtEs/6cXk66spOCqqzLzYPf6WtJp0pWVmCUl+yzbal3LYsOkHxFfs4bev/wlOWf9YN/r2LbTIdrDdLOOJoHpwfD7EVC3CSY/B4PGU76xhh/OWURBlocPbjlljx+wyHvvUf/8P4h++OGuOTWGQf+/PuFMtLYs9Lw87HB4j/Pp7HgcOxxGKYVZWEhyy1aMXgWtTm5Rtk3VfffT+PbbxFeuRHO7KbzxBnLPOQetjS9RpRTh1+aDUgTHn9LmXL+OFl2yFFfAj3fIkE5/rq4QW/4J9S++SO6556B5PJmD+L6kq6v5dIwTqHoGD2bgiy/ssWz1449TOfs3gJPBKrzxRrJP/84B94JT27ax7uRTAGdKRMlvZqPn5+/XdpMbNlD7f/9H3kU/IV2xw/kCr6vDM2iQM7zl0vAcOoi8yednMkXJLVv2+T5wev4vkTPp3Ew2ACC+Zi1bLr+c1LZtGL17U3r/fbgPOYTYRx/hHTq0R52sdjCpZJL4qlV4hw1r1+ddJZNs+uklxJYupWjGDHJ/NCnzmFVXR/TDD9F8Pjb/5GL0ggLsSMQ5SQkInHA8/R59tMNfy1dRfM0aZ55uTk63CcgPhFKK6Pvvs+Wqq7EbGwmddRZmcRH+444jcMwxHf584TffYssVV+AdPpz4qlVgWRy6YMF+ZyL3Zss11xJ+9dXWD2gaRTNmkDflggN+jvaw43FU2mrXyY0HkwSmna12A/xhJLgMuHEjeLKIJS2G3/5vLFux8KaTKclpOQFepVJUzP4NtU891WK5UVxM39//rtWJJkJUP/oY4TfeoODSS8j65jf3WE7ZNg2v/AvvYUMzJ691lIZXXqHu2WcpnjkzkznrzuxYDM3t3muWWHQMZVkoy9pjpjj2yYpMhnR3vWfPIufMMzu5dqI7S1VUUPf3Z8g970ftOlm3vaJLlrLx/PPRc3IyU9mGLFvaIVnC6scep/I3vwFNY8jSJWgeD4m1n2L2LpaOcBvaE6/J5aK+jA1Nc2lKjgKPk7XxuXUGFWaxekeYldsaWgWmO/94XyYoDZ52GqEzv5+ZTyNfoqIt+Rf9mPyLfrzPcprLReiMiZ1Sh+zTTyf79NM7ZdudQS51dfBour7XY5eeHWxzeXD8qZ1VJdFDmEVF9Lr6qk5/Hj03ByATlGpud4cNXeeccw6a7iL47W9nttnWdALRfhKYfhk7m87i7XNUi8WHNgWmn1c1AkWZ5VY4TM0TzlzPwIkn0ueu37Q5lC6EEF8VrmDrwNQsKem2Q43iq+eL16vuyOt36lkB8qZO7bDtiV0kMP0yajc4v3PLWiwe2MvJnn5W2fLMyvDrb6DicdwDB1L64J+/khfgF0KI3e0+Bxh2XXNSiIPli0PqrpAMsfcEEph+GZnAdECLxYf0cjIBTsbUodJpah59BIDsiadLUCqE+Fr44nVrD33jdZl7Jw4qTddbXJNWz+7edzwSjs6/eNVXjVJ7DEwHFTpDV6u3h7Ft55yyumefI/HpOvRQiLwLDu5ZekII0V1IUCq6wu63+TUKD/xsfNH5JDBtr3g9JJruiJTT8tpkg4uy8Jouwok0n1dFUMkkVX9yLhhdcOWV3f7+tEII0ZE8g5zrH/e69touron4uto9MDWLi7uwJmJ/yVB+e4Wb7ijjDYG75SR+Q3cxok+IDzbU8vGWOgr++yHpHTvQexWQM+ncLqisEEJ0nZK77yaxZi3ZnXTVCCH2ZfebUZi9JTDtCSRj2l4NTbdRzO7T5sNDip3h/M8qGqh+xLlXed7Uqa0ufi+EEF913sGDCX33DJlbL7pMi6H84t5dWBOxvyQwba/mjGmw7Td4WYFzJqq54A2S6z7DlZVF7o9+dLBqJ4QQQogmLYbyJWPaI8hQfns1NAWm2W0HpgMLAnjSCU587XEA8qZNQ2/jen5CCCGE6FzabhfUN2SOaY8gGdP2aqxwfu8hY3pIryxO3PYx/mQUepdQcPllB7FyQgghhGhmR3ZdV7wzb38qOo4Epu3VfEa+t+0z7Pvl+zm7chkAK75xEpohSWkhhBCiK1gN9Zm/NZeEPD2BtFJ7JZounu9pe3g+vnYt/beswULj7QHHHMSKCSGEEGJ3dn1DV1dBtJMEpu3VnDF1Z7X5cO1f/wrAwpIRLImabZYRQgghROdzl5Xtu5DoViQwba9E2PntafsuJtHyJQDM7zeabfVx6mOpg1UzIYQQQuym98xfkv3d7zLg2We7uipiP0lg2l6ZwLT1UL7VGCG5fj0AqUOGAPDW6sqDVjUhhBBC7GKWlNDnt3fhGz6sq6si9pMEpu2VbJ5j2nooP7akHJTCKC5m7DFOYPr6qoqDWTshhBBCiB5LAtP22kPGVKXTVD/k3OkpeMopHDswD4BPttYjhBBCCCH2TQLT9rDSkIo6f39hjmndc/OIfvABmttN7uTJDC9xLie1oTpKQ1zmmQohhBBC7IsEpu2RDO/6e7ez8hvfeYcdt90GQK+fXY1nYBm5ATeleT4AyjfUHtRqCiGEEEL0RBKYtkfzNUx1DxjuzOKavzyR+Tvn7LMzf48bUgjAv1fsODj1E0IIIYTowSQwbY9UzPlt+jKL0tXVRN57D4CBr7yMnpOTeey0Yc59eV9bWUHasg9aNYUQQggheiIJTNvDSji/DU9mUfi118Cy8A4fjmfgwBbFjynLI8dvUhNJsvCz6oNZUyGEEEKIHkcC0/awks5v3RnGV0pR+8wzAGSffnqr4obu4vsjSwD445ufHpw6CiGEEEL0UBKYtke6ZWBa85e/kFi5Cs3rJfSDM9tc5Ypxh+LS4IMNtWytix2kigohhBBC9DwSmLbHbkP5SilqHv8LAAVXXoGRm9vmKkXZXkb1dx57faVcbF8IIYQQYk8kMG0PK8XOT7JY8+d6Vo84gvSOHWgeD3lTpux1tVMPLwJgvgSmQgghhBB7JIFpOyQ2bqHqk2zshIJ0GoDA2LG4vN69rnfq4c7Z+e99Xk1FQ7zT6ymEEEII0RNJYNoOyU1bAXB5XQROOAGzpISiG2/Y53plBQGOHpBL2lbc9eqazq6mEEIIIUSPJIFpOyS3OEPxWYdk0e/RRzjk9fm4+/Xbr3V/MWEILg2eW7KFheuqOrOaQgghhBA9kgSm7ZDcthMAd75zgX3Ntf+779iB+Uw5rj8AD7+7vuMrJ4QQQgjRw0lg2g7JHc5F8t0F/i+1/rlHlwKweH0Nlq06rF5CCCGEEF8FEpi2Q+E5x9P7mFr8A/O+1PpDi7MJegwaE2k+2VrfwbUTQgghhOjZJDBtB1//XHIGxjALsr/U+rpLY+ygAgCeen9jR1ZNCCGEEKLHk8C0Pb5w56cv4ydjywB4pnwL/5WToIQQQgghMiQwbY/d7vz0ZY0ekMd5x5SiFPzimY9Ipu0OqpwQQgghRM8mgWl7WM0ZU/OANvM/ZwyjV9DD9vo4977xaQdUTAghhBCi55PAtD0yQ/lfPmMK4HPrTD7Wuf7pfW+tY8mm2gOtmRBCCCFEjyeBaXtYBz7HtNnVJw9i7KHOiVAz5i1nZzhxwNsUQgghhOjJJDBtj+bA1DjwwFR3acw6awS9gh5W7whz2ZPlKCXXNhVCCCHE15cEpu3RgRlTgNI8P3+/9Hi8povyjbX8/JmPJDgVQgghxNeWBKbtkW4abu+gwBSgrCDA9d8eCsC8JVv5y8INHbZtIYQQQoieRALT9sgM5R/YyU9f9JOxZdxw2hAAZr60kvKNNR26fSGEEEKInkAC0/bo4KH83V1+0iGcNqwYW8F5D73P/y3eJMP6QgghhPhaMbq6Aj3KpCed4XzT1+Gb1jSNO84cTl0syXuf13DzvOV8sL6GWT8cgcfQO/z5hBBCCCG6G8mYtoc7AP68TglMAXoFPTx18XFc3HTb0nlLtzLk1ld5c3VFpzyfEEIIIUR3IoFpN6O7NG4+/TB+furgzLLLn1zC7+avZXNNtAtrJoQQQgjRuTTVgycyNjQ0EAqFqK+vJzs7u6ur0+E+3lLHRY9/QFWjM7fVbbg4d3RfJh/bn8N6f/VerxBCCCG+etoTr0lg2s3ZtuKFj7by5HubKN+469alp48o5idjyzi0MEjIZ3ZhDYUQQggh9kwC068gpRRvr9nJo/9dz38+rcosd2nQL8/P4SXZ3HTaYfTL93dhLYUQQgghWpLA9CuufGMNM19axUeb61osN1waRdlevtEvh1OGFlKa56dfnp+ibG/XVFQIIYQQX3sSmH4N2LaiNpqkfGMtd/17DesqG/dYtjjbS47fZHifEIOLshhUGCQv4GZYSTaGLue/CSGEEKLzSGD6NaSUYtX2MHXRJAvW7mTxhhq21saoDCf2uE7Qa5DjNynI8pDjMxlQECA/4KasIItBRVnk+t0EvQam7kJ3aQfx1QghhBDiq0ICU5GxrjLMJ1sb2FAdYW1FmHjKZltdjPVVERJpe7+2obs0CrLc5Pjc5Ge5GVIcRCnI9hpk+0zyAm4CHgMNMHUXQa+BpmkEPDo5PjfFIZlKIIQQQnxdSWAq9ikcT7G2opFNNRHW74yggNU7wsRTFg2xFJ9XRWhMpOmId4fu0nDrLsoKAsRSFn1zfeT43UQTadyGiy21MUpyvPTN9aMBtgJbKUrz/OT6TdK2Ittr0CvoIZq0sBVkeQy8pot4yiLba2LqLkzDRe9sLwrY0RAnP+DGshWRZJqQz5Q7aAkhhBBdQAJT0SHiKYtY0iJp2VQ2JNhaF2XFtgaSlo3h0gjH09RGU6zZ0YDX1NFdGtGERTieIpxIE46nu6TeHsPVZjbYZ+oYLg3TcFGS4yWZtqluTBJNWuQF3DQm0qQsm/wsN72yPFRHkgTcBkGvQa7fjcsFybTCsm1s5dypy6WBUpC2FW7dRU00SWHQQ5bXwLYVNZEUOX6TaDKNpml4DR2P6cJr6Fi2TV0sham76B3yUhzykrJsXJpGbSRJjt+Nx3DRmEgTT9v4TR1D10hbiryAm0gyTV7ATSJlo2kQ9JpkeQwsWxFPW+yoj6MBWV6DoNe5pFhDLIXh0ijM9pJIW0STFi7NeV2GrtE/z4+lFPWxFJFEGp9pEPKbxJIWtdEk2V6TeMrCpWkYuobh0tCbfmqjKQqy3Cjl7BOFavrtTDVp7lCkLJuGeIqQzyTX78ZSimRTe0UTFknLwmPoxFNOJyQ3YJJM2yjlXMs3bSt8pk7A43Q0lHIy9S4NLFuRshQp20bZkO0zUAriaQu/27kDc2MijVt34Tb2b361UoqGeBpT1zB1F4ZLQ9O0zGONiTRZHiOzrC2WrVBKyZxuIcTXkgSmossppUikbVZub8Ctu3BpGltqo0STFo2JNNvqYvhMPTMVoDKcoLIhTkM8jeHSSNuKLbVRbKUwdRfVjUlqIkmCXie4CMfTVEcSJNM2frdBPGWhcAIAIZoVZHlIpCzCTcGj19Spakxg6hq5fjd2U8BsK6djEfAYaBrURJJYTZ0NhfP/3TUHqbZSxFM2BVke0raNBngMnWjSGQ0wdRexlEV9LIVL09A1DZ9bJ8tj4Hfr1Dd1TNK2TdBr0hhPZ4J2pZwgWndphJo+J9WRBD5TJ20p3IYz99tr6vjdOpqm8VnTSZBuw0XKcjosvbKckQZPU31M3YWhO6MYVZEksWSaHL8zuhBNWsSSaWIpi4DbIC/gJt7UgQGnE1DYNHKhaTifbZdGPGWRTNv43Dq2gkTKwm24SKZtNE2jLpqkV9BD2lJNdQUNp0OTtm0aYmk8pgutqe6RhPN5zvI4r8u2FbGU04G0laK6MYnbcMpXNCQozPbgNXU8TZ2NgNugIZ4iN+Am129SFU7iMV3UNHU2E2mLkhwfjYk0kaaRG4/hdHZqoymnjV0aLk0jnEijN3U6NI1M3Zv+oWkaSil21MfJ8bvxmi68pk44nqIhliaRtsjP8pDlMaiLJkk2dT5Tlk1+wENJjpfNNTG8ptPpCvlM6mMp4ilnn3sMHa+pE09b1EdTFGV78bldpC1F2lbEkha5AZOAx0DXNLbXxwl6DQIeg53hBCnLxtSdzrqugc+tY+oulIJE2iIcT+NrSiy4Ddeu1wcYukaOz000laYhlsKyFTl+N1XhBNWRZKaNi7O9FGZ7nA6mBpbldMLC8RR5ATc1kSSRhPM+cxsuttbFnH2Qtgl6DaJJi6JsL+F4im31MYDMez6SsNhQHcGlafTK8uD36KQsRZbH+fx4TZ2A26AxkcZrOu2Ytm2iSYugx/m+6BX0YCuIJi3iKec1p21n/7ua2tSlNbep1mKZhtO5bk7IJFI2eU0d8ETaJs9vYjXty53hBJat8DYlQUI+k+31cbK9Bh5Tx7Z3HW92/dC03Pm7uRNrKYXhcpHjN9laGyM34EYDcgPulh3/pg5v0GtQ2ZDIHDMCTR1xy7ZpTFgUZjuv1fl8OvvdVoqLTxzY0YfdNklgKr6W4iknq9cYT1Oa5yfSlGmsiyabDiDOl1o0aVHREM8cjK2moCRtq0xgWx9LkeMzqY2miKUsosl0JjNn6Boo2NnonFiWSFl4mgIeXdPQdY1k2vkyyPYa1ERSBL0GLk0jnrZIpGziaSuTJczyGFQ1JqhoiGPqTkDRK+ihPpbKBC55fjfhRBqlFLpLY2c4Qa7fOeDrLidICidSRBKW83+XRlHIS31T/V2ahq0U2V6TlO1kwH1uJ6BJpGySlo1bd7GjIY6pO8GOUhBNOgc5bbcDdI7PbAooVFOG0iZtORnXeMrCcDV/uZFZj6aDfEM8hcdw4XcbrYI9IHPgNHUts2/qmoK65rbRXZp0QIQQ4gAZLo0VM799UKa5tSdeMzq9NvvwwAMP8Nvf/pbt27czbNgwfv/733PiiSd2dbVED+Q1dXqHfBDa9X+APjm+VmWH9wkdzKr1GGnLbjHc3NxvVao5W3RgV2ewbYWr6QoPdtP8X1N34TFcmW3btkJrClB3D0Kd7BuZTGU0YZG2nfpatiJtO8G1obswdWe9Vdsb8Bg6/fP9VIYTxJLOHOdw3Jlq4nLtyookLbspY2g50x88Bom0TcqyObQwC8PlImk5/09ZNqm0QuFkkD7b2YjfrePSNKJJiyyPQdq2nSyiqTtzqpuC/HjKJpJIE0mm0XAyhmnLGRkI+UyCXiPT6clpuqtbXSzF9vo4vbI8WLbC1DXqmqZlpCybSMIiZdkcUpiV6dw0t9uW2ii5ATeW5eyjlKUyr8FwuTLTQgyXhs/tZHK9hk5FQ5yUZeN16/hNnbSt0HA6ZFkeg3jKJpG2MHQX7qY2ac4WpS0bBQQ8BpZtE/KZVDQ42d7m0Q2lIGXZKJwOU/OUi0TKJuAxcGkQSaZxaU7Hy2fq1EadzFthtrdpqolOUchLRX2c6kgSj+GMzkSSToa8JpKkNpokx+dMGfG7daoakxgup0x+wE2WxyRt2yRSdlNmG7J9JpatqIkmQUFxyOt8LpTK1F3t9jc461Q3JtBdGknLJttrku0zSTRlzC3bmbIS8BjEUhaFQQ8VDXHqYynyszyA896uiybxmjrZTVdESVp25rbUJSEvO8MJ4mkLXdPwNGXLayNJYimLlKUyU2uSaZssj5M5tZs6tC6NTEdUKWfaU8hv0hBLoWnOOpkun1IkLJuGWIqA2znR1VaKSMKZlqS7NHqHvHgNnfXVESIJp62ap/kkUhb+pqxt0Ou8r2wFsaRFaZ4/85lrjKfJ8ZvsDCfwuw0Kgm5nNCGaoqoxid/jvP/C8TTZPpPGRBpPU1Y96DWIJNKkmrKMkYTlZJNthd7UmdVdGlWNCZRypjtpmnNciSTSmRESmqYc2bbTprZSLdq4uW1iKQujadqSrynDHU04mW1d18hp2kc+09nn9TFnKley6Tji0jRcTe2ga06H36Vp6E3HoebHmvdjOO6MXjRn0ZVysuTQ3PGHlOVkWxvjaQqyPHhN5xjYnHGH5tdroZRyMvwoqhqTlOb6iCftbnf+RZdmTJ9++mmmTJnCAw88wJgxY/jzn//Mww8/zMqVK+nXr98+15eMqRBCCCFE99ZjhvKPPfZYjjrqKObMmZNZdthhh3HmmWcya9asVuUTiQSJxK7rcjY0NFBaWiqBqRBCCCFEN9WewLTLThFNJpOUl5czYcKEFssnTJjAwoUL21xn1qxZhEKhzE9paenBqKoQQgghhDgIuiwwraqqwrIsioqKWiwvKipix44dba5z8803U19fn/nZvHnzwaiqEEIIIYQ4CLr85KcvnkzRPDm3LR6PB4/HczCqJYQQQgghDrIuy5gWFBSg63qr7GhlZWWrLKoQQgghhPjq67LA1O12M2rUKObPn99i+fz58znhhBO6qFZCCCGEEKKrdOlQ/nXXXceUKVMYPXo0xx9/PA8++CCbNm3isssu68pqCSGEEEKILtClgemkSZOorq5m5syZbN++neHDh/PKK6/Qv3//rqyWEEIIIYToAnJLUiGEEEII0Wl6xHVMhRBCCCGE2J0EpkIIIYQQoluQwFQIIYQQQnQLEpgKIYQQQohuQQJTIYQQQgjRLUhgKoQQQgghugUJTIUQQgghRLcggakQQgghhOgWJDAVQgghhBDdQpfekvRANd+0qqGhoYtrIoQQQggh2tIcp+3PzUZ7dGAaDocBKC0t7eKaCCGEEEKIvQmHw4RCob2W0dT+hK/dlG3bbNu2jWAwiKZpnf58DQ0NlJaWsnnz5n3e61V0T9KGPZ+0Yc8nbdjzSRv2fAezDZVShMNhSkpKcLn2Pou0R2dMXS4Xffv2PejPm52dLR/EHk7asOeTNuz5pA17PmnDnu9gteG+MqXN5OQnIYQQQgjRLUhgKoQQQgghugUJTNvB4/Fw22234fF4uroq4kuSNuz5pA17PmnDnk/asOfrrm3Yo09+EkIIIYQQXx2SMRVCCCGEEN2CBKZCCCGEEKJbkMBUCCGEEEJ0CxKYCiGEEEKIbkEC03Z44IEHKCsrw+v1MmrUKP7zn/90dZUEMGvWLI4++miCwSCFhYWceeaZrFmzpkUZpRS33347JSUl+Hw+vvWtb7FixYoWZRKJBFdffTUFBQUEAgG+973vsWXLloP5UgROe2qaxjXXXJNZJu3XM2zdupULLriA/Px8/H4/3/jGNygvL888Lu3YfaXTaW699VbKysrw+XwMHDiQmTNnYtt2poy0X/fyzjvv8N3vfpeSkhI0TeMf//hHi8c7qr1qa2uZMmUKoVCIUCjElClTqKur67wXpsR+mTt3rjJNUz300ENq5cqVavr06SoQCKiNGzd2ddW+9r797W+rxx57TH3yySdq2bJlauLEiapfv36qsbExU2b27NkqGAyq5557Ti1fvlxNmjRJ9e7dWzU0NGTKXHbZZapPnz5q/vz5asmSJWrcuHFq5MiRKp1Od8XL+lpavHixGjBggDriiCPU9OnTM8ul/bq/mpoa1b9/f3XhhReq999/X61fv169/vrrat26dZky0o7d169+9SuVn5+vXnrpJbV+/Xr1zDPPqKysLPX73/8+U0bar3t55ZVX1C233KKee+45Bajnn3++xeMd1V6nnXaaGj58uFq4cKFauHChGj58uDrjjDM67XVJYLqfjjnmGHXZZZe1WDZ06FB10003dVGNxJ5UVlYqQC1YsEAppZRt26q4uFjNnj07UyYej6tQKKT+9Kc/KaWUqqurU6Zpqrlz52bKbN26VblcLvXqq68e3BfwNRUOh9WgQYPU/Pnz1UknnZQJTKX9eoYbb7xRjR07do+PSzt2bxMnTlQXXXRRi2VnnXWWuuCCC5RS0n7d3RcD045qr5UrVypAvffee5kyixYtUoBavXp1p7wWGcrfD8lkkvLyciZMmNBi+YQJE1i4cGEX1UrsSX19PQB5eXkArF+/nh07drRoP4/Hw0knnZRpv/LyclKpVIsyJSUlDB8+XNr4ILnyyiuZOHEi48ePb7Fc2q9nePHFFxk9ejTnnHMOhYWFHHnkkTz00EOZx6Udu7exY8fyxhtvsHbtWgA++ugj3n33XU4//XRA2q+n6aj2WrRoEaFQiGOPPTZT5rjjjiMUCnVamxqdstWvmKqqKizLoqioqMXyoqIiduzY0UW1Em1RSnHdddcxduxYhg8fDpBpo7bab+PGjZkybreb3NzcVmWkjTvf3LlzWbJkCR988EGrx6T9eobPP/+cOXPmcN111zFjxgwWL17Mz372MzweD1OnTpV27OZuvPFG6uvrGTp0KLquY1kWd955J+eddx4gn8OepqPaa8eOHRQWFrbafmFhYae1qQSm7aBpWov/K6VaLRNd66qrruLjjz/m3XffbfXYl2k/aePOt3nzZqZPn85rr72G1+vdYzlpv+7Ntm1Gjx7Nr3/9awCOPPJIVqxYwZw5c5g6dWqmnLRj9/T000/z5JNP8re//Y1hw4axbNkyrrnmGkpKSpg2bVqmnLRfz9IR7dVW+c5sUxnK3w8FBQXout6qd1BZWdmqNyK6ztVXX82LL77IW2+9Rd++fTPLi4uLAfbafsXFxSSTSWpra/dYRnSO8vJyKisrGTVqFIZhYBgGCxYs4N5778UwjMz+l/br3nr37s3hhx/eYtlhhx3Gpk2bAPkcdnfXX389N910Ez/60Y8YMWIEU6ZM4dprr2XWrFmAtF9P01HtVVxcTEVFRavt79y5s9PaVALT/eB2uxk1ahTz589vsXz+/PmccMIJXVQr0UwpxVVXXcW8efN48803KSsra/F4WVkZxcXFLdovmUyyYMGCTPuNGjUK0zRblNm+fTuffPKJtHEnO+WUU1i+fDnLli3L/IwePZrJkyezbNkyBg4cKO3XA4wZM6bVZdrWrl1L//79AfkcdnfRaBSXq2VIoOt65nJR0n49S0e11/HHH099fT2LFy/OlHn//fepr6/vvDbtlFOqvoKaLxf1yCOPqJUrV6prrrlGBQIBtWHDhq6u2tfe5ZdfrkKhkHr77bfV9u3bMz/RaDRTZvbs2SoUCql58+ap5cuXq/POO6/Ny2b07dtXvf7662rJkiXq5JNPlsucdJHdz8pXStqvJ1i8eLEyDEPdeeed6tNPP1VPPfWU8vv96sknn8yUkXbsvqZNm6b69OmTuVzUvHnzVEFBgbrhhhsyZaT9updwOKyWLl2qli5dqgB1zz33qKVLl2YuY9lR7XXaaaepI444Qi1atEgtWrRIjRgxQi4X1V3cf//9qn///srtdqujjjoqczki0bWANn8ee+yxTBnbttVtt92miouLlcfjUd/85jfV8uXLW2wnFoupq666SuXl5Smfz6fOOOMMtWnTpoP8aoRSrQNTab+e4Z///KcaPny48ng8aujQoerBBx9s8bi0Y/fV0NCgpk+frvr166e8Xq8aOHCguuWWW1QikciUkfbrXt566602v/umTZumlOq49qqurlaTJ09WwWBQBYNBNXnyZFVbW9tpr0tTSqnOycUKIYQQQgix/2SOqRBCCCGE6BYkMBVCCCGEEN2CBKZCCCGEEKJbkMBUCCGEEEJ0CxKYCiGEEEKIbkECUyGEEEII0S1IYCqEEEIIIboFCUyFEEIIIUS3IIGpEEL0QG+//TaaplFXV9fVVRFCiA4jgakQQgghhOgWJDAVQgghhBDdggSmQgjxJSiluOuuuxg4cCA+n4+RI0fy7LPPAruG2V9++WVGjhyJ1+vl2GOPZfny5S228dxzzzFs2DA8Hg8DBgzg7rvvbvF4IpHghhtuoLS0FI/Hw6BBg3jkkUdalCkvL2f06NH4/X5OOOEE1qxZk3nso48+Yty4cQSDQbKzsxk1ahQffvhhJ+0RIYQ4cEZXV0AIIXqiW2+9lXnz5jFnzhwGDRrEO++8wwUXXECvXr0yZa6//nr+8Ic/UFxczIwZM/je977H2rVrMU2T8vJyzj33XG6//XYmTZrEwoULueKKK8jPz+fCCy8EYOrUqSxatIh7772XkSNHsn79eqqqqlrU45ZbbuHuu++mV69eXHbZZVx00UX897//BWDy5MkceeSRzJkzB13XWbZsGaZpHrR9JIQQ7aUppVRXV0IIIXqSSCRCQUEBb775Jscff3xm+cUXX0w0GuWSSy5h3LhxzJ07l0mTJgFQU1ND3759efzxxzn33HOZPHkyO3fu5LXXXsusf8MNN/Dyyy+zYsUK1q5dy5AhQ5g/fz7jx49vVYe3336bcePG8frrr3PKKacA8MorrzBx4kRisRher5fs7Gz++Mc/Mm3atE7eI0II0TFkKF8IIdpp5cqVxONxTj31VLKysjI/TzzxBJ999lmm3O5Ba15eHkOGDGHVqlUArFq1ijFjxrTY7pgxY/j000+xLItly5ah6zonnXTSXutyxBFHZP7u3bs3AJWVlQBcd911XHzxxYwfP57Zs2e3qJsQQnRHEpgKIUQ72bYNwMsvv8yyZcsyPytXrszMM90TTdMAZ45q89/Ndh/A8vl8+1WX3Yfmm7fXXL/bb7+dFStWMHHiRN58800OP/xwnn/++f3arhBCdAUJTIUQop0OP/xwPB4PmzZt4tBDD23xU1pamin33nvvZf6ura1l7dq1DB06NLONd999t8V2Fy5cyODBg9F1nREjRmDbNgsWLDigug4ePJhrr72W1157jbPOOovHHnvsgLYnhBCdSU5+EkKIdgoGg/ziF7/g2muvxbZtxo4dS0NDAwsXLiQrK4v+/fsDMHPmTPLz8ykqKuKWW26hoKCAM888E4Cf//znHH300dxxxx1MmjSJRYsWcd999/HAAw8AMGDAAKZNm8ZFF12UOflp48aNVFZWcu655+6zjrFYjOuvv56zzz6bsrIytmzZwgcffMAPf/jDTtsvQghxoCQwFUKIL+GOO+6gsLCQWbNm8fnnn5OTk8NRRx3FjBkzMkPps2fPZvr06Xz66aeMHDmSF198EbfbDcBRRx3F3//+d/7nf/6HO+64g969ezNz5szMGfkAc+bMYcaMGVxxxRVUV1fTr18/ZsyYsV/103Wd6upqpk6dSkVFBQUFBZx11ln88pe/7PB9IYQQHUXOyhdCiA7WfMZ8bW0tOTk5XV0dIYToMWSOqRBCCCGE6BYkMBVCCCGEEN2CDOULIYQQQohuQTKmQgghhBCiW5DAVAghhBBCdAsSmAohhBBCiG5BAlMhhBBCCNEtSGAqhBBCCCG6BQlMhRBCCCFEtyCBqRBCCCGE6BYkMBVCCCGEEN3C/wcvjnkVqrDRsgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_train_loss(history, 'Fine-Tuned EfficientNet-kPiece Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03390fa6",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48701107",
   "metadata": {},
   "source": [
    "#### On Train Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0594069",
   "metadata": {},
   "source": [
    "Now, after the model is being trained, we will look at the performance of the model on the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cf7ded38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 09:08:58.339620: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-10-08 09:08:58.396581: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "475/475 [==============================] - 9s 17ms/step\n"
     ]
    }
   ],
   "source": [
    "train_preds = model1.predict([fine_tune_data_train_4P_red, fine_tune_data_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f2a28269",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pred_labels = np.argmax(train_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "768775b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 4, 4, ..., 4, 9, 9])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pred_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "eff52a3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3336"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(train_pred_labels == train_labels['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fd403fa",
   "metadata": {},
   "source": [
    "Looking at the misclassified images to get a better idea about what features the model is unable to capture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "8f594a33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3597</th>\n",
       "      <td>3781</td>\n",
       "      <td>./Data/train/train/3781.tif</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12255</th>\n",
       "      <td>12896</td>\n",
       "      <td>./Data/train/train/12896.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14569</th>\n",
       "      <td>15333</td>\n",
       "      <td>./Data/train/train/15333.tif</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                        images  label\n",
       "3597    3781   ./Data/train/train/3781.tif      0\n",
       "12255  12896  ./Data/train/train/12896.tif      9\n",
       "14569  15333  ./Data/train/train/15333.tif      7"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_train_data = train_labels[train_pred_labels != train_labels['label']]\n",
    "mis_train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "2ea8f949",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fscore</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998978</td>\n",
       "      <td>0.999488</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.998926</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999463</td>\n",
       "      <td>930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998905</td>\n",
       "      <td>0.999452</td>\n",
       "      <td>913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998951</td>\n",
       "      <td>0.999475</td>\n",
       "      <td>953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.997957</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998978</td>\n",
       "      <td>977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    precision    recall    fscore  support\n",
       "0    1.000000  0.998978  0.999488      978\n",
       "1    1.000000  1.000000  1.000000      925\n",
       "2    1.000000  1.000000  1.000000      925\n",
       "3    1.000000  1.000000  1.000000      947\n",
       "4    0.998926  1.000000  0.999463      930\n",
       "5    1.000000  1.000000  1.000000      962\n",
       "6    1.000000  1.000000  1.000000      998\n",
       "7    1.000000  0.998905  0.999452      913\n",
       "8    1.000000  1.000000  1.000000     1005\n",
       "9    1.000000  0.998951  0.999475      953\n",
       "10   1.000000  1.000000  1.000000      916\n",
       "11   1.000000  1.000000  1.000000      955\n",
       "12   0.997957  1.000000  0.998978      977\n",
       "13   1.000000  1.000000  1.000000      943\n",
       "14   1.000000  1.000000  1.000000      928\n",
       "15   1.000000  1.000000  1.000000      945"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = score(train_labels['label'], train_pred_labels)\n",
    "pd.DataFrame({\n",
    "    'precision': precision,\n",
    "    'recall': recall,\n",
    "    'fscore': fscore,\n",
    "    'support': support\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6e51475",
   "metadata": {},
   "source": [
    "#### On Train-Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "14a7854d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25/25 [==============================] - 0s 3ms/step\n"
     ]
    }
   ],
   "source": [
    "test_preds = model1.predict(fine_tune_data_test_red)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "d585fa70",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred_labels = np.argmax(test_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "9a77dc3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11,  8,  6,  7,  1,  9, 13,  9, 14,  1,  6,  0,  7, 14,  1, 11,  0,\n",
       "        9,  6, 12, 12, 14,  3,  9,  6,  0,  3,  0,  3,  3, 11,  1,  9,  2,\n",
       "        1,  8,  6,  7,  1,  9,  6, 12, 14,  8,  5, 14,  3, 11, 13,  4,  0,\n",
       "       10, 12,  0, 11, 14, 15,  6,  9,  2, 15,  9,  2, 11,  4, 10,  8,  0,\n",
       "        6,  3,  4,  4, 12,  0,  8,  6,  7,  0, 12, 11, 11,  5,  4, 12,  9,\n",
       "        8,  5, 13, 13, 13, 10, 11,  4, 10,  1, 10, 11,  8, 13, 10,  9, 11,\n",
       "       12,  9,  1, 14,  8,  2,  5, 10,  7, 10,  9,  4, 12, 14, 12,  1, 10,\n",
       "        6,  8,  1,  0, 12,  1,  7, 10, 12, 15,  8,  3, 11,  6,  5,  6,  9,\n",
       "       14, 10,  7, 14, 10,  8, 11, 12,  8, 10,  8,  4,  2, 13,  0,  0,  4,\n",
       "       14,  4,  9,  4, 14,  7,  7,  8,  4,  4, 10, 13,  8,  2,  8,  8,  8,\n",
       "       13,  6,  3, 11,  7, 12, 10, 14, 14,  8,  6, 11,  0, 11,  6, 13,  4,\n",
       "       14,  1,  7, 15,  0,  3,  0, 10, 13, 13, 10,  1, 11,  7,  1,  9,  1,\n",
       "        9,  5, 13, 15, 12,  1,  9,  8,  7,  9, 13, 12,  4, 12,  8,  0,  9,\n",
       "       13, 15,  5,  0, 10,  2,  2,  9, 11,  7,  0,  0, 12,  3,  5,  2,  1,\n",
       "        4, 11, 12,  0, 12,  8, 13, 10,  0,  0, 11,  0, 11, 13, 15, 15,  8,\n",
       "        1, 10, 13, 12,  1,  9, 14,  6, 13, 12,  3, 11, 10, 11,  1, 14,  3,\n",
       "        0, 11,  4,  3,  0, 11,  6,  2,  7,  8, 10, 10,  8,  5,  7,  8, 13,\n",
       "        6,  8, 15,  0,  2,  3,  4, 10, 12,  8, 13,  2,  7, 15, 10, 10,  5,\n",
       "        3, 13, 11, 10,  9,  2, 13,  3,  0,  7, 12,  4, 14,  7, 13,  2, 15,\n",
       "        9, 15, 15,  1,  0, 15,  3, 11,  0, 15, 15, 13, 10,  4, 13,  1, 15,\n",
       "       13,  0,  3,  3,  0,  8,  2, 15, 11,  3,  3,  2,  5, 14,  4,  0,  3,\n",
       "        2, 12,  1,  3,  0, 12,  9,  0,  7, 15, 15,  3,  2, 12, 15,  3,  6,\n",
       "       13, 14, 14,  0,  5, 13, 12,  6, 12, 13,  6,  9,  8,  7, 10,  1,  2,\n",
       "       11, 14,  2, 14, 15, 13,  0,  1,  2, 10, 13,  0, 14,  6,  9,  5,  2,\n",
       "        1,  3, 14,  8, 13, 11,  4,  8, 13,  1,  4,  7, 13, 14,  5, 11,  8,\n",
       "        0,  1, 11,  0,  1,  3,  7,  2,  5, 14, 15,  9,  1, 13,  3, 11,  8,\n",
       "       12,  0,  3, 11,  5,  2,  6, 14,  7,  8,  3,  2, 13, 12, 11, 13, 11,\n",
       "        9,  2, 10,  5, 12,  0, 15,  1,  9,  9,  0, 12,  7, 14,  9,  9,  1,\n",
       "       15, 10,  4,  8,  9,  3,  7,  8,  6, 12, 13, 10,  7,  5,  1, 13, 10,\n",
       "        2,  3,  1, 10,  9,  7,  8,  1,  4, 11,  6, 11, 15,  0,  8,  7, 11,\n",
       "        0,  1,  0, 12, 11,  1, 10,  7,  9,  1, 14,  3, 15, 14,  6, 14,  7,\n",
       "        1, 14,  6, 14,  9,  7,  1, 14,  1, 14,  2, 13,  7,  2,  0,  8,  8,\n",
       "        0,  8,  6, 15, 14,  3,  6, 11,  8, 14,  6,  7,  1,  9, 11,  0,  0,\n",
       "        3, 10,  8, 10,  5,  2,  7, 12, 14,  9,  8, 12,  9, 13, 10,  5,  5,\n",
       "        9, 11,  6, 10, 12,  8, 10, 14, 14,  9,  6, 15,  8,  1,  8,  5,  5,\n",
       "        5,  5,  0, 15, 14,  5,  3,  2, 12,  1, 15, 11, 10,  8, 13,  1,  7,\n",
       "       12, 11,  4,  4,  4,  0, 11, 15, 14, 13, 12,  8,  5, 12, 13,  4, 13,\n",
       "       15, 15,  1,  5,  3,  7, 13, 12,  8,  7,  5, 15, 12,  9,  4,  2, 14,\n",
       "        1, 13,  6,  6,  3,  7,  7,  5,  5, 12,  6,  8,  0,  4,  7,  7,  0,\n",
       "       12,  6, 12,  0,  3, 11,  2,  3,  5,  4, 13,  1, 15,  7, 14,  0,  8,\n",
       "       12,  5, 12,  7,  2, 15,  8,  8,  4,  6, 12,  2,  5, 13,  7,  6,  3,\n",
       "       13,  3,  2, 14,  0,  8, 12, 12,  8, 13, 13, 14,  4,  6,  2,  5, 14,\n",
       "       12, 13, 13,  4, 12,  0,  1,  0,  8,  1,  0,  8,  4,  9,  2, 11,  0,\n",
       "        1,  2, 14, 15,  0,  8,  9, 11, 14,  8,  9,  7, 12, 11, 12, 13,  3,\n",
       "        7,  7, 13,  8, 12,  2,  5, 13,  7, 10,  7,  8, 12,  6, 14,  0, 10,\n",
       "       11,  3, 12, 13, 14,  7,  2,  8,  2, 13,  2,  5,  8, 14, 12, 14, 12,\n",
       "        3,  5,  8, 12,  4, 11, 15,  0,  9,  9,  5, 13, 10,  2,  2, 12, 15,\n",
       "        6])"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "8bce361f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "605"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(test_pred_labels == test_labels['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "e07d40ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>./Data/train/train/41.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65</td>\n",
       "      <td>./Data/train/train/65.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>100</td>\n",
       "      <td>./Data/train/train/100.tif</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>164</td>\n",
       "      <td>./Data/train/train/164.tif</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>372</td>\n",
       "      <td>./Data/train/train/372.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>15508</td>\n",
       "      <td>./Data/train/train/15508.tif</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>15568</td>\n",
       "      <td>./Data/train/train/15568.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>15621</td>\n",
       "      <td>./Data/train/train/15621.tif</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>15687</td>\n",
       "      <td>./Data/train/train/15687.tif</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>15832</td>\n",
       "      <td>./Data/train/train/15832.tif</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>195 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                        images  label\n",
       "3       41     ./Data/train/train/41.tif      1\n",
       "4       65     ./Data/train/train/65.tif     14\n",
       "7      100    ./Data/train/train/100.tif     10\n",
       "11     164    ./Data/train/train/164.tif     15\n",
       "19     372    ./Data/train/train/372.tif     13\n",
       "..     ...                           ...    ...\n",
       "773  15508  ./Data/train/train/15508.tif     10\n",
       "779  15568  ./Data/train/train/15568.tif      9\n",
       "785  15621  ./Data/train/train/15621.tif     15\n",
       "788  15687  ./Data/train/train/15687.tif     12\n",
       "797  15832  ./Data/train/train/15832.tif     11\n",
       "\n",
       "[195 rows x 3 columns]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_train_data = test_labels[test_pred_labels != test_labels['label']]\n",
    "mis_train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "0f0b85fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fscore</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.650794</td>\n",
       "      <td>0.759259</td>\n",
       "      <td>0.700855</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.529412</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.568421</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.911111</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.913043</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.821918</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.641026</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.561798</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.878049</td>\n",
       "      <td>0.782609</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.784314</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>0.776699</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.962963</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.787234</td>\n",
       "      <td>0.787234</td>\n",
       "      <td>0.787234</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.755556</td>\n",
       "      <td>0.586207</td>\n",
       "      <td>0.660194</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.837209</td>\n",
       "      <td>0.679245</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.786885</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.768000</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.796875</td>\n",
       "      <td>0.864407</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.725000</td>\n",
       "      <td>0.674419</td>\n",
       "      <td>0.698795</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    precision    recall    fscore  support\n",
       "0    0.650794  0.759259  0.700855       54\n",
       "1    0.529412  0.613636  0.568421       44\n",
       "2    0.911111  0.953488  0.931818       43\n",
       "3    0.933333  0.913043  0.923077       46\n",
       "4    0.810811  0.833333  0.821918       36\n",
       "5    0.641026  0.500000  0.561798       50\n",
       "6    0.878049  0.782609  0.827586       46\n",
       "7    0.784314  0.769231  0.776699       52\n",
       "8    0.787879  0.962963  0.866667       54\n",
       "9    0.787234  0.787234  0.787234       47\n",
       "10   0.755556  0.586207  0.660194       58\n",
       "11   0.692308  0.642857  0.666667       56\n",
       "12   0.571429  0.837209  0.679245       43\n",
       "13   0.786885  0.750000  0.768000       64\n",
       "14   0.944444  0.796875  0.864407       64\n",
       "15   0.725000  0.674419  0.698795       43"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = score(test_labels['label'], test_pred_labels)\n",
    "pd.DataFrame({\n",
    "    'precision': precision,\n",
    "    'recall': recall,\n",
    "    'fscore': fscore,\n",
    "    'support': support\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "ffd70520",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7564612780924312"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(fscore)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "067c5096",
   "metadata": {},
   "source": [
    "#### On Validation Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e665591",
   "metadata": {},
   "source": [
    "Generating Predictions for the Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "20d57a31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29/29 [==============================] - 0s 3ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-07 23:28:45.355855: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    }
   ],
   "source": [
    "valid_preds = model1.predict(fine_tune_data_validation_red)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "db7d1256",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_pred_labels = np.argmax(valid_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "315c93f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11,  6,  4,  8,  3,  8,  1,  1,  2,  4,  3,  0, 12, 12, 13,  9,  1,\n",
       "        1, 11, 12, 14,  9, 13,  6,  1,  2,  7,  3,  8,  1,  0, 14,  8, 10,\n",
       "       14,  6, 11,  8,  6, 10,  4, 14, 12,  3,  2, 10,  3,  2,  4,  2, 10,\n",
       "        0, 10,  0,  1,  9,  2,  8, 13,  1,  2, 11,  4,  5,  5,  8, 11,  7,\n",
       "        8,  4,  5,  5,  4,  8,  4,  6,  4, 15, 11,  0,  9, 15,  5, 11,  7,\n",
       "       15, 11, 10,  0, 10, 12,  5, 15, 15,  3,  4, 11, 10,  6,  5,  6, 11,\n",
       "        1, 12,  7,  1, 15,  9, 10,  1,  7,  7,  7, 14,  2,  4,  1, 14, 12,\n",
       "        1, 13,  1,  3,  6,  1, 10,  0,  6,  4, 12,  8,  1, 13,  8, 10,  3,\n",
       "       11, 11,  5,  5,  6,  6,  2,  3,  2,  9,  9,  0,  0, 14, 10, 10,  1,\n",
       "       11,  7,  3, 14,  1,  1, 14,  3, 11,  2, 13,  7, 14,  4,  5,  7,  9,\n",
       "       12, 10,  4,  7,  4,  0,  1,  6, 11,  8, 15, 11,  3,  2,  3,  3, 14,\n",
       "        3, 11,  2,  9,  5,  6,  2,  4,  9,  2,  9, 10,  7,  1, 13,  1,  0,\n",
       "       15,  1,  4,  1,  4,  4,  8,  6,  6,  7,  4,  4, 13,  3, 10, 12,  1,\n",
       "        6,  0,  8, 10,  2, 12,  4,  1,  1, 12, 15,  8, 11,  2,  6,  7, 12,\n",
       "       11, 12,  9,  3, 14, 13,  7, 10,  8,  0,  4,  4,  9,  8,  3,  7, 13,\n",
       "        8,  9, 15,  6,  0,  4, 11,  0,  2,  9,  5, 12,  5,  0,  4,  3,  2,\n",
       "        6,  5,  8, 14,  9,  2, 10,  2,  8,  1,  3,  3, 13, 11,  2,  1,  9,\n",
       "        9, 11, 11,  5,  0, 15, 11,  3, 11,  4,  0,  3,  2, 15,  5, 14,  8,\n",
       "       12,  2, 10,  9, 13,  0, 10,  0, 10,  7,  6,  8, 11,  4, 10,  6,  6,\n",
       "        3,  0,  5,  1,  0, 15, 12,  5, 10, 11, 13, 13,  2, 14,  1, 10,  4,\n",
       "       15, 11,  4,  9,  1,  4, 10,  1, 13,  0,  3, 12,  5, 14,  4,  0,  5,\n",
       "       15, 10,  8, 12, 15,  6,  0,  3,  1,  7, 10,  3, 11, 15,  6, 10,  2,\n",
       "        6, 11,  9, 10, 11,  7,  8, 10,  6,  3,  4, 10,  2,  0, 12, 11, 15,\n",
       "        1, 14,  4,  0, 15,  2,  1,  7, 15,  0,  2, 12,  5, 12, 10,  0,  1,\n",
       "        3, 14, 10,  6, 13,  1,  2,  1, 14,  4,  3,  0,  2,  0, 15, 14,  4,\n",
       "       11, 12,  9,  2,  6,  3,  2,  6,  9,  0,  2,  8,  6, 10, 12,  6,  5,\n",
       "       14, 12, 14,  7,  3,  7,  0, 15, 10, 11, 12,  1,  8,  0,  4, 11,  2,\n",
       "        5,  4,  3, 10, 14,  8, 11,  4,  4,  1,  9,  4,  6, 12,  6,  4,  9,\n",
       "        6,  1,  3, 11,  2, 15,  8,  4,  5,  4,  4, 14,  6,  5, 12, 13,  9,\n",
       "        7,  7, 13,  2,  3, 15,  1,  7,  9, 12,  8,  2,  0,  8, 10,  0, 13,\n",
       "       14, 15,  8,  5,  8,  0,  2,  7, 12,  3,  8, 12, 14,  9,  0, 13, 15,\n",
       "       13,  0,  3, 12, 10,  6, 13,  3, 12, 14,  5, 14, 10,  9,  8, 12, 11,\n",
       "        9,  9,  8,  7,  9,  1, 13,  8,  8, 12,  4, 11,  1,  2,  3,  0,  0,\n",
       "        1,  0, 11,  5, 11,  5, 14,  6, 10,  4,  4, 15, 13,  3, 15,  0,  4,\n",
       "        5,  0,  8, 13, 11,  3,  8,  2, 10,  9,  7,  0,  1,  2, 11,  9, 12,\n",
       "       13,  6, 12,  0,  4,  8,  1,  1,  1,  2,  0,  4,  7,  8, 10,  8, 14,\n",
       "        9, 13,  0, 14,  6,  3,  7, 11,  8, 12,  4, 14,  0, 14,  0, 11,  3,\n",
       "       14,  6,  1,  7, 14,  7, 11, 12, 10, 12,  2, 12,  3,  8,  1, 14,  9,\n",
       "        9,  4, 12,  2, 13,  2, 15,  0,  4,  6,  3,  8,  6,  5,  3,  7, 10,\n",
       "        2,  7, 13, 10, 15, 15,  2,  1,  4,  1, 12, 13, 15, 10, 13,  7, 14,\n",
       "       13,  9, 15,  7, 11,  0,  8,  5,  4, 15,  6,  8, 13,  3, 13, 14, 11,\n",
       "       12, 10,  4, 10,  8,  2, 15,  4,  5, 11,  3, 12, 13,  1,  5,  9,  8,\n",
       "        3,  3,  7, 10,  1,  7,  6, 13, 14,  6, 14,  3,  5,  1,  0,  5, 11,\n",
       "        5,  7,  4,  9,  4,  3, 12, 14,  5,  8, 10, 15, 13, 15, 15,  2,  0,\n",
       "        0,  6, 14,  9,  1,  7,  6,  1,  6,  8,  7,  2,  2,  4,  0,  7,  5,\n",
       "        6,  0, 13,  5,  4,  3, 14, 11, 15,  0,  1,  2,  7,  0,  0, 11,  6,\n",
       "        0,  3,  4,  8, 14,  6, 14,  2, 12,  0,  1, 13, 14,  1,  2,  6, 15,\n",
       "       10,  9,  4, 15, 10,  7,  2,  2,  8,  6,  4, 13,  7, 15, 15,  3,  2,\n",
       "       13,  4,  9, 11,  9, 12,  5, 14, 12,  9,  2,  4,  0,  5,  5, 15,  1,\n",
       "        6,  7,  8,  0, 15, 15,  4,  8,  9,  3,  4,  8, 14, 13,  5, 13, 11,\n",
       "        2, 15, 15,  7,  2, 12,  6, 15,  6,  9, 10,  0,  5,  6, 14,  1,  0,\n",
       "        8,  9,  8,  8, 15,  4,  0, 11,  3,  3,  3, 11,  2, 13,  5,  9, 15,\n",
       "        5,  1, 13,  2, 11,  4, 15,  7,  6, 15,  8,  7, 12,  8,  3,  4])"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_pred_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bece594",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "304c47de",
   "metadata": {},
   "source": [
    "For Kaggle Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "0e59c09c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  6.31446934, 953.5032959 , 124.96561432, 983.04998779,\n",
       "          1.        ,   1.        ],\n",
       "       [  6.83573103, 967.67987061, 231.95384216, 982.98071289,\n",
       "          1.        ,   1.        ],\n",
       "       [ 21.61216927, 904.17651367, 725.11962891, 909.07617188,\n",
       "          5.        ,   1.        ],\n",
       "       [ 35.6613121 ,  15.60731411,  43.65130234,  22.02859116,\n",
       "          1.        ,   1.        ],\n",
       "       [ 78.91178131,  29.19843483, 692.08996582, 202.60029602,\n",
       "          6.        ,   1.        ],\n",
       "       [110.42622375, 127.7110672 , 637.99682617, 773.59735107,\n",
       "          2.        ,   1.        ],\n",
       "       [165.22764587, 847.66424561, 662.84649658, 854.0112915 ,\n",
       "          5.        ,   1.        ],\n",
       "       [532.09185791, 859.65454102, 639.74468994, 874.27435303,\n",
       "          1.        ,   1.        ],\n",
       "       [712.800354  ,   4.76252604, 715.38275146,  71.32498932,\n",
       "          1.        ,   1.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ]])"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_info_validation[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "121c612a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "e7ca9d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17801</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17802</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17803</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17804</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17805</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>18696</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>18697</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>18698</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>18699</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>18700</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  label\n",
       "0    17801     11\n",
       "1    17802      6\n",
       "2    17803      4\n",
       "3    17804      8\n",
       "4    17805      3\n",
       "..     ...    ...\n",
       "895  18696      7\n",
       "896  18697     12\n",
       "897  18698      8\n",
       "898  18699      3\n",
       "899  18700      4\n",
       "\n",
       "[900 rows x 2 columns]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_data = pd.DataFrame({'id':list(range(17801, 18701))})\n",
    "submission_data['label'] = valid_pred_labels\n",
    "submission_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "6eea7d9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_data.to_csv('./Submissions/submission-effnet-kPiece-PCA.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}