inceptionresnet50v2-roi-feature-extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "57fd5dfc",
   "metadata": {
    "papermill": {
     "duration": 0.010793,
     "end_time": "2022-10-08T10:58:04.067668",
     "exception": false,
     "start_time": "2022-10-08T10:58:04.056875",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25968bee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:04.088941Z",
     "iopub.status.busy": "2022-10-08T10:58:04.088130Z",
     "iopub.status.idle": "2022-10-08T10:58:04.093883Z",
     "shell.execute_reply": "2022-10-08T10:58:04.092888Z"
    },
    "papermill": {
     "duration": 0.019227,
     "end_time": "2022-10-08T10:58:04.096444",
     "exception": false,
     "start_time": "2022-10-08T10:58:04.077217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !pip uninstall tensorflow -y\n",
    "# !pip uninstall tensorflow-gpu -y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5b84ac55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:04.117777Z",
     "iopub.status.busy": "2022-10-08T10:58:04.117201Z",
     "iopub.status.idle": "2022-10-08T10:58:04.121293Z",
     "shell.execute_reply": "2022-10-08T10:58:04.120486Z"
    },
    "papermill": {
     "duration": 0.015834,
     "end_time": "2022-10-08T10:58:04.123252",
     "exception": false,
     "start_time": "2022-10-08T10:58:04.107418",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !pip install -U tensorflow-gpu==2.10\n",
    "# !pip install -U tensorflow==2.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f825fa21",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:04.143334Z",
     "iopub.status.busy": "2022-10-08T10:58:04.142556Z",
     "iopub.status.idle": "2022-10-08T10:58:10.092246Z",
     "shell.execute_reply": "2022-10-08T10:58:10.091295Z"
    },
    "papermill": {
     "duration": 5.962696,
     "end_time": "2022-10-08T10:58:10.095037",
     "exception": false,
     "start_time": "2022-10-08T10:58:04.132341",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import gc\n",
    "from zipfile import ZipFile\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import load_img, img_to_array\n",
    "\n",
    "from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAvgPool2D, BatchNormalization\n",
    "from tensorflow.keras.applications import InceptionResNetV2\n",
    "from tensorflow.keras.applications.inception_resnet_v2 import preprocess_input\n",
    "# from efficientnet_v2 import EfficientNetV2S \n",
    "from tensorflow.keras.regularizers import l2\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support as score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c0d10cb",
   "metadata": {
    "papermill": {
     "duration": 0.008845,
     "end_time": "2022-10-08T10:58:10.113319",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.104474",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ac9662e",
   "metadata": {
    "papermill": {
     "duration": 0.008644,
     "end_time": "2022-10-08T10:58:10.130980",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.122336",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "A detailed discussion and visualization of the data can be seen in [here](Data-Overview.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "081588dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.150196Z",
     "iopub.status.busy": "2022-10-08T10:58:10.149713Z",
     "iopub.status.idle": "2022-10-08T10:58:10.158241Z",
     "shell.execute_reply": "2022-10-08T10:58:10.157366Z"
    },
    "papermill": {
     "duration": 0.020221,
     "end_time": "2022-10-08T10:58:10.160203",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.139982",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sample_submission.csv', 'validation', 'train_labels.csv', 'train']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('../input/datathonindoml-2022')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "993b4448",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.179911Z",
     "iopub.status.busy": "2022-10-08T10:58:10.179111Z",
     "iopub.status.idle": "2022-10-08T10:58:10.227302Z",
     "shell.execute_reply": "2022-10-08T10:58:10.226282Z"
    },
    "papermill": {
     "duration": 0.060378,
     "end_time": "2022-10-08T10:58:10.229672",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.169294",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/0.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/1.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/2.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/3.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/4.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15995</th>\n",
       "      <td>15995</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15995...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15996</th>\n",
       "      <td>15996</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15996...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15997</th>\n",
       "      <td>15997</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15997...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15998</th>\n",
       "      <td>15998</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15998...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15999</th>\n",
       "      <td>15999</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15999...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                             images  label\n",
       "0          0     ../input/datathonindoml-2022/train/train/0.tif      1\n",
       "1          1     ../input/datathonindoml-2022/train/train/1.tif     13\n",
       "2          2     ../input/datathonindoml-2022/train/train/2.tif     13\n",
       "3          3     ../input/datathonindoml-2022/train/train/3.tif     14\n",
       "4          4     ../input/datathonindoml-2022/train/train/4.tif      6\n",
       "...      ...                                                ...    ...\n",
       "15995  15995  ../input/datathonindoml-2022/train/train/15995...      2\n",
       "15996  15996  ../input/datathonindoml-2022/train/train/15996...     15\n",
       "15997  15997  ../input/datathonindoml-2022/train/train/15997...      3\n",
       "15998  15998  ../input/datathonindoml-2022/train/train/15998...      9\n",
       "15999  15999  ../input/datathonindoml-2022/train/train/15999...      9\n",
       "\n",
       "[16000 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = pd.read_csv(\"../input/datathonindoml-2022/train_labels.csv\")\n",
    "images = ['../input/datathonindoml-2022/train/train/'+str(name)+'.tif' for name in labels['id']]\n",
    "labels['images'] = images\n",
    "labels = labels[['id', 'images', 'label']]\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "90474fdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.249904Z",
     "iopub.status.busy": "2022-10-08T10:58:10.249065Z",
     "iopub.status.idle": "2022-10-08T10:58:10.256660Z",
     "shell.execute_reply": "2022-10-08T10:58:10.255679Z"
    },
    "papermill": {
     "duration": 0.019952,
     "end_time": "2022-10-08T10:58:10.258932",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.238980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class_labels = list(set(labels['label']))\n",
    "class_names = [\n",
    "    'letter', 'form', 'email', 'handwritten', 'advertisement', 'scientific report', 'scientific publication',\n",
    "    'specification', 'file folder', 'news article', 'budget', 'invoice', 'presentation', 'questionnaire', 'resume',\n",
    "    'memo'\n",
    "]\n",
    "label_names = pd.DataFrame({\n",
    "    'labels': class_labels,\n",
    "    'names': class_names\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "737a0f40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.279298Z",
     "iopub.status.busy": "2022-10-08T10:58:10.278494Z",
     "iopub.status.idle": "2022-10-08T10:58:10.333162Z",
     "shell.execute_reply": "2022-10-08T10:58:10.332268Z"
    },
    "papermill": {
     "duration": 0.067086,
     "end_time": "2022-10-08T10:58:10.335220",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.268134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/0.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/1.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/2.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/3.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/4.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15195</th>\n",
       "      <td>15995</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15995...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15196</th>\n",
       "      <td>15996</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15996...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15197</th>\n",
       "      <td>15997</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15997...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15198</th>\n",
       "      <td>15998</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15998...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15199</th>\n",
       "      <td>15999</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15999...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15200 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                             images  label\n",
       "0          0     ../input/datathonindoml-2022/train/train/0.tif      1\n",
       "1          1     ../input/datathonindoml-2022/train/train/1.tif     13\n",
       "2          2     ../input/datathonindoml-2022/train/train/2.tif     13\n",
       "3          3     ../input/datathonindoml-2022/train/train/3.tif     14\n",
       "4          4     ../input/datathonindoml-2022/train/train/4.tif      6\n",
       "...      ...                                                ...    ...\n",
       "15195  15995  ../input/datathonindoml-2022/train/train/15995...      2\n",
       "15196  15996  ../input/datathonindoml-2022/train/train/15996...     15\n",
       "15197  15997  ../input/datathonindoml-2022/train/train/15997...      3\n",
       "15198  15998  ../input/datathonindoml-2022/train/train/15998...      9\n",
       "15199  15999  ../input/datathonindoml-2022/train/train/15999...      9\n",
       "\n",
       "[15200 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels = pd.read_csv('../input/dataroi/train_labels_1.csv')\n",
    "test_labels = pd.read_csv('../input/dataroi/test_labels_1.csv')\n",
    "train_labels['images'] = ['../input/datathonindoml-2022/train/train/'+path.split('/')[-1] for path in list(train_labels['images'])]\n",
    "test_labels['images'] = ['../input/datathonindoml-2022/validation/validation/'+path.split('/')[-1] for path in list(test_labels['images'])]\n",
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "19f5b080",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.355329Z",
     "iopub.status.busy": "2022-10-08T10:58:10.355065Z",
     "iopub.status.idle": "2022-10-08T10:58:10.367381Z",
     "shell.execute_reply": "2022-10-08T10:58:10.366414Z"
    },
    "papermill": {
     "duration": 0.024593,
     "end_time": "2022-10-08T10:58:10.369422",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.344829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>15816</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>15824</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>15832</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>15863</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>15994</td>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                             images  label\n",
       "0       18  ../input/datathonindoml-2022/validation/valida...     11\n",
       "1       25  ../input/datathonindoml-2022/validation/valida...      8\n",
       "2       33  ../input/datathonindoml-2022/validation/valida...      6\n",
       "3       41  ../input/datathonindoml-2022/validation/valida...      1\n",
       "4       65  ../input/datathonindoml-2022/validation/valida...     14\n",
       "..     ...                                                ...    ...\n",
       "795  15816  ../input/datathonindoml-2022/validation/valida...      2\n",
       "796  15824  ../input/datathonindoml-2022/validation/valida...      2\n",
       "797  15832  ../input/datathonindoml-2022/validation/valida...     11\n",
       "798  15863  ../input/datathonindoml-2022/validation/valida...     15\n",
       "799  15994  ../input/datathonindoml-2022/validation/valida...      6\n",
       "\n",
       "[800 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8d5367da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.390675Z",
     "iopub.status.busy": "2022-10-08T10:58:10.390411Z",
     "iopub.status.idle": "2022-10-08T10:58:10.883053Z",
     "shell.execute_reply": "2022-10-08T10:58:10.881903Z"
    },
    "papermill": {
     "duration": 0.506506,
     "end_time": "2022-10-08T10:58:10.885644",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.379138",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "roi_info = np.load('../input/roi-info/train_roi_viz.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cd185690",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.907797Z",
     "iopub.status.busy": "2022-10-08T10:58:10.906773Z",
     "iopub.status.idle": "2022-10-08T10:58:10.914251Z",
     "shell.execute_reply": "2022-10-08T10:58:10.913342Z"
    },
    "papermill": {
     "duration": 0.020434,
     "end_time": "2022-10-08T10:58:10.916258",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.895824",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 100, 6)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_info.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0db05594",
   "metadata": {
    "papermill": {
     "duration": 0.009944,
     "end_time": "2022-10-08T10:58:10.935948",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.926004",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e68520e",
   "metadata": {
    "papermill": {
     "duration": 0.009548,
     "end_time": "2022-10-08T10:58:10.955225",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.945677",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We will be using the image data above and the ROI extraction data as explained [here](RoI-Extraction.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0b7cdec5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:10.977020Z",
     "iopub.status.busy": "2022-10-08T10:58:10.976075Z",
     "iopub.status.idle": "2022-10-08T10:58:10.980682Z",
     "shell.execute_reply": "2022-10-08T10:58:10.979761Z"
    },
    "papermill": {
     "duration": 0.017728,
     "end_time": "2022-10-08T10:58:10.982727",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.964999",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 50\n",
    "# INPUT_SHAPE = (800, 600) # --> Takes too much time on Kaggle\n",
    "INPUT_SHAPE = (400, 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4768cde9",
   "metadata": {
    "papermill": {
     "duration": 0.010066,
     "end_time": "2022-10-08T10:58:11.002449",
     "exception": false,
     "start_time": "2022-10-08T10:58:10.992383",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now due to the quite varying nature of the ROI shapes, the following pre-processing steps on each ROI converts them to same shape as that of the original document images by resizing and padding with the maximum color in the ROI which is mostly white and sometimes greyish, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "27f67e23",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:11.023907Z",
     "iopub.status.busy": "2022-10-08T10:58:11.023083Z",
     "iopub.status.idle": "2022-10-08T10:58:11.032855Z",
     "shell.execute_reply": "2022-10-08T10:58:11.032014Z"
    },
    "papermill": {
     "duration": 0.022532,
     "end_time": "2022-10-08T10:58:11.034747",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.012215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def resize_and_pad_with_doc_max_col(img, f_target_height=INPUT_SHAPE[0], f_target_width=INPUT_SHAPE[1]):\n",
    "    f_width = img.shape[1]\n",
    "    f_height = img.shape[0]\n",
    "    ratio = np.max([f_width / f_target_width, f_height / f_target_height])\n",
    "    resized_height_float = f_height / ratio\n",
    "    resized_width_float = f_width / ratio\n",
    "    resized_height = tf.cast(\n",
    "        np.floor(resized_height_float), dtype=tf.int32)\n",
    "    resized_width = tf.cast(\n",
    "        np.floor(resized_width_float), dtype=tf.int32)\n",
    "\n",
    "    padding_height = (f_target_height - resized_height_float) / 2\n",
    "    padding_width = (f_target_width - resized_width_float) / 2\n",
    "    f_padding_height = np.floor(padding_height)\n",
    "    f_padding_width = np.floor(padding_width)\n",
    "    p_height = np.max([0, tf.cast(f_padding_height, dtype=tf.int32)])\n",
    "    p_width = np.max([0, tf.cast(f_padding_width, dtype=tf.int32)])\n",
    "\n",
    "    resized_padded_image = tf.image.resize_with_pad(img, f_target_height, f_target_width).numpy()\n",
    "    white_color = np.max(resized_padded_image)\n",
    "    resized_padded_image[:p_height, :] = white_color\n",
    "    resized_padded_image[(f_target_height-p_height-1):, :] = white_color\n",
    "    resized_padded_image[:, :p_width] = white_color\n",
    "    resized_padded_image[:, (f_target_width-p_width-1):] = white_color\n",
    "    return resized_padded_image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c9758eb",
   "metadata": {
    "papermill": {
     "duration": 0.009555,
     "end_time": "2022-10-08T10:58:11.054240",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.044685",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The data generator as created below does the following things:\n",
    "- It uses the RoI extraction information and the image to extract the portions of the document image and returns a list of these snippet regions."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "542aae11",
   "metadata": {
    "papermill": {
     "duration": 0.009695,
     "end_time": "2022-10-08T10:58:11.073994",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.064299",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now, after we have seen how each document image is processed which is an extension to the [4 Piece Add On Model's](EfficientNet-4Piece-Model.ipynb) pre-processing which worked with only 4 pieces."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c18aa076",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:11.096158Z",
     "iopub.status.busy": "2022-10-08T10:58:11.095604Z",
     "iopub.status.idle": "2022-10-08T10:58:11.108209Z",
     "shell.execute_reply": "2022-10-08T10:58:11.107281Z"
    },
    "papermill": {
     "duration": 0.02621,
     "end_time": "2022-10-08T10:58:11.110193",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.083983",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# class ImageDataGenerator:\n",
    "#     def __init__(self, df, X_col, roi_info, y_col, batch_size, roi_pad=5, input_size=(800, 600), shuffle=True, base=0):\n",
    "#         self.df = df.copy() # DataFrame consisting image paths of inputs and the labels for the outputs\n",
    "#         self.X_col = X_col # Input column, specifying image path, in the DataFrame\n",
    "#         self.roi_info = roi_info # Contains valid information for the pre-processing of each image\n",
    "#         self.y_col = y_col # Output column, specifying corresponding label, in the DataFrame\n",
    "#         self.batch_size = batch_size # Batch Size\n",
    "#         self.roi_pad = roi_pad # Padding on 4 sides for each ROI\n",
    "#         self.input_size = input_size # Input Image size\n",
    "#         self.shuffle = shuffle # Shuffle Data after each epoch\n",
    "#         self.n = len(self.df) # length of the entire data\n",
    "#         self.base=base # base value\n",
    "        \n",
    "#     def __get_input(self, path):\n",
    "#         image_data = img_to_array(load_img(path))\n",
    "#         idx = int(path.split('/')[-1].split('.')[0]) - self.base\n",
    "#         # some other pre-processing / data-augmentation goes here\n",
    "#         roi_data = self.roi_info[idx]\n",
    "#         return [resize_and_pad_with_doc_max_col(image_data[max(0, int(y_top)-self.roi_pad):min(image_data.shape[0], int(y_bottom)+self.roi_pad), max(0, int(x_top)-self.roi_pad):min(image_data.shape[1], int(x_bottom)+self.roi_pad)])/255.0 for x_top, y_top, x_bottom, y_bottom, roi_type, pad_mask in roi_data if pad_mask == 1]\n",
    "\n",
    "#     def __get_data(self, batches):\n",
    "#         batch_paths = batches[self.X_col]\n",
    "#         batch_labels = batches[self.y_col]\n",
    "#         X_batch = [np.array(self.__get_input(path)) for path in batch_paths]\n",
    "#         y_batch = tf.cast(batch_labels, dtype=tf.float32)\n",
    "#         return X_batch, y_batch\n",
    "    \n",
    "#     def __getitem__(self, index):\n",
    "#         batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "#         X, y = self.__get_data(batches)\n",
    "#         return X, y\n",
    "    \n",
    "#     def __len__(self):\n",
    "#         return self.n // self.batch_size\n",
    "\n",
    "class ImageDataGenerator:\n",
    "    def __init__(self, df, X_col, roi_info, y_col, batch_size, roi_pad=5, input_size=(800, 600), shuffle=True, base=0):\n",
    "        self.df = df.copy() # DataFrame consisting image paths of inputs and the labels for the outputs\n",
    "        self.X_col = X_col # Input column, specifying image path, in the DataFrame\n",
    "        self.roi_info = roi_info # Contains valid information for the pre-processing of each image\n",
    "        self.y_col = y_col # Output column, specifying corresponding label, in the DataFrame\n",
    "        self.batch_size = batch_size # Batch Size\n",
    "        self.roi_pad = roi_pad # Padding on 4 sides for each ROI\n",
    "        self.input_size = input_size # Input Image size\n",
    "        self.shuffle = shuffle # Shuffle Data after each epoch\n",
    "        self.n = len(self.df) # length of the entire data\n",
    "        self.base=base # base value\n",
    "    \n",
    "    # def preprocess(self, image):  \n",
    "    #     return (tf.cast(image, dtype=tf.float32) - 128.00) / 128.00\n",
    "        \n",
    "    def __get_input(self, path):\n",
    "        image_data = img_to_array(load_img(path))\n",
    "        idx = int(path.split('/')[-1].split('.')[0]) - self.base\n",
    "        # some other pre-processing / data-augmentation goes here\n",
    "        roi_data = self.roi_info[idx]\n",
    "        return [preprocess_input(resize_and_pad_with_doc_max_col(image_data[max(0, int(y_top)-self.roi_pad):min(image_data.shape[0], int(y_bottom)+self.roi_pad), max(0, int(x_top)-self.roi_pad):min(image_data.shape[1], int(x_bottom)+self.roi_pad)])) for x_top, y_top, x_bottom, y_bottom, roi_type, pad_mask in roi_data if pad_mask == 1]\n",
    "\n",
    "    def __get_data(self, batches):\n",
    "        batch_paths = batches[self.X_col]\n",
    "        batch_labels = batches[self.y_col]\n",
    "        X_batch = [np.array(self.__get_input(path)) for path in batch_paths]\n",
    "        y_batch = tf.cast(batch_labels, dtype=tf.float32)\n",
    "        return X_batch, y_batch\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "        X, y = self.__get_data(batches)\n",
    "        return X, y\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.n // self.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f132e7b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:11.130880Z",
     "iopub.status.busy": "2022-10-08T10:58:11.130623Z",
     "iopub.status.idle": "2022-10-08T10:58:11.136203Z",
     "shell.execute_reply": "2022-10-08T10:58:11.135338Z"
    },
    "papermill": {
     "duration": 0.0181,
     "end_time": "2022-10-08T10:58:11.138080",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.119980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_generator = ImageDataGenerator(\n",
    "    df=labels,\n",
    "    X_col='images',\n",
    "    roi_info=roi_info,\n",
    "    y_col='label',\n",
    "    batch_size=BATCH_SIZE,\n",
    "    input_size=INPUT_SHAPE,\n",
    "    shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9c612e40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:11.158682Z",
     "iopub.status.busy": "2022-10-08T10:58:11.158427Z",
     "iopub.status.idle": "2022-10-08T10:58:11.162813Z",
     "shell.execute_reply": "2022-10-08T10:58:11.161591Z"
    },
    "papermill": {
     "duration": 0.017546,
     "end_time": "2022-10-08T10:58:11.165260",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.147714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# batch = train_generator[0]\n",
    "# print('*** Batch Overview ***')\n",
    "# print('Batch Size:', len(batch[0]))\n",
    "# for i in range(BATCH_SIZE):\n",
    "#     print(f'Number of ROIs for the {i+1}th input image:', batch[0][i].shape[0])\n",
    "# print('ROI Dimensions:', batch[0][0].shape[1:])\n",
    "# print('Output Label Tensor Shape:', batch[1].shape)\n",
    "# del batch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "becf9ef2",
   "metadata": {
    "papermill": {
     "duration": 0.009773,
     "end_time": "2022-10-08T10:58:11.184694",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.174921",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Because of too many images, the batch visualization is not provided but it is similar to that of above shown for one document which is equivalent to batch size of 1."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28ba0b55",
   "metadata": {
    "papermill": {
     "duration": 0.009484,
     "end_time": "2022-10-08T10:58:11.203928",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.194444",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model Building"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16a6b5f2",
   "metadata": {
    "papermill": {
     "duration": 0.00955,
     "end_time": "2022-10-08T10:58:11.223214",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.213664",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "After building a couple of CNN models consdering only the entire document image [EfficientNet Only Model](EfficientNet-Only-Model.ipynb) and [4 Piece Add On Model](EfficientNet-4Piece-Model.ipynb). It is now time to build much more better models considering other structures we will start by incorporating the ROI visual features and then later much more.\n",
    "\n",
    "We will use the following pre-trained model to learn features for these ROIs."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3687a6ee",
   "metadata": {
    "papermill": {
     "duration": 0.009559,
     "end_time": "2022-10-08T10:58:11.242644",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.233085",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- EfficientNetV2S"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8eaaf3d",
   "metadata": {
    "papermill": {
     "duration": 0.009502,
     "end_time": "2022-10-08T10:58:11.261842",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.252340",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The EfficientNetV2S is a large model and since, we are not pre-training the entire model, we will just fine-tune it with few extra layers. So, to fasten training we precompute the output of the EfficientNetV2S model and use this for training the added Dense Layers for Fine-Tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6448f081",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:11.283674Z",
     "iopub.status.busy": "2022-10-08T10:58:11.283406Z",
     "iopub.status.idle": "2022-10-08T10:58:19.493367Z",
     "shell.execute_reply": "2022-10-08T10:58:19.492361Z"
    },
    "papermill": {
     "duration": 8.223563,
     "end_time": "2022-10-08T10:58:19.495793",
     "exception": false,
     "start_time": "2022-10-08T10:58:11.272230",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:58:11.370092: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:11.460228: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:11.461053: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:11.462231: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-10-08 10:58:11.471418: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:11.472242: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:11.472982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:13.496112: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:13.497002: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:13.497744: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 10:58:13.498375: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15401 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "219062272/219055592 [==============================] - 1s 0us/step\n",
      "219070464/219055592 [==============================] - 1s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# model_xception = Xception(weights=\"imagenet\", input_shape=(*INPUT_SHAPE, 3), include_top=False)\n",
    "# model_effnet = EfficientNetV2S(include_top=False, weights='imagenet', input_shape=(*INPUT_SHAPE, 3))\n",
    "model_inception_resnet = InceptionResNetV2(include_top=False, weights='imagenet', input_shape=(*INPUT_SHAPE, 3))\n",
    "model_inception_resnet.trainable = False\n",
    "# model_xception.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41457694",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:19.521624Z",
     "iopub.status.busy": "2022-10-08T10:58:19.521287Z",
     "iopub.status.idle": "2022-10-08T10:58:21.058202Z",
     "shell.execute_reply": "2022-10-08T10:58:21.057089Z"
    },
    "papermill": {
     "duration": 1.551285,
     "end_time": "2022-10-08T10:58:21.060689",
     "exception": false,
     "start_time": "2022-10-08T10:58:19.509404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inp = Input(shape=(*INPUT_SHAPE, 3))\n",
    "out = model_inception_resnet(inp, training=False)\n",
    "# out = model_xception(inp, training=False)\n",
    "output = GlobalAvgPool2D()(out)\n",
    "model = tf.keras.Model(inp, output, name='Inception-ResNet-kPiece')\n",
    "# model = tf.keras.Model(inp, output, name='Xception-kPiece')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bbe12e26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:21.084530Z",
     "iopub.status.busy": "2022-10-08T10:58:21.084199Z",
     "iopub.status.idle": "2022-10-08T10:58:21.104887Z",
     "shell.execute_reply": "2022-10-08T10:58:21.103999Z"
    },
    "papermill": {
     "duration": 0.034799,
     "end_time": "2022-10-08T10:58:21.106954",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.072155",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.compile(loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'], optimizer = Adam(learning_rate = 0.0003))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a1673fa3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:21.129928Z",
     "iopub.status.busy": "2022-10-08T10:58:21.129656Z",
     "iopub.status.idle": "2022-10-08T10:58:21.159699Z",
     "shell.execute_reply": "2022-10-08T10:58:21.158671Z"
    },
    "papermill": {
     "duration": 0.044926,
     "end_time": "2022-10-08T10:58:21.162740",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.117814",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"Inception-ResNet-kPiece\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 400, 300, 3)]     0         \n",
      "_________________________________________________________________\n",
      "inception_resnet_v2 (Functio (None, 11, 8, 1536)       54336736  \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 1536)              0         \n",
      "=================================================================\n",
      "Total params: 54,336,736\n",
      "Trainable params: 0\n",
      "Non-trainable params: 54,336,736\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fc8775b",
   "metadata": {
    "papermill": {
     "duration": 0.011099,
     "end_time": "2022-10-08T10:58:21.185661",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.174562",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0bae98c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:21.208673Z",
     "iopub.status.busy": "2022-10-08T10:58:21.208402Z",
     "iopub.status.idle": "2022-10-08T10:58:21.214472Z",
     "shell.execute_reply": "2022-10-08T10:58:21.213418Z"
    },
    "papermill": {
     "duration": 0.019999,
     "end_time": "2022-10-08T10:58:21.216641",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.196642",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_generator_precompute = ImageDataGenerator(\n",
    "    df=labels,\n",
    "    X_col='images',\n",
    "    roi_info=roi_info,\n",
    "    y_col='label',\n",
    "    batch_size=BATCH_SIZE,\n",
    "    input_size=INPUT_SHAPE,\n",
    "    shuffle=False\n",
    ")\n",
    "\n",
    "checkpnt_each = 100\n",
    "assert checkpnt_each % BATCH_SIZE == 0\n",
    "checkpnt_i = checkpnt_each // BATCH_SIZE\n",
    "precompute_output = list()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95d0531c",
   "metadata": {
    "papermill": {
     "duration": 0.010696,
     "end_time": "2022-10-08T10:58:21.238263",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.227567",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "To avoid memory issues the vector representation of the ROIs of one document is followed immediately by the vectors for the ROIs of the next document. So, when using we will have to keep that in mind. Also, if no ROI is found in a document image then that is simply skipped, and must be handled later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4d120cd0",
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2022-10-08T10:58:21.262250Z",
     "iopub.status.busy": "2022-10-08T10:58:21.261975Z",
     "iopub.status.idle": "2022-10-08T12:21:12.753890Z",
     "shell.execute_reply": "2022-10-08T12:21:12.752837Z"
    },
    "papermill": {
     "duration": 4971.506917,
     "end_time": "2022-10-08T12:21:12.756350",
     "exception": false,
     "start_time": "2022-10-08T10:58:21.249433",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1th Batch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:58:27.333673: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 1236960000 exceeds 10% of free system memory.\n",
      "2022-10-08 10:58:28.323749: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2022-10-08 10:58:32.208542: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2th Batch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:58:57.233190: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 1169280000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3th Batch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:59:10.666306: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 1019520000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4th Batch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:59:23.529760: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 1186560000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5th Batch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 10:59:41.102797: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 1049760000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6th Batch\n",
      "7th Batch\n",
      "8th Batch\n",
      "9th Batch\n",
      "10th Batch\n",
      "11th Batch\n",
      "12th Batch\n",
      "13th Batch\n",
      "14th Batch\n",
      "15th Batch\n",
      "16th Batch\n",
      "17th Batch\n",
      "18th Batch\n",
      "19th Batch\n",
      "20th Batch\n",
      "21th Batch\n",
      "22th Batch\n",
      "23th Batch\n",
      "24th Batch\n",
      "25th Batch\n",
      "26th Batch\n",
      "27th Batch\n",
      "28th Batch\n",
      "29th Batch\n",
      "30th Batch\n",
      "31th Batch\n",
      "32th Batch\n",
      "33th Batch\n",
      "34th Batch\n",
      "35th Batch\n",
      "36th Batch\n",
      "37th Batch\n",
      "38th Batch\n",
      "39th Batch\n",
      "40th Batch\n",
      "41th Batch\n",
      "42th Batch\n",
      "43th Batch\n",
      "44th Batch\n",
      "45th Batch\n",
      "46th Batch\n",
      "47th Batch\n",
      "48th Batch\n",
      "49th Batch\n",
      "50th Batch\n",
      "51th Batch\n",
      "52th Batch\n",
      "53th Batch\n",
      "54th Batch\n",
      "55th Batch\n",
      "56th Batch\n",
      "57th Batch\n",
      "58th Batch\n",
      "59th Batch\n",
      "60th Batch\n",
      "61th Batch\n",
      "62th Batch\n",
      "63th Batch\n",
      "64th Batch\n",
      "65th Batch\n",
      "66th Batch\n",
      "67th Batch\n",
      "68th Batch\n",
      "69th Batch\n",
      "70th Batch\n",
      "71th Batch\n",
      "72th Batch\n",
      "73th Batch\n",
      "74th Batch\n",
      "75th Batch\n",
      "76th Batch\n",
      "77th Batch\n",
      "78th Batch\n",
      "79th Batch\n",
      "80th Batch\n",
      "81th Batch\n",
      "82th Batch\n",
      "83th Batch\n",
      "84th Batch\n",
      "85th Batch\n",
      "86th Batch\n",
      "87th Batch\n",
      "88th Batch\n",
      "89th Batch\n",
      "90th Batch\n",
      "91th Batch\n",
      "92th Batch\n",
      "93th Batch\n",
      "94th Batch\n",
      "95th Batch\n",
      "96th Batch\n",
      "97th Batch\n",
      "98th Batch\n",
      "99th Batch\n",
      "100th Batch\n",
      "101th Batch\n",
      "102th Batch\n",
      "103th Batch\n",
      "104th Batch\n",
      "105th Batch\n",
      "106th Batch\n",
      "107th Batch\n",
      "108th Batch\n",
      "109th Batch\n",
      "110th Batch\n",
      "111th Batch\n",
      "112th Batch\n",
      "113th Batch\n",
      "114th Batch\n",
      "115th Batch\n",
      "116th Batch\n",
      "117th Batch\n",
      "118th Batch\n",
      "119th Batch\n",
      "120th Batch\n",
      "121th Batch\n",
      "122th Batch\n",
      "123th Batch\n",
      "124th Batch\n",
      "125th Batch\n",
      "126th Batch\n",
      "127th Batch\n",
      "128th Batch\n",
      "129th Batch\n",
      "130th Batch\n",
      "131th Batch\n",
      "132th Batch\n",
      "133th Batch\n",
      "134th Batch\n",
      "135th Batch\n",
      "136th Batch\n",
      "137th Batch\n",
      "138th Batch\n",
      "139th Batch\n",
      "140th Batch\n",
      "141th Batch\n",
      "142th Batch\n",
      "143th Batch\n",
      "144th Batch\n",
      "145th Batch\n",
      "146th Batch\n",
      "147th Batch\n",
      "148th Batch\n",
      "149th Batch\n",
      "150th Batch\n",
      "151th Batch\n",
      "152th Batch\n",
      "153th Batch\n",
      "154th Batch\n",
      "155th Batch\n",
      "156th Batch\n",
      "157th Batch\n",
      "158th Batch\n",
      "159th Batch\n",
      "160th Batch\n",
      "161th Batch\n",
      "162th Batch\n",
      "163th Batch\n",
      "164th Batch\n",
      "165th Batch\n",
      "166th Batch\n",
      "167th Batch\n",
      "168th Batch\n",
      "169th Batch\n",
      "170th Batch\n",
      "171th Batch\n",
      "172th Batch\n",
      "173th Batch\n",
      "174th Batch\n",
      "175th Batch\n",
      "176th Batch\n",
      "177th Batch\n",
      "178th Batch\n",
      "179th Batch\n",
      "180th Batch\n",
      "181th Batch\n",
      "182th Batch\n",
      "183th Batch\n",
      "184th Batch\n",
      "185th Batch\n",
      "186th Batch\n",
      "187th Batch\n",
      "188th Batch\n",
      "189th Batch\n",
      "190th Batch\n",
      "191th Batch\n",
      "192th Batch\n",
      "193th Batch\n",
      "194th Batch\n",
      "195th Batch\n",
      "196th Batch\n",
      "197th Batch\n",
      "198th Batch\n",
      "199th Batch\n",
      "200th Batch\n",
      "201th Batch\n",
      "202th Batch\n",
      "203th Batch\n",
      "204th Batch\n",
      "205th Batch\n",
      "206th Batch\n",
      "207th Batch\n",
      "208th Batch\n",
      "209th Batch\n",
      "210th Batch\n",
      "211th Batch\n",
      "212th Batch\n",
      "213th Batch\n",
      "214th Batch\n",
      "215th Batch\n",
      "216th Batch\n",
      "217th Batch\n",
      "218th Batch\n",
      "219th Batch\n",
      "220th Batch\n",
      "221th Batch\n",
      "222th Batch\n",
      "223th Batch\n",
      "224th Batch\n",
      "225th Batch\n",
      "226th Batch\n",
      "227th Batch\n",
      "228th Batch\n",
      "229th Batch\n",
      "230th Batch\n",
      "231th Batch\n",
      "232th Batch\n",
      "233th Batch\n",
      "234th Batch\n",
      "235th Batch\n",
      "236th Batch\n",
      "237th Batch\n",
      "238th Batch\n",
      "239th Batch\n",
      "240th Batch\n",
      "241th Batch\n",
      "242th Batch\n",
      "243th Batch\n",
      "244th Batch\n",
      "245th Batch\n",
      "246th Batch\n",
      "247th Batch\n",
      "248th Batch\n",
      "249th Batch\n",
      "250th Batch\n",
      "251th Batch\n",
      "252th Batch\n",
      "253th Batch\n",
      "254th Batch\n",
      "255th Batch\n",
      "256th Batch\n",
      "257th Batch\n",
      "258th Batch\n",
      "259th Batch\n",
      "260th Batch\n",
      "261th Batch\n",
      "262th Batch\n",
      "263th Batch\n",
      "264th Batch\n",
      "265th Batch\n",
      "266th Batch\n",
      "267th Batch\n",
      "268th Batch\n",
      "269th Batch\n",
      "270th Batch\n",
      "271th Batch\n",
      "272th Batch\n",
      "273th Batch\n",
      "274th Batch\n",
      "275th Batch\n",
      "276th Batch\n",
      "277th Batch\n",
      "278th Batch\n",
      "279th Batch\n",
      "280th Batch\n",
      "281th Batch\n",
      "282th Batch\n",
      "283th Batch\n",
      "284th Batch\n",
      "285th Batch\n",
      "286th Batch\n",
      "287th Batch\n",
      "288th Batch\n",
      "289th Batch\n",
      "290th Batch\n",
      "291th Batch\n",
      "292th Batch\n",
      "293th Batch\n",
      "294th Batch\n",
      "295th Batch\n",
      "296th Batch\n",
      "297th Batch\n",
      "298th Batch\n",
      "299th Batch\n",
      "300th Batch\n",
      "301th Batch\n",
      "302th Batch\n",
      "303th Batch\n",
      "304th Batch\n",
      "305th Batch\n",
      "306th Batch\n",
      "307th Batch\n",
      "308th Batch\n",
      "309th Batch\n",
      "310th Batch\n",
      "311th Batch\n",
      "312th Batch\n",
      "313th Batch\n",
      "314th Batch\n",
      "315th Batch\n",
      "316th Batch\n",
      "317th Batch\n",
      "318th Batch\n",
      "319th Batch\n",
      "320th Batch\n"
     ]
    }
   ],
   "source": [
    "prev_crash_i = 0 # Set it to the value of i where the previous running of this cell crashed, if running for first time set it to 0\n",
    "for i in range(prev_crash_i, 16000//BATCH_SIZE):\n",
    "    print(f'{i+1}th Batch')\n",
    "    batch = train_generator_precompute[i]\n",
    "    batch = [x for x in batch[0] if len(x.shape) == 4] # Sometimes a document image has no ROI handles that\n",
    "    batch = tf.concat(batch, axis=0) # Concatenating the ROIs of each document image togetheraq\n",
    "    precompute_output.append(model.predict(batch))\n",
    "    tf.keras.backend.clear_session()\n",
    "    _ = gc.collect()\n",
    "    if (i+1) % checkpnt_i == 0:\n",
    "        precompute_output = tf.concat(precompute_output, axis=0).numpy()\n",
    "        # np.save(f'./Data/EffNet-kP-train-precompute/train_data_precomp-{(i+1)//checkpnt_i}.npy', precompute_output)\n",
    "        np.save(f'train_data_precomp-{(i+1)//checkpnt_i}.npy', precompute_output)\n",
    "        precompute_output = list()\n",
    "        del batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7aebb93f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:12.813188Z",
     "iopub.status.busy": "2022-10-08T12:21:12.812888Z",
     "iopub.status.idle": "2022-10-08T12:21:12.817161Z",
     "shell.execute_reply": "2022-10-08T12:21:12.816184Z"
    },
    "papermill": {
     "duration": 0.035222,
     "end_time": "2022-10-08T12:21:12.819495",
     "exception": false,
     "start_time": "2022-10-08T12:21:12.784273",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for i in tqdm(range(1, 17)):\n",
    "#     with ZipFile(f'result-{i}.zip', 'w') as zipObj:\n",
    "#         for j in range(1, 11):\n",
    "#             zipObj.write(f'./train_data_precomp-{(i*10+j)-10}.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a210423b",
   "metadata": {
    "papermill": {
     "duration": 0.027929,
     "end_time": "2022-10-08T12:21:12.874220",
     "exception": false,
     "start_time": "2022-10-08T12:21:12.846291",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "After the feature vectors being generated for each ROI. We will now try different methods to combine the RoIs for each document to come up with a better representation vector for the overall document image which will then go into the fine-tuning extra layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "225702e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:12.929810Z",
     "iopub.status.busy": "2022-10-08T12:21:12.929267Z",
     "iopub.status.idle": "2022-10-08T12:21:12.933814Z",
     "shell.execute_reply": "2022-10-08T12:21:12.932920Z"
    },
    "papermill": {
     "duration": 0.033913,
     "end_time": "2022-10-08T12:21:12.935803",
     "exception": false,
     "start_time": "2022-10-08T12:21:12.901890",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# roi_count = [np.sum([roi[-1]==1 for roi in doc]) for doc in roi_info] # RoIs per document Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "32865ae2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:12.990754Z",
     "iopub.status.busy": "2022-10-08T12:21:12.990194Z",
     "iopub.status.idle": "2022-10-08T12:21:12.994875Z",
     "shell.execute_reply": "2022-10-08T12:21:12.994057Z"
    },
    "papermill": {
     "duration": 0.034498,
     "end_time": "2022-10-08T12:21:12.997000",
     "exception": false,
     "start_time": "2022-10-08T12:21:12.962502",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# max(roi_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "89b1d728",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.051694Z",
     "iopub.status.busy": "2022-10-08T12:21:13.051422Z",
     "iopub.status.idle": "2022-10-08T12:21:13.055331Z",
     "shell.execute_reply": "2022-10-08T12:21:13.054417Z"
    },
    "papermill": {
     "duration": 0.03367,
     "end_time": "2022-10-08T12:21:13.057518",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.023848",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data = list()\n",
    "# for i in range(16000 // (BATCH_SIZE*checkpnt_i)):\n",
    "#     k = 0\n",
    "#     batch_data = np.load(f'./train_data_precomp-{i+1}.npy')\n",
    "#     batch_roi = roi_count[(i*(BATCH_SIZE*checkpnt_i)):((i+1)*(BATCH_SIZE*checkpnt_i))]\n",
    "#     for j in range(BATCH_SIZE*checkpnt_i):\n",
    "#         fine_tune_data.append(np.concatenate([batch_data[k:(k+batch_roi[j])], np.zeros((100-batch_roi[j], 1280))])[np.newaxis,:])\n",
    "#         k += batch_roi[j]\n",
    "# fine_tune_data = np.concatenate(fine_tune_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "161ecb28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.112757Z",
     "iopub.status.busy": "2022-10-08T12:21:13.112009Z",
     "iopub.status.idle": "2022-10-08T12:21:13.116224Z",
     "shell.execute_reply": "2022-10-08T12:21:13.115380Z"
    },
    "papermill": {
     "duration": 0.033649,
     "end_time": "2022-10-08T12:21:13.118322",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.084673",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ab1c1ecd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.173651Z",
     "iopub.status.busy": "2022-10-08T12:21:13.173325Z",
     "iopub.status.idle": "2022-10-08T12:21:13.177760Z",
     "shell.execute_reply": "2022-10-08T12:21:13.176797Z"
    },
    "papermill": {
     "duration": 0.034097,
     "end_time": "2022-10-08T12:21:13.179693",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.145596",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# np.save(f'./Data/EffNet-kP-train-precompute/whole_train_data_precomp.npy', fine_tune_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30746064",
   "metadata": {
    "papermill": {
     "duration": 0.026925,
     "end_time": "2022-10-08T12:21:13.233390",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.206465",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The EfficientNetV2L gives a 3D output per image with $1280$ channels which are collapsed using GlobalAveragePooling2D and we get a 1280 representation vector per ROI. Now, since there are multiple ROIs per image we need to come up with a single vector containing all these information, later we will see a method to not throw this information away. But as of now this single vector summarizing the ROI feature vectors is calculated using average. We will combine this information with the $4$ Piece fine-tune data i.e. we will stack this ROI feature vector with the those 4 image piece feature vectors and a final feature vector for the entire image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b84500e7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.290310Z",
     "iopub.status.busy": "2022-10-08T12:21:13.288941Z",
     "iopub.status.idle": "2022-10-08T12:21:13.293199Z",
     "shell.execute_reply": "2022-10-08T12:21:13.292381Z"
    },
    "papermill": {
     "duration": 0.034531,
     "end_time": "2022-10-08T12:21:13.295085",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.260554",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data = np.load(f'./Data/EffNet-kP-train-precompute/whole_train_data_precomp.npy')\n",
    "# fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5dfcb54e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.351141Z",
     "iopub.status.busy": "2022-10-08T12:21:13.349682Z",
     "iopub.status.idle": "2022-10-08T12:21:13.354115Z",
     "shell.execute_reply": "2022-10-08T12:21:13.353269Z"
    },
    "papermill": {
     "duration": 0.034022,
     "end_time": "2022-10-08T12:21:13.356137",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.322115",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data = np.squeeze(np.mean(fine_tune_data, axis=1, keepdims=True))\n",
    "# fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3cad7c86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.411417Z",
     "iopub.status.busy": "2022-10-08T12:21:13.410677Z",
     "iopub.status.idle": "2022-10-08T12:21:13.414789Z",
     "shell.execute_reply": "2022-10-08T12:21:13.413979Z"
    },
    "papermill": {
     "duration": 0.033705,
     "end_time": "2022-10-08T12:21:13.416823",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.383118",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data_4P = np.load('./Data/EffNet-4P-train-precompute/whole_train_data_precomp.npy')\n",
    "# fine_tune_data_4P = np.reshape(fine_tune_data_4P, (16000, -1))\n",
    "# fine_tune_data_4P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "98675c08",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.472647Z",
     "iopub.status.busy": "2022-10-08T12:21:13.472384Z",
     "iopub.status.idle": "2022-10-08T12:21:13.476190Z",
     "shell.execute_reply": "2022-10-08T12:21:13.475218Z"
    },
    "papermill": {
     "duration": 0.034151,
     "end_time": "2022-10-08T12:21:13.478225",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.444074",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data = np.concatenate([fine_tune_data, fine_tune_data_4P], axis=-1)\n",
    "# fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "403786bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.533450Z",
     "iopub.status.busy": "2022-10-08T12:21:13.532710Z",
     "iopub.status.idle": "2022-10-08T12:21:13.537092Z",
     "shell.execute_reply": "2022-10-08T12:21:13.536058Z"
    },
    "papermill": {
     "duration": 0.033892,
     "end_time": "2022-10-08T12:21:13.539086",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.505194",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# np.save(f'./Data/EffNet-kP-train-precompute/whole_train_data_precomp_roi_n_4p.npy', fine_tune_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bf8b508a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.594469Z",
     "iopub.status.busy": "2022-10-08T12:21:13.594195Z",
     "iopub.status.idle": "2022-10-08T12:21:13.598141Z",
     "shell.execute_reply": "2022-10-08T12:21:13.597143Z"
    },
    "papermill": {
     "duration": 0.033549,
     "end_time": "2022-10-08T12:21:13.600064",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.566515",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_labels = labels['label'].to_numpy()\n",
    "# fine_tune_labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8568b0fb",
   "metadata": {
    "papermill": {
     "duration": 0.026568,
     "end_time": "2022-10-08T12:21:13.653316",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.626748",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Validation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2b3825b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.708752Z",
     "iopub.status.busy": "2022-10-08T12:21:13.708217Z",
     "iopub.status.idle": "2022-10-08T12:21:13.932916Z",
     "shell.execute_reply": "2022-10-08T12:21:13.931991Z"
    },
    "papermill": {
     "duration": 0.254455,
     "end_time": "2022-10-08T12:21:13.934945",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.680490",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>images</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17801</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17802</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>777</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17803</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17804</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17805</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18696</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18697</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18698</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>710</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18699</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18700</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                images     id  label\n",
       "57   ../input/datathonindoml-2022/validation/valida...  17801     -1\n",
       "141  ../input/datathonindoml-2022/validation/valida...  17802     -1\n",
       "777  ../input/datathonindoml-2022/validation/valida...  17803     -1\n",
       "273  ../input/datathonindoml-2022/validation/valida...  17804     -1\n",
       "30   ../input/datathonindoml-2022/validation/valida...  17805     -1\n",
       "..                                                 ...    ...    ...\n",
       "396  ../input/datathonindoml-2022/validation/valida...  18696     -1\n",
       "111  ../input/datathonindoml-2022/validation/valida...  18697     -1\n",
       "817  ../input/datathonindoml-2022/validation/valida...  18698     -1\n",
       "710  ../input/datathonindoml-2022/validation/valida...  18699     -1\n",
       "293  ../input/datathonindoml-2022/validation/valida...  18700     -1\n",
       "\n",
       "[900 rows x 3 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_data = pd.DataFrame({'images':['../input/datathonindoml-2022/validation/validation/'+name for name in os.listdir('../input/datathonindoml-2022/validation/validation/')]})\n",
    "validation_data['id'] = [name.split('.')[0] for name in os.listdir('../input/datathonindoml-2022/validation/validation/')]\n",
    "validation_data['label'] = -1 # Simply added to prevent re-writing code\n",
    "validation_data.sort_values(by=['id'], inplace=True)\n",
    "validation_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0b2262d8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:13.992436Z",
     "iopub.status.busy": "2022-10-08T12:21:13.991661Z",
     "iopub.status.idle": "2022-10-08T12:21:14.049212Z",
     "shell.execute_reply": "2022-10-08T12:21:14.048343Z"
    },
    "papermill": {
     "duration": 0.088579,
     "end_time": "2022-10-08T12:21:14.051320",
     "exception": false,
     "start_time": "2022-10-08T12:21:13.962741",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 100, 6)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_info_validation = np.load('../input/roiinfo/validation_roi_viz.npy')\n",
    "roi_info_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2570eae0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:14.108478Z",
     "iopub.status.busy": "2022-10-08T12:21:14.107704Z",
     "iopub.status.idle": "2022-10-08T12:21:14.113784Z",
     "shell.execute_reply": "2022-10-08T12:21:14.112970Z"
    },
    "papermill": {
     "duration": 0.036867,
     "end_time": "2022-10-08T12:21:14.116111",
     "exception": false,
     "start_time": "2022-10-08T12:21:14.079244",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "validation_generator_precompute = ImageDataGenerator(\n",
    "    df=validation_data,\n",
    "    X_col='images',\n",
    "    roi_info=roi_info_validation,\n",
    "    y_col='label',\n",
    "    batch_size=BATCH_SIZE,\n",
    "    input_size=INPUT_SHAPE,\n",
    "    shuffle=False,\n",
    "    base=17801\n",
    ")\n",
    "\n",
    "checkpnt_each = 100\n",
    "assert checkpnt_each % BATCH_SIZE == 0\n",
    "checkpnt_i = checkpnt_each // BATCH_SIZE\n",
    "precompute_output = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2656e0bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:21:14.173514Z",
     "iopub.status.busy": "2022-10-08T12:21:14.172113Z",
     "iopub.status.idle": "2022-10-08T12:27:50.842752Z",
     "shell.execute_reply": "2022-10-08T12:27:50.841709Z"
    },
    "papermill": {
     "duration": 396.701568,
     "end_time": "2022-10-08T12:27:50.845228",
     "exception": false,
     "start_time": "2022-10-08T12:21:14.143660",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1th Batch\n",
      "2th Batch\n",
      "3th Batch\n",
      "4th Batch\n",
      "5th Batch\n",
      "6th Batch\n",
      "7th Batch\n",
      "8th Batch\n",
      "9th Batch\n",
      "10th Batch\n",
      "11th Batch\n",
      "12th Batch\n",
      "13th Batch\n",
      "14th Batch\n",
      "15th Batch\n",
      "16th Batch\n",
      "17th Batch\n",
      "18th Batch\n"
     ]
    }
   ],
   "source": [
    "prev_crash_i = 0 # Set it to the value of i where the previous running of this cell crashed, if running for first time set it to 0\n",
    "for i in range(prev_crash_i, 900//BATCH_SIZE):\n",
    "    print(f'{i+1}th Batch')\n",
    "    batch = validation_generator_precompute[i]\n",
    "    batch = [x for x in batch[0] if len(x.shape) == 4] # Sometimes a document image has no ROI handles that\n",
    "    batch = tf.concat(batch, axis=0) # Concatenating the ROIs of each document image together\n",
    "    precompute_output.append(model.predict(batch))\n",
    "    precompute_output.append(model.predict(batch))\n",
    "    tf.keras.backend.clear_session()\n",
    "    _ = gc.collect()\n",
    "    if (i+1) % checkpnt_i == 0:\n",
    "        precompute_output = tf.concat(precompute_output, axis=0).numpy()\n",
    "        np.save(f'validation_data_precomp-{(i+1)//checkpnt_i}.npy', precompute_output)\n",
    "        precompute_output = list()\n",
    "        del batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "339fcdb3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T12:27:50.904871Z",
     "iopub.status.busy": "2022-10-08T12:27:50.903964Z",
     "iopub.status.idle": "2022-10-08T12:27:50.908474Z",
     "shell.execute_reply": "2022-10-08T12:27:50.907533Z"
    },
    "papermill": {
     "duration": 0.035883,
     "end_time": "2022-10-08T12:27:50.910348",
     "exception": false,
     "start_time": "2022-10-08T12:27:50.874465",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# with ZipFile(f'result.zip', 'w') as zipObj:\n",
    "#     for j in range(1, 10):\n",
    "#         zipObj.write(f'./validation_data_precomp-{j}.npy')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 5397.866964,
   "end_time": "2022-10-08T12:27:54.230771",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-10-08T10:57:56.363807",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}