resnet50v2-4piece-feature-extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "af0d9f61",
   "metadata": {
    "papermill": {
     "duration": 0.008476,
     "end_time": "2022-10-08T06:01:20.114940",
     "exception": false,
     "start_time": "2022-10-08T06:01:20.106464",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49b75420",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:20.132726Z",
     "iopub.status.busy": "2022-10-08T06:01:20.131544Z",
     "iopub.status.idle": "2022-10-08T06:01:25.804430Z",
     "shell.execute_reply": "2022-10-08T06:01:25.803469Z"
    },
    "papermill": {
     "duration": 5.684092,
     "end_time": "2022-10-08T06:01:25.807065",
     "exception": false,
     "start_time": "2022-10-08T06:01:20.122973",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import load_img, img_to_array\n",
    "\n",
    "from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAvgPool2D, BatchNormalization\n",
    "from tensorflow.keras.applications import ResNet50V2\n",
    "from tensorflow.keras.applications.resnet_v2 import preprocess_input\n",
    "from tensorflow.keras.regularizers import l2\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support as score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e8a202b",
   "metadata": {
    "papermill": {
     "duration": 0.007675,
     "end_time": "2022-10-08T06:01:25.822143",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.814468",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe6fd1ef",
   "metadata": {
    "papermill": {
     "duration": 0.006905,
     "end_time": "2022-10-08T06:01:25.836035",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.829130",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "A detailed discussion and visualization of the data can be seen in [here](Data-Overview.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2ca16bcf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:25.851441Z",
     "iopub.status.busy": "2022-10-08T06:01:25.850925Z",
     "iopub.status.idle": "2022-10-08T06:01:25.886746Z",
     "shell.execute_reply": "2022-10-08T06:01:25.885859Z"
    },
    "papermill": {
     "duration": 0.045985,
     "end_time": "2022-10-08T06:01:25.888972",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.842987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "labels = pd.read_csv(\"../input/datathonindoml-2022/train_labels.csv\")\n",
    "images = ['../input/datathonindoml-2022/train/train/'+str(name)+'.tif' for name in labels['id']]\n",
    "labels['images'] = images\n",
    "labels = labels[['id', 'images', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea9c20f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:25.906022Z",
     "iopub.status.busy": "2022-10-08T06:01:25.905717Z",
     "iopub.status.idle": "2022-10-08T06:01:25.923786Z",
     "shell.execute_reply": "2022-10-08T06:01:25.922802Z"
    },
    "papermill": {
     "duration": 0.028108,
     "end_time": "2022-10-08T06:01:25.925751",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.897643",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/0.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/1.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/2.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/3.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/4.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15995</th>\n",
       "      <td>15995</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15995...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15996</th>\n",
       "      <td>15996</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15996...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15997</th>\n",
       "      <td>15997</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15997...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15998</th>\n",
       "      <td>15998</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15998...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15999</th>\n",
       "      <td>15999</td>\n",
       "      <td>../input/datathonindoml-2022/train/train/15999...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                             images  label\n",
       "0          0     ../input/datathonindoml-2022/train/train/0.tif      1\n",
       "1          1     ../input/datathonindoml-2022/train/train/1.tif     13\n",
       "2          2     ../input/datathonindoml-2022/train/train/2.tif     13\n",
       "3          3     ../input/datathonindoml-2022/train/train/3.tif     14\n",
       "4          4     ../input/datathonindoml-2022/train/train/4.tif      6\n",
       "...      ...                                                ...    ...\n",
       "15995  15995  ../input/datathonindoml-2022/train/train/15995...      2\n",
       "15996  15996  ../input/datathonindoml-2022/train/train/15996...     15\n",
       "15997  15997  ../input/datathonindoml-2022/train/train/15997...      3\n",
       "15998  15998  ../input/datathonindoml-2022/train/train/15998...      9\n",
       "15999  15999  ../input/datathonindoml-2022/train/train/15999...      9\n",
       "\n",
       "[16000 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9cb38eb7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:25.942264Z",
     "iopub.status.busy": "2022-10-08T06:01:25.941394Z",
     "iopub.status.idle": "2022-10-08T06:01:25.945686Z",
     "shell.execute_reply": "2022-10-08T06:01:25.944835Z"
    },
    "papermill": {
     "duration": 0.014476,
     "end_time": "2022-10-08T06:01:25.947632",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.933156",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# test_labels = labels.sample(800).sort_values(by='id')\n",
    "# labels = labels[~labels['id'].isin(list(test_labels['id']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59aec2c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:25.964144Z",
     "iopub.status.busy": "2022-10-08T06:01:25.963347Z",
     "iopub.status.idle": "2022-10-08T06:01:25.967552Z",
     "shell.execute_reply": "2022-10-08T06:01:25.966722Z"
    },
    "papermill": {
     "duration": 0.014606,
     "end_time": "2022-10-08T06:01:25.969407",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.954801",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eca239ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:25.985712Z",
     "iopub.status.busy": "2022-10-08T06:01:25.984806Z",
     "iopub.status.idle": "2022-10-08T06:01:25.989006Z",
     "shell.execute_reply": "2022-10-08T06:01:25.988187Z"
    },
    "papermill": {
     "duration": 0.014018,
     "end_time": "2022-10-08T06:01:25.990942",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.976924",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "35391e88",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:26.008028Z",
     "iopub.status.busy": "2022-10-08T06:01:26.007250Z",
     "iopub.status.idle": "2022-10-08T06:01:26.011257Z",
     "shell.execute_reply": "2022-10-08T06:01:26.010451Z"
    },
    "papermill": {
     "duration": 0.013805,
     "end_time": "2022-10-08T06:01:26.013195",
     "exception": false,
     "start_time": "2022-10-08T06:01:25.999390",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# labels.to_csv('./Data/train_labels_1.csv', index=False)\n",
    "# test_labels.to_csv('./Data/test_labels_1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2b339541",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:26.029279Z",
     "iopub.status.busy": "2022-10-08T06:01:26.028502Z",
     "iopub.status.idle": "2022-10-08T06:01:26.036227Z",
     "shell.execute_reply": "2022-10-08T06:01:26.035162Z"
    },
    "papermill": {
     "duration": 0.017585,
     "end_time": "2022-10-08T06:01:26.038212",
     "exception": false,
     "start_time": "2022-10-08T06:01:26.020627",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class_labels = list(set(labels['label']))\n",
    "class_names = [\n",
    "    'letter', 'form', 'email', 'handwritten', 'advertisement', 'scientific report', 'scientific publication',\n",
    "    'specification', 'file folder', 'news article', 'budget', 'invoice', 'presentation', 'questionnaire', 'resume',\n",
    "    'memo'\n",
    "]\n",
    "label_names = pd.DataFrame({\n",
    "    'labels': class_labels,\n",
    "    'names': class_names\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feaa676d",
   "metadata": {
    "papermill": {
     "duration": 0.007064,
     "end_time": "2022-10-08T06:01:26.052511",
     "exception": false,
     "start_time": "2022-10-08T06:01:26.045447",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a761c35",
   "metadata": {
    "papermill": {
     "duration": 0.006986,
     "end_time": "2022-10-08T06:01:26.066984",
     "exception": false,
     "start_time": "2022-10-08T06:01:26.059998",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Since, there is a lot of image data instead of using the whole data at once in a tensor form, we would be using a data generator to prevent memory issues."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f3b1182a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:26.083216Z",
     "iopub.status.busy": "2022-10-08T06:01:26.082423Z",
     "iopub.status.idle": "2022-10-08T06:01:28.921668Z",
     "shell.execute_reply": "2022-10-08T06:01:28.920661Z"
    },
    "papermill": {
     "duration": 2.850504,
     "end_time": "2022-10-08T06:01:28.924696",
     "exception": false,
     "start_time": "2022-10-08T06:01:26.074192",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 06:01:26.222542: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:26.316765: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:26.317582: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:26.324987: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-10-08 06:01:26.325334: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:26.326055: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:26.326698: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:28.493202: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:28.494046: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:28.494719: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-10-08 06:01:28.495385: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15401 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Mean Image Height: 1000.0\n",
      "Mean Image Width: 765.0\n",
      "Min Image Height: 1000\n",
      "Min Image Width: 754\n"
     ]
    }
   ],
   "source": [
    "imgs = [tf.cast(img_to_array(load_img('../input/datathonindoml-2022/validation/validation/'+name)), dtype=tf.uint8) for name in os.listdir('../input/datathonindoml-2022/validation/validation')[:4]]\n",
    "heights = [img.shape[0] for img in imgs]\n",
    "widths = [img.shape[1] for img in imgs]\n",
    "print(\"\")\n",
    "print(\"Mean Image Height:\", np.mean(heights))\n",
    "print(\"Mean Image Width:\", np.mean(widths))\n",
    "print(\"Min Image Height:\", np.min(heights))\n",
    "print(\"Min Image Width:\", np.min(widths))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05aa9c2a",
   "metadata": {
    "papermill": {
     "duration": 0.007578,
     "end_time": "2022-10-08T06:01:28.940671",
     "exception": false,
     "start_time": "2022-10-08T06:01:28.933093",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Observing the average image heights and widths, it is observed that setting image height to $1000$ and image width to $750$ is reseonable for training the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a77ee815",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:28.957313Z",
     "iopub.status.busy": "2022-10-08T06:01:28.956999Z",
     "iopub.status.idle": "2022-10-08T06:01:28.961812Z",
     "shell.execute_reply": "2022-10-08T06:01:28.960768Z"
    },
    "papermill": {
     "duration": 0.016039,
     "end_time": "2022-10-08T06:01:28.964308",
     "exception": false,
     "start_time": "2022-10-08T06:01:28.948269",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 40\n",
    "INPUT_SHAPE = (800, 600) # Full Image\n",
    "INPUT_HDR_SHAPE = INPUT_FTR_SHAPE = (250, 600)\n",
    "INPUT_BDL_SHAPE = INPUT_BDR_SHAPE = (300, 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "588258e5",
   "metadata": {
    "papermill": {
     "duration": 0.007404,
     "end_time": "2022-10-08T06:01:28.979195",
     "exception": false,
     "start_time": "2022-10-08T06:01:28.971791",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Since, the Keras' own data generator does not support TIFF images, we resort to writing our own custom data generator. It can also be leveraged to put custom image pre-processing or multi-input or output in the data processing pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25ba6753",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:28.995467Z",
     "iopub.status.busy": "2022-10-08T06:01:28.994628Z",
     "iopub.status.idle": "2022-10-08T06:01:29.008099Z",
     "shell.execute_reply": "2022-10-08T06:01:29.007243Z"
    },
    "papermill": {
     "duration": 0.0235,
     "end_time": "2022-10-08T06:01:29.009973",
     "exception": false,
     "start_time": "2022-10-08T06:01:28.986473",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ImageDataGenerator(tf.keras.utils.Sequence):\n",
    "    def __init__(self, df, X_col, y_col, batch_size, input_size=(800, 600), shuffle=True):\n",
    "        self.df = df.copy() # DataFrame consisting image paths of inputs and the labels for the outputs\n",
    "        self.X_col = X_col # Input column, specifying image path, in the DataFrame\n",
    "        self.y_col = y_col # Output column, specifying corresponding label, in the DataFrame\n",
    "        self.batch_size = batch_size # Batch Size\n",
    "        self.input_size = input_size # Input Image size\n",
    "        self.shuffle = shuffle # Shuffle Data after each epoch\n",
    "        self.n = len(self.df) # length of the entire data\n",
    "    \n",
    "    def on_epoch_end(self):\n",
    "        if self.shuffle:\n",
    "            self.df = self.df.sample(frac=1).reset_index(drop=True)\n",
    "    \n",
    "    def __get_input(self, path):\n",
    "        img_arr = img_to_array(load_img(path))\n",
    "        # some other pre-processing / data-augmentation goes here\n",
    "        img_arr = tf.image.resize(img_arr, self.input_size)\n",
    "        img_hdr, img_bdl, img_bdr, img_ftr = img_arr[:250], img_arr[250:-250, :300], img_arr[250:-250, -300:], img_arr[-250:]\n",
    "        return [preprocess_input(img_hdr), preprocess_input(img_bdl), preprocess_input(img_bdr), preprocess_input(img_ftr), preprocess_input(img_arr)]\n",
    "    \n",
    "    def __get_data(self, batches):\n",
    "        batch_paths = batches[self.X_col]\n",
    "        batch_labels = batches[self.y_col]\n",
    "        X_batch_4 = [self.__get_input(path) for path in batch_paths]\n",
    "        X_batch = tf.cast([img[0] for img in X_batch_4], dtype=tf.float32), tf.cast([img[1] for img in X_batch_4], dtype=tf.float32), tf.cast([img[2] for img in X_batch_4], dtype=tf.float32), tf.cast([img[3] for img in X_batch_4], dtype=tf.float32), tf.cast([img[4] for img in X_batch_4], dtype=tf.float32)\n",
    "        y_batch = tf.cast(batch_labels, dtype=tf.float32)\n",
    "        return X_batch, y_batch\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "        X, y = self.__get_data(batches)\n",
    "        return X, y\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.n // self.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ecf34786",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:29.026296Z",
     "iopub.status.busy": "2022-10-08T06:01:29.025526Z",
     "iopub.status.idle": "2022-10-08T06:01:29.030934Z",
     "shell.execute_reply": "2022-10-08T06:01:29.030083Z"
    },
    "papermill": {
     "duration": 0.015403,
     "end_time": "2022-10-08T06:01:29.032886",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.017483",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_generator = ImageDataGenerator(\n",
    "    df=labels,\n",
    "    X_col='images',\n",
    "    y_col='label',\n",
    "    batch_size=BATCH_SIZE,\n",
    "    input_size=INPUT_SHAPE,\n",
    "    shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29acf420",
   "metadata": {
    "papermill": {
     "duration": 0.007094,
     "end_time": "2022-10-08T06:01:29.047544",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.040450",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Below shows one batch from the ImageDataGenerator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "254d8779",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:29.063609Z",
     "iopub.status.busy": "2022-10-08T06:01:29.063352Z",
     "iopub.status.idle": "2022-10-08T06:01:29.663421Z",
     "shell.execute_reply": "2022-10-08T06:01:29.659113Z"
    },
    "papermill": {
     "duration": 0.615403,
     "end_time": "2022-10-08T06:01:29.670246",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.054843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Batch Overview ***\n",
      "Number of Inputs: 5\n",
      "Input Image Header Tensor Shape: (40, 250, 600, 3)\n",
      "Input Image Body(L) Tensor Shape: (40, 300, 300, 3)\n",
      "Input Image Body(R) Tensor Shape: (40, 300, 300, 3)\n",
      "Input Image Footer Tensor Shape: (40, 250, 600, 3)\n",
      "Input Image Full Tensor Shape: (40, 800, 600, 3)\n",
      "Output Label Tensor Shape: (40,)\n"
     ]
    }
   ],
   "source": [
    "batch = next(iter(train_generator))\n",
    "print('*** Batch Overview ***')\n",
    "print('Number of Inputs:', len(batch[0]))\n",
    "print('Input Image Header Tensor Shape:', batch[0][0].shape)\n",
    "print('Input Image Body(L) Tensor Shape:', batch[0][1].shape)\n",
    "print('Input Image Body(R) Tensor Shape:', batch[0][2].shape)\n",
    "print('Input Image Footer Tensor Shape:', batch[0][3].shape)\n",
    "print('Input Image Full Tensor Shape:', batch[0][4].shape)\n",
    "print('Output Label Tensor Shape:', batch[1].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cadfccb",
   "metadata": {
    "papermill": {
     "duration": 0.007336,
     "end_time": "2022-10-08T06:01:29.686707",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.679371",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Each row below shows the $4$ parts of the same image and the full image, where the first being the header, second being the body(left), third is body(right) and last one is footer and the full image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9fb30ee2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:29.703065Z",
     "iopub.status.busy": "2022-10-08T06:01:29.702731Z",
     "iopub.status.idle": "2022-10-08T06:01:29.707385Z",
     "shell.execute_reply": "2022-10-08T06:01:29.706398Z"
    },
    "papermill": {
     "duration": 0.015347,
     "end_time": "2022-10-08T06:01:29.709516",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.694169",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots(BATCH_SIZE//2, 5, figsize=(12, 60))\n",
    "# imgs = batch[0]\n",
    "# labs = list(batch[1].numpy())\n",
    "# for j in range(BATCH_SIZE//2):\n",
    "#     for k in range(5):\n",
    "#         ax[j, k].imshow((imgs[k][j]*255).numpy().astype(np.uint8))\n",
    "#         ax[j, k].axis('off')\n",
    "#         ax[j, k].set_title(labs[j])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83d081a9",
   "metadata": {
    "papermill": {
     "duration": 0.007327,
     "end_time": "2022-10-08T06:01:29.724784",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.717457",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model Building"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "013baf3c",
   "metadata": {
    "papermill": {
     "duration": 0.00749,
     "end_time": "2022-10-08T06:01:29.739770",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.732280",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now, a basic CNN-based model will be put to place. It will utilize the visual features only to classify the documents. Later we will be building much more better models considering other structures and multi-modality of the images and distinctive features."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd55a464",
   "metadata": {
    "papermill": {
     "duration": 0.007181,
     "end_time": "2022-10-08T06:01:29.754324",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.747143",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- ResNet50V2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05172bff",
   "metadata": {
    "papermill": {
     "duration": 0.007147,
     "end_time": "2022-10-08T06:01:29.769151",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.762004",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The ResNet50V2 is a large model and since, we are not pre-training the entire model, we will just fine-tune it with two extra layers. So, to fasten training we precompute the output of the ResNet50V2 model and use this for training the added Dense Layers for Fine-Tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "44af5f63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:29.785719Z",
     "iopub.status.busy": "2022-10-08T06:01:29.785367Z",
     "iopub.status.idle": "2022-10-08T06:01:31.751107Z",
     "shell.execute_reply": "2022-10-08T06:01:31.749963Z"
    },
    "papermill": {
     "duration": 1.97717,
     "end_time": "2022-10-08T06:01:31.753971",
     "exception": false,
     "start_time": "2022-10-08T06:01:29.776801",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "94674944/94668760 [==============================] - 1s 0us/step\n",
      "94683136/94668760 [==============================] - 1s 0us/step\n"
     ]
    }
   ],
   "source": [
    "model_resnet = ResNet50V2(include_top=False, weights='imagenet')\n",
    "model_resnet.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "690ff453",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:31.782086Z",
     "iopub.status.busy": "2022-10-08T06:01:31.781687Z",
     "iopub.status.idle": "2022-10-08T06:01:33.479740Z",
     "shell.execute_reply": "2022-10-08T06:01:33.478794Z"
    },
    "papermill": {
     "duration": 1.714407,
     "end_time": "2022-10-08T06:01:33.482078",
     "exception": false,
     "start_time": "2022-10-08T06:01:31.767671",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = [Input(shape=(*INPUT_HDR_SHAPE, 3)), Input(shape=(*INPUT_BDL_SHAPE, 3)), Input(shape=(*INPUT_BDR_SHAPE, 3)), Input(shape=(*INPUT_FTR_SHAPE, 3)), Input(shape=(*INPUT_SHAPE, 3))]\n",
    "outputs = [model_resnet(inp) for inp in inputs]\n",
    "outputs = [GlobalAvgPool2D()(out) for out in outputs]\n",
    "model = tf.keras.Model(inputs, outputs, name='ResNet-4Piece')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f0a64ea4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:33.500592Z",
     "iopub.status.busy": "2022-10-08T06:01:33.500302Z",
     "iopub.status.idle": "2022-10-08T06:01:33.514152Z",
     "shell.execute_reply": "2022-10-08T06:01:33.513102Z"
    },
    "papermill": {
     "duration": 0.02687,
     "end_time": "2022-10-08T06:01:33.517716",
     "exception": false,
     "start_time": "2022-10-08T06:01:33.490846",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"ResNet-4Piece\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 250, 600, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 300, 300, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            [(None, 300, 300, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            [(None, 250, 600, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            [(None, 800, 600, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "resnet50v2 (Functional)         (None, None, None, 2 23564800    input_2[0][0]                    \n",
      "                                                                 input_3[0][0]                    \n",
      "                                                                 input_4[0][0]                    \n",
      "                                                                 input_5[0][0]                    \n",
      "                                                                 input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 2048)         0           resnet50v2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 2048)         0           resnet50v2[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 2048)         0           resnet50v2[2][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_3 (Glo (None, 2048)         0           resnet50v2[3][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_4 (Glo (None, 2048)         0           resnet50v2[4][0]                 \n",
      "==================================================================================================\n",
      "Total params: 23,564,800\n",
      "Trainable params: 0\n",
      "Non-trainable params: 23,564,800\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "14e5a47e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:01:33.535757Z",
     "iopub.status.busy": "2022-10-08T06:01:33.535490Z",
     "iopub.status.idle": "2022-10-08T06:14:42.133369Z",
     "shell.execute_reply": "2022-10-08T06:14:42.132215Z"
    },
    "papermill": {
     "duration": 788.610006,
     "end_time": "2022-10-08T06:14:42.136236",
     "exception": false,
     "start_time": "2022-10-08T06:01:33.526230",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch-1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-08 06:01:34.096479: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2022-10-08 06:01:38.757166: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch-2\n",
      "Batch-3\n",
      "Batch-4\n",
      "Batch-5\n",
      "Batch-6\n",
      "Batch-7\n",
      "Batch-8\n",
      "Batch-9\n",
      "Batch-10\n"
     ]
    }
   ],
   "source": [
    "precompute_ResNet_output = 0\n",
    "for idx in range(len(labels)//1600):\n",
    "    print(f\"Batch-{idx+1}\")\n",
    "    train_generator_precompute = ImageDataGenerator(\n",
    "        df=labels[idx*1600:(idx+1)*1600],\n",
    "        X_col='images',\n",
    "        y_col='label',\n",
    "        batch_size=BATCH_SIZE,\n",
    "        input_size=INPUT_SHAPE,\n",
    "        shuffle=False\n",
    "    )\n",
    "    precompute_ResNet_output = model.predict(train_generator_precompute)\n",
    "    tf.keras.backend.clear_session()\n",
    "    _ = gc.collect()\n",
    "    np.save(f'./train_data_precomp-{idx}.npy', precompute_ResNet_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "908dbbcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:14:42.157001Z",
     "iopub.status.busy": "2022-10-08T06:14:42.156657Z",
     "iopub.status.idle": "2022-10-08T06:14:42.160749Z",
     "shell.execute_reply": "2022-10-08T06:14:42.159800Z"
    },
    "papermill": {
     "duration": 0.016334,
     "end_time": "2022-10-08T06:14:42.162733",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.146399",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fine_tune_data = np.concatenate([np.transpose(np.load(f'./Data/EffNet-4P-train-precompute/train_data_precomp-{idx}.npy'), (1,0,2)) for idx in range(10)])\n",
    "# np.save(f'./Data/EffNet-4P-train-precompute/whole_train_data_precomp.npy', fine_tune_data)\n",
    "# fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0b4937f",
   "metadata": {
    "papermill": {
     "duration": 0.008979,
     "end_time": "2022-10-08T06:14:42.180970",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.171991",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The EfficientNetV2L gives a 3D output per image with $1280$ channels which are collapsed using GlobalAveragePooling2D and we get a 1280 representation vector per 4 parts of the image and full image ($4+1=5$) which will be used to train the later extra layers of our model."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60db55b8",
   "metadata": {
    "papermill": {
     "duration": 0.008858,
     "end_time": "2022-10-08T06:14:42.198966",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.190108",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### On Validation Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7b8e661",
   "metadata": {
    "papermill": {
     "duration": 0.008945,
     "end_time": "2022-10-08T06:14:42.216971",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.208026",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Generating Predictions for the Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6e99a69f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:14:42.236357Z",
     "iopub.status.busy": "2022-10-08T06:14:42.236084Z",
     "iopub.status.idle": "2022-10-08T06:14:42.248559Z",
     "shell.execute_reply": "2022-10-08T06:14:42.247709Z"
    },
    "papermill": {
     "duration": 0.024463,
     "end_time": "2022-10-08T06:14:42.250469",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.226006",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ImageDataGenerator(tf.keras.utils.Sequence):\n",
    "    def __init__(self, df, X_col, batch_size, input_size=(800, 600), shuffle=True):\n",
    "        self.df = df.copy() # DataFrame consisting image paths of inputs and the labels for the outputs\n",
    "        self.X_col = X_col # Input column, specifying image path, in the DataFrame\n",
    "        self.batch_size = batch_size # Batch Size\n",
    "        self.input_size = input_size # Input Image size\n",
    "        self.shuffle = shuffle # Shuffle Data after each epoch\n",
    "        self.n = len(self.df) # length of the entire data\n",
    "    \n",
    "    def on_epoch_end(self):\n",
    "        if self.shuffle:\n",
    "            self.df = self.df.sample(frac=1).reset_index(drop=True)\n",
    "    \n",
    "    def __get_input(self, path):\n",
    "        img_arr = img_to_array(load_img(path))\n",
    "        # some other pre-processing / data-augmentation goes here\n",
    "        img_arr = tf.image.resize(img_arr, self.input_size)\n",
    "        img_hdr, img_bdl, img_bdr, img_ftr = img_arr[:250], img_arr[250:-250, :300], img_arr[250:-250, -300:], img_arr[-250:]\n",
    "        return [preprocess_input(img_hdr), preprocess_input(img_bdl), preprocess_input(img_bdr), preprocess_input(img_ftr), preprocess_input(img_arr)]\n",
    "    \n",
    "    def __get_data(self, batches):\n",
    "        batch_paths = batches[self.X_col]\n",
    "        X_batch_4 = [self.__get_input(path) for path in batch_paths]\n",
    "        X_batch = tf.cast([img[0] for img in X_batch_4], dtype=tf.float32), tf.cast([img[1] for img in X_batch_4], dtype=tf.float32), tf.cast([img[2] for img in X_batch_4], dtype=tf.float32), tf.cast([img[3] for img in X_batch_4], dtype=tf.float32), tf.cast([img[4] for img in X_batch_4], dtype=tf.float32)\n",
    "        return (X_batch, )\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        batches = self.df[index * self.batch_size:(index + 1) * self.batch_size]\n",
    "        X = self.__get_data(batches)\n",
    "        return X\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.n // self.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d95537ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:14:42.270072Z",
     "iopub.status.busy": "2022-10-08T06:14:42.269318Z",
     "iopub.status.idle": "2022-10-08T06:14:42.287873Z",
     "shell.execute_reply": "2022-10-08T06:14:42.286937Z"
    },
    "papermill": {
     "duration": 0.030414,
     "end_time": "2022-10-08T06:14:42.289974",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.259560",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>images</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17801</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17802</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>777</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17803</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17804</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>17805</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18696</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18697</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18698</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>710</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18699</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>../input/datathonindoml-2022/validation/valida...</td>\n",
       "      <td>18700</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                images     id  label\n",
       "57   ../input/datathonindoml-2022/validation/valida...  17801     -1\n",
       "141  ../input/datathonindoml-2022/validation/valida...  17802     -1\n",
       "777  ../input/datathonindoml-2022/validation/valida...  17803     -1\n",
       "273  ../input/datathonindoml-2022/validation/valida...  17804     -1\n",
       "30   ../input/datathonindoml-2022/validation/valida...  17805     -1\n",
       "..                                                 ...    ...    ...\n",
       "396  ../input/datathonindoml-2022/validation/valida...  18696     -1\n",
       "111  ../input/datathonindoml-2022/validation/valida...  18697     -1\n",
       "817  ../input/datathonindoml-2022/validation/valida...  18698     -1\n",
       "710  ../input/datathonindoml-2022/validation/valida...  18699     -1\n",
       "293  ../input/datathonindoml-2022/validation/valida...  18700     -1\n",
       "\n",
       "[900 rows x 3 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_data = pd.DataFrame({'images':['../input/datathonindoml-2022/validation/validation/'+name for name in os.listdir('../input/datathonindoml-2022/validation/validation/')]})\n",
    "validation_data['id'] = [name.split('.')[0] for name in os.listdir('../input/datathonindoml-2022/validation/validation/')]\n",
    "validation_data['label'] = -1 # Simply added to prevent re-writing code\n",
    "validation_data.sort_values(by=['id'], inplace=True)\n",
    "validation_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ba76bbef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:14:42.310580Z",
     "iopub.status.busy": "2022-10-08T06:14:42.309093Z",
     "iopub.status.idle": "2022-10-08T06:14:42.315150Z",
     "shell.execute_reply": "2022-10-08T06:14:42.314318Z"
    },
    "papermill": {
     "duration": 0.017742,
     "end_time": "2022-10-08T06:14:42.316926",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.299184",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "valid_generator = ImageDataGenerator(\n",
    "    df=validation_data,\n",
    "    X_col='images',\n",
    "    batch_size=30,\n",
    "    shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7a0c5516",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:14:42.337081Z",
     "iopub.status.busy": "2022-10-08T06:14:42.336554Z",
     "iopub.status.idle": "2022-10-08T06:15:28.159195Z",
     "shell.execute_reply": "2022-10-08T06:15:28.158174Z"
    },
    "papermill": {
     "duration": 45.835397,
     "end_time": "2022-10-08T06:15:28.161609",
     "exception": false,
     "start_time": "2022-10-08T06:14:42.326212",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "valid_preds = model.predict(valid_generator)\n",
    "valid_preds = np.transpose(np.concatenate([dat[np.newaxis,:,:] for dat in valid_preds]), (1,0,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f6a0da29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:15:28.182072Z",
     "iopub.status.busy": "2022-10-08T06:15:28.181747Z",
     "iopub.status.idle": "2022-10-08T06:15:28.187836Z",
     "shell.execute_reply": "2022-10-08T06:15:28.186960Z"
    },
    "papermill": {
     "duration": 0.018367,
     "end_time": "2022-10-08T06:15:28.189849",
     "exception": false,
     "start_time": "2022-10-08T06:15:28.171482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 2048)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c856600e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-08T06:15:28.210151Z",
     "iopub.status.busy": "2022-10-08T06:15:28.209364Z",
     "iopub.status.idle": "2022-10-08T06:15:28.499875Z",
     "shell.execute_reply": "2022-10-08T06:15:28.498925Z"
    },
    "papermill": {
     "duration": 0.303082,
     "end_time": "2022-10-08T06:15:28.502254",
     "exception": false,
     "start_time": "2022-10-08T06:15:28.199172",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.save('./whole_validation_data_precomp.npy', valid_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "614b9bca",
   "metadata": {
    "papermill": {
     "duration": 0.009368,
     "end_time": "2022-10-08T06:15:28.521421",
     "exception": false,
     "start_time": "2022-10-08T06:15:28.512053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 858.616876,
   "end_time": "2022-10-08T06:15:31.357651",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-10-08T06:01:12.740775",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}