mnist_cnn_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d71b829b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "import timeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e50e5c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_data, train_labels), (test_data, test_labels) = tf.keras.datasets.fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b8abdee3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e940717a",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Shape, Dtype, Normalize, One-Hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "deed54d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.reshape((-1, 28, 28, 1)).astype(np.float32) / 255.\n",
    "test_data = test_data.reshape((-1, 28, 28, 1)).astype(np.float32) / 255.\n",
    "train_labels = to_categorical(train_labels, num_classes=10)\n",
    "test_labels = to_categorical(test_labels, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d358eafe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, ReLU, Add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1a6e9120",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2b8c49ed",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_7\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            [(None, 28, 28, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 28, 28, 32)   320         input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_26 (ReLU)                 (None, 28, 28, 32)   0           conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 28, 28, 32)   9248        re_lu_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_27 (ReLU)                 (None, 28, 28, 32)   0           conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 28, 28, 32)   0           input_5[0][0]                    \n",
      "                                                                 re_lu_27[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 9,568\n",
      "Trainable params: 9,568\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n# CNN\\n# BLOCK 1\\nx = Conv2D(filters=16, kernel_size=(5, 5), strides=1, padding=\"same\")(i)\\nx = ReLU()(x)\\nx = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\\n\\n# BLOCK 2\\nx = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\\nx = ReLU()(x)\\nx = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\\nx = ReLU()(x)\\nx = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\\n\\n# BLOCK 3\\nx = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\\nx = ReLU()(x)\\nx = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\\nx = ReLU()(x)\\nx = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\\n\\n# CLASSIFIER\\n\\n# Option 1\\nx = Flatten()(x)\\nc = Dense(units=256, activation=\"relu\")(x)\\n\\n# Option 2\\n# c = GlobalAveragePooling2D()(x)\\n\\n# DROPOUT\\nc = Dropout(0.5)(c)\\n\\n# PREDICTIONS\\np = Dense(units=10, activation=\"softmax\")(c)\\n\\n# CREATE MODEL\\ncnn_model = tf.keras.models.Model(i, p)\\ncnn_model.summary()\\n\\n# start_time = timeit.default_timer()\\n# cnn_model(np.random.rand(32, 28, 28, 1))\\n# end_time = timeit.default_timer()\\n# print(\"Time = {}\".format(end_time - start_time))\\n'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# INPUT\n",
    "i = Input((28, 28, 1))\n",
    "\n",
    "# RESIDUAL BLOCK\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(i)\n",
    "x = ReLU()(x)\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = Add()([i, x])\n",
    "\n",
    "residual_block = tf.keras.models.Model(i, x)\n",
    "residual_block.summary()\n",
    "\n",
    "\"\"\"\n",
    "# CNN\n",
    "# BLOCK 1\n",
    "x = Conv2D(filters=16, kernel_size=(5, 5), strides=1, padding=\"same\")(i)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# BLOCK 2\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = Conv2D(filters=32, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# BLOCK 3\n",
    "x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding=\"same\")(x)\n",
    "x = ReLU()(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2), strides=2, padding=\"same\")(x)\n",
    "\n",
    "# CLASSIFIER\n",
    "\n",
    "# Option 1\n",
    "x = Flatten()(x)\n",
    "c = Dense(units=256, activation=\"relu\")(x)\n",
    "\n",
    "# Option 2\n",
    "# c = GlobalAveragePooling2D()(x)\n",
    "\n",
    "# DROPOUT\n",
    "c = Dropout(0.5)(c)\n",
    "\n",
    "# PREDICTIONS\n",
    "p = Dense(units=10, activation=\"softmax\")(c)\n",
    "\n",
    "# CREATE MODEL\n",
    "cnn_model = tf.keras.models.Model(i, p)\n",
    "cnn_model.summary()\n",
    "\n",
    "# start_time = timeit.default_timer()\n",
    "# cnn_model(np.random.rand(32, 28, 28, 1))\n",
    "# end_time = timeit.default_timer()\n",
    "# print(\"Time = {}\".format(end_time - start_time))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "107bc05d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"categorical_accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d26e94ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_best = tf.keras.callbacks.ModelCheckpoint(\"best_cnn_model.h5\", save_best_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cb8e20ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "1500/1500 [==============================] - 38s 26ms/step - loss: 0.5433 - categorical_accuracy: 0.8033 - val_loss: 0.3452 - val_categorical_accuracy: 0.8665\n",
      "Epoch 2/20\n",
      "1500/1500 [==============================] - 39s 26ms/step - loss: 0.3368 - categorical_accuracy: 0.8766 - val_loss: 0.3014 - val_categorical_accuracy: 0.8862\n",
      "Epoch 3/20\n",
      "1500/1500 [==============================] - 39s 26ms/step - loss: 0.2896 - categorical_accuracy: 0.8942 - val_loss: 0.2731 - val_categorical_accuracy: 0.9007\n",
      "Epoch 4/20\n",
      "1500/1500 [==============================] - 38s 26ms/step - loss: 0.2634 - categorical_accuracy: 0.9015 - val_loss: 0.2430 - val_categorical_accuracy: 0.9099\n",
      "Epoch 5/20\n",
      "1500/1500 [==============================] - 35s 23ms/step - loss: 0.2384 - categorical_accuracy: 0.9117 - val_loss: 0.2492 - val_categorical_accuracy: 0.9107\n",
      "Epoch 6/20\n",
      "1500/1500 [==============================] - 41s 27ms/step - loss: 0.2185 - categorical_accuracy: 0.9186 - val_loss: 0.2472 - val_categorical_accuracy: 0.9075\n",
      "Epoch 7/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.2038 - categorical_accuracy: 0.9244 - val_loss: 0.2425 - val_categorical_accuracy: 0.9124\n",
      "Epoch 8/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.1901 - categorical_accuracy: 0.9288 - val_loss: 0.2366 - val_categorical_accuracy: 0.9149\n",
      "Epoch 9/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.1760 - categorical_accuracy: 0.9345 - val_loss: 0.2476 - val_categorical_accuracy: 0.9086\n",
      "Epoch 10/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.1688 - categorical_accuracy: 0.9362 - val_loss: 0.2427 - val_categorical_accuracy: 0.9109\n",
      "Epoch 11/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.1542 - categorical_accuracy: 0.9416 - val_loss: 0.2537 - val_categorical_accuracy: 0.9128\n",
      "Epoch 12/20\n",
      "1500/1500 [==============================] - 40s 27ms/step - loss: 0.1475 - categorical_accuracy: 0.9436 - val_loss: 0.2703 - val_categorical_accuracy: 0.9135\n",
      "Epoch 13/20\n",
      "1500/1500 [==============================] - 42s 28ms/step - loss: 0.1385 - categorical_accuracy: 0.9469 - val_loss: 0.2759 - val_categorical_accuracy: 0.9137\n",
      "Epoch 14/20\n",
      "1500/1500 [==============================] - 33s 22ms/step - loss: 0.1318 - categorical_accuracy: 0.9493 - val_loss: 0.2852 - val_categorical_accuracy: 0.9157\n",
      "Epoch 15/20\n",
      "1500/1500 [==============================] - 34s 22ms/step - loss: 0.1264 - categorical_accuracy: 0.9514 - val_loss: 0.2894 - val_categorical_accuracy: 0.9122\n",
      "Epoch 16/20\n",
      "1500/1500 [==============================] - 34s 22ms/step - loss: 0.1216 - categorical_accuracy: 0.9538 - val_loss: 0.2770 - val_categorical_accuracy: 0.9193\n",
      "Epoch 17/20\n",
      "1500/1500 [==============================] - 37s 25ms/step - loss: 0.1179 - categorical_accuracy: 0.9544 - val_loss: 0.2976 - val_categorical_accuracy: 0.9194\n",
      "Epoch 18/20\n",
      "1500/1500 [==============================] - 37s 25ms/step - loss: 0.1118 - categorical_accuracy: 0.9579 - val_loss: 0.3057 - val_categorical_accuracy: 0.9172 loss: 0.1118 - categorical_accuracy: 0.95\n",
      "Epoch 19/20\n",
      "1500/1500 [==============================] - 43s 28ms/step - loss: 0.1096 - categorical_accuracy: 0.9581 - val_loss: 0.3118 - val_categorical_accuracy: 0.9167\n",
      "Epoch 20/20\n",
      "1500/1500 [==============================] - 36s 24ms/step - loss: 0.1042 - categorical_accuracy: 0.9607 - val_loss: 0.3450 - val_categorical_accuracy: 0.9175\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x14445d84f48>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_model.fit(x=train_data, y=train_labels, epochs=20, batch_size=32,\n",
    "             validation_split=0.2, callbacks=[save_best])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5730204b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 2s 6ms/step - loss: 0.2507 - categorical_accuracy: 0.9134\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.25065842270851135, 0.9133999943733215]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model = tf.keras.models.load_model(\"best_cnn_model.h5\")\n",
    "best_model.evaluate(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7473f7f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "shifted_test_data = []\n",
    "si = 6\n",
    "sih = 3\n",
    "for image in test_data:    \n",
    "    shifted_image = np.zeros((28, 28, 1))\n",
    "    choice_rand = np.random.rand()\n",
    "    if choice_rand < 0.25:\n",
    "        shifted_image[si:, si:, :] = image[sih:-sih, sih:-sih, :]\n",
    "    elif choice_rand < 0.5:\n",
    "        shifted_image[:-si, :-si, :] = image[sih:-sih, sih:-sih, :]\n",
    "    elif choice_rand < 0.75:\n",
    "        shifted_image[si:, :-si, :] = image[sih:-sih, sih:-sih, :]\n",
    "    else:\n",
    "        shifted_image[:-si, si:, :] = image[sih:-sih, sih:-sih, :]\n",
    "        \n",
    "    shifted_test_data.append(shifted_image)\n",
    "    \n",
    "shifted_test_data = np.array(shifted_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0c21aaeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 2s 6ms/step - loss: 1.8943 - categorical_accuracy: 0.4776\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.8943020105361938, 0.47760000824928284]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.evaluate(shifted_test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a317693",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}