Assignment_3_1st_DNN_9913.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment 3 - 1st DNN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Abhinavl3v3l/AUT/blob/master/Assignment_3_1st_DNN_9913.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aNyZv-Ec52ot",
        "colab_type": "text"
      },
      "source": [
        "# **Import Libraries and modules**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3m3w1Cw49Zkt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# https://keras.io/\n",
        "!pip install -q keras\n",
        "import keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eso6UHE080D4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten, Add\n",
        "from keras.layers import Convolution2D, MaxPooling2D\n",
        "from keras.utils import np_utils\n",
        "\n",
        "from keras.datasets import mnist"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zByEi95J86RD",
        "colab_type": "text"
      },
      "source": [
        "### Load pre-shuffled MNIST data into train and test sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7eRM0QWN83PV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4a4Be72j8-ZC",
        "colab_type": "code",
        "outputId": "b4a291e1-7249-4bbb-fb16-cfa48d5151b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        }
      },
      "source": [
        "print (X_train.shape)\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline\n",
        "plt.imshow(X_train[0])"
      ],
      "execution_count": 370,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 28, 28)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f22c5e12940>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 370
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADoBJREFUeJzt3X2MXOV1x/HfyXq9jo1JvHHYboiL\nHeMEiGlMOjIgLKCiuA5CMiiKiRVFDiFxmuCktK4EdavGrWjlVgmRQynS0ri2I95CAsJ/0CR0FUGi\nwpbFMeYtvJlNY7PsYjZgQ4i9Xp/+sdfRBnaeWc/cmTu75/uRVjtzz71zj6792zszz8x9zN0FIJ53\nFd0AgGIQfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQU1r5M6mW5vP0KxG7hII5bd6U4f9kE1k\n3ZrCb2YrJG2W1CLpP9x9U2r9GZqls+2iWnYJIKHHuye8btVP+82sRdJNkj4h6QxJq83sjGofD0Bj\n1fKaf6mk5919j7sflnSHpJX5tAWg3moJ/8mSfjXm/t5s2e8xs7Vm1mtmvcM6VMPuAOSp7u/2u3uX\nu5fcvdSqtnrvDsAE1RL+fZLmjbn/wWwZgEmglvA/ImmRmS0ws+mSPi1pRz5tAai3qof63P2Ima2T\n9CONDvVtcfcnc+sMQF3VNM7v7vdJui+nXgA0EB/vBYIi/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/\nEBThB4Ii/EBQhB8IivADQRF+IKiaZuk1sz5JByWNSDri7qU8mkJ+bFr6n7jl/XPruv9n/np+2drI\nzKPJbU9ZOJisz/yKJesv3zC9bG1n6c7ktvtH3kzWz75rfbJ+6l89nKw3g5rCn/kTd9+fw+MAaCCe\n9gNB1Rp+l/RjM3vUzNbm0RCAxqj1af8yd99nZidJut/MfuHuD45dIfujsFaSZmhmjbsDkJeazvzu\nvi/7PSjpHklLx1mny91L7l5qVVstuwOQo6rDb2azzGz2sduSlkt6Iq/GANRXLU/7OyTdY2bHHuc2\nd/9hLl0BqLuqw+/ueyR9LMdepqyW0xcl697Wmqy/dMF7k/W3zik/Jt3+nvR49U8/lh7vLtJ//WZ2\nsv4v/7YiWe8587aytReH30puu2ng4mT9Az/1ZH0yYKgPCIrwA0ERfiAowg8ERfiBoAg/EFQe3+oL\nb+TCjyfrN2y9KVn/cGv5r55OZcM+kqz//Y2fS9anvZkebjv3rnVla7P3HUlu27Y/PRQ4s7cnWZ8M\nOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM8+eg7ZmXkvVHfzsvWf9w60Ce7eRqff85yfqeN9KX\n/t668Ptla68fTY/Td3z7f5L1epr8X9itjDM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRl7o0b0TzR\n2v1su6hh+2sWQ1eem6wfWJG+vHbL7hOS9ce+cuNx93TM9fv/KFl/5IL0OP7Ia68n635u+au7930t\nuakWrH4svQLeoce7dcCH0nOXZzjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQFcf5zWyLpEslDbr7\n4mxZu6Q7Jc2X1Cdplbv/utLOoo7zV9Iy933J+sirQ8n6i7eVH6t/8vwtyW2X/vNXk/WTbiruO/U4\nfnmP82+V9PaJ0K+T1O3uiyR1Z/cBTCIVw+/uD0p6+6lnpaRt2e1tki7LuS8AdVbta/4Od+/Pbr8s\nqSOnfgA0SM1v+PnomwZl3zgws7Vm1mtmvcM6VOvuAOSk2vAPmFmnJGW/B8ut6O5d7l5y91Kr2qrc\nHYC8VRv+HZLWZLfXSLo3n3YANErF8JvZ7ZIekvQRM9trZldJ2iTpYjN7TtKfZvcBTCIVr9vv7qvL\nlBiwz8nI/ldr2n74wPSqt/3oZ55K1l+5uSX9AEdHqt43isUn/ICgCD8QFOEHgiL8QFCEHwiK8ANB\nMUX3FHD6tc+WrV15ZnpE9j9P6U7WL/jU1cn67DsfTtbRvDjzA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQjPNPAalpsl/98unJbf9vx1vJ+nXXb0/W/2bV5cm6//w9ZWvz/umh5LZq4PTxEXHmB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgKk7RnSem6G4+Q58/N1m/9evfSNYXTJtR9b4/un1dsr7olv5k/cie\nvqr3PVXlPUU3gCmI8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2ZbJF0qadDdF2fLNkr6oqRXstU2\nuPt9lXbGOP/k4+ctSdZP3LQ3Wb/9Qz+qet+n/eQLyfpH/qH8dQwkaeS5PVXve7LKe5x/q6QV4yz/\nlrsvyX4qBh9Ac6kYfnd/UNJQA3oB0EC1vOZfZ2a7zWyLmc3JrSMADVFt+G+WtFDSEkn9kr5ZbkUz\nW2tmvWbWO6xDVe4OQN6qCr+7D7j7iLsflXSLpKWJdbvcveTupVa1VdsngJxVFX4z6xxz93JJT+TT\nDoBGqXjpbjO7XdKFkuaa2V5JX5d0oZktkeSS+iR9qY49AqgDvs+PmrR0nJSsv3TFqWVrPdduTm77\nrgpPTD/z4vJk/fVlrybrUxHf5wdQEeEHgiL8QFCEHwiK8ANBEX4gKIb6UJjv7U1P0T3Tpifrv/HD\nyfqlX72m/GPf05PcdrJiqA9ARYQfCIrwA0ERfiAowg8ERfiBoAg/EFTF7/MjtqPL0pfufuFT6Sm6\nFy/pK1urNI5fyY1DZyXrM+/trenxpzrO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOP8U5yVFifr\nz34tPdZ+y3nbkvXzZ6S/U1+LQz6crD88tCD9AEf7c+xm6uHMDwRF+IGgCD8QFOEHgiL8QFCEHwiK\n8ANBVRznN7N5krZL6pDkkrrcfbOZtUu6U9J8SX2SVrn7r+vXalzTFpySrL9w5QfK1jZecUdy20+e\nsL+qnvKwYaCUrD+w+Zxkfc629HX/kTaRM/8RSevd/QxJ50i62szOkHSdpG53XySpO7sPYJKoGH53\n73f3ndntg5KelnSypJWSjn38a5uky+rVJID8HddrfjObL+ksST2SOtz92OcnX9boywIAk8SEw29m\nJ0j6gaRr3P3A2JqPTvg37qR/ZrbWzHrNrHdYh2pqFkB+JhR+M2vVaPBvdfe7s8UDZtaZ1TslDY63\nrbt3uXvJ3UutasujZwA5qBh+MzNJ35H0tLvfMKa0Q9Ka7PYaSffm3x6AepnIV3rPk/RZSY+b2a5s\n2QZJmyR9z8yukvRLSavq0+LkN23+Hybrr/9xZ7J+xT/+MFn/8/fenazX0/r+9HDcQ/9efjivfev/\nJredc5ShvHqqGH53/5mkcvN9X5RvOwAahU/4AUERfiAowg8ERfiBoAg/EBThB4Li0t0TNK3zD8rW\nhrbMSm775QUPJOurZw9U1VMe1u1blqzvvDk9Rffc7z+RrLcfZKy+WXHmB4Ii/EBQhB8IivADQRF+\nICjCDwRF+IGgwozzH/6z9GWiD//lULK+4dT7ytaWv/vNqnrKy8DIW2Vr5+9Yn9z2tL/7RbLe/lp6\nnP5osopmxpkfCIrwA0ERfiAowg8ERfiBoAg/EBThB4IKM87fd1n679yzZ95Vt33f9NrCZH3zA8uT\ndRspd+X0Uadd/2LZ2qKBnuS2I8kqpjLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLl7egWzeZK2\nS+qQ5JK63H2zmW2U9EVJr2SrbnD38l96l3SitfvZxqzeQL30eLcO+FD6gyGZiXzI54ik9e6+08xm\nS3rUzO7Pat9y929U2yiA4lQMv7v3S+rPbh80s6clnVzvxgDU13G95jez+ZLOknTsM6PrzGy3mW0x\nszlltllrZr1m1jusQzU1CyA/Ew6/mZ0g6QeSrnH3A5JulrRQ0hKNPjP45njbuXuXu5fcvdSqthxa\nBpCHCYXfzFo1Gvxb3f1uSXL3AXcfcfejkm6RtLR+bQLIW8Xwm5lJ+o6kp939hjHLO8esdrmk9HSt\nAJrKRN7tP0/SZyU9bma7smUbJK02syUaHf7rk/SlunQIoC4m8m7/zySNN26YHNMH0Nz4hB8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoipfuznVnZq9I+uWY\nRXMl7W9YA8enWXtr1r4keqtWnr2d4u7vn8iKDQ3/O3Zu1uvupcIaSGjW3pq1L4neqlVUbzztB4Ii\n/EBQRYe/q+D9pzRrb83al0Rv1Sqkt0Jf8wMoTtFnfgAFKST8ZrbCzJ4xs+fN7LoieijHzPrM7HEz\n22VmvQX3ssXMBs3siTHL2s3sfjN7Lvs97jRpBfW20cz2Zcdul5ldUlBv88zsJ2b2lJk9aWZ/kS0v\n9Ngl+irkuDX8ab+ZtUh6VtLFkvZKekTSand/qqGNlGFmfZJK7l74mLCZnS/pDUnb3X1xtuxfJQ25\n+6bsD+ccd7+2SXrbKOmNomduziaU6Rw7s7SkyyR9TgUeu0Rfq1TAcSvizL9U0vPuvsfdD0u6Q9LK\nAvpoeu7+oKShty1eKWlbdnubRv/zNFyZ3pqCu/e7+87s9kFJx2aWLvTYJfoqRBHhP1nSr8bc36vm\nmvLbJf3YzB41s7VFNzOOjmzadEl6WVJHkc2Mo+LMzY30tpmlm+bYVTPjdd54w++dlrn7xyV9QtLV\n2dPbpuSjr9maabhmQjM3N8o4M0v/TpHHrtoZr/NWRPj3SZo35v4Hs2VNwd33Zb8HJd2j5pt9eODY\nJKnZ78GC+/mdZpq5ebyZpdUEx66ZZrwuIvyPSFpkZgvMbLqkT0vaUUAf72Bms7I3YmRmsyQtV/PN\nPrxD0prs9hpJ9xbYy+9plpmby80srYKPXdPNeO3uDf+RdIlG3/F/QdLfFtFDmb4+JOmx7OfJonuT\ndLtGnwYOa/S9kaskvU9St6TnJP23pPYm6u27kh6XtFujQessqLdlGn1Kv1vSruznkqKPXaKvQo4b\nn/ADguINPyAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQf0/sEWOix6VKakAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkmprriw9AnZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = X_train.reshape(X_train.shape[0], 28, 28,1)\n",
        "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X2m4YS4E9CRh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Mn0vAYD9DvB",
        "colab_type": "code",
        "outputId": "e080e62d-4bfd-4714-f90f-f782df4287eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_train[:10]"
      ],
      "execution_count": 373,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4], dtype=uint8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 373
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZG8JiXR39FHC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Convert 1-dimensional class arrays to 10-dimensional class matrices\n",
        "Y_train = np_utils.to_categorical(y_train, 10)\n",
        "Y_test = np_utils.to_categorical(y_test, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYlFRvKS9HMB",
        "colab_type": "code",
        "outputId": "7b772b5f-fe9d-4336-9436-b4bf5b140d2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "Y_train[:10]\n"
      ],
      "execution_count": 375,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
              "       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 375
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "osKqT73Q9JJB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.layers import Activation\n",
        "model = Sequential()\n",
        "\n",
        "\n",
        "model.add(Convolution2D(8, (3, 3), activation='relu', input_shape=(28,28,1)))#26  \n",
        "\n",
        "model.add(Convolution2D(16, (3, 3), activation='relu'))  #24\n",
        "model.add(Convolution2D(16, (3, 3), activation='relu'))  #22\n",
        "model.add(Convolution2D(16, (3, 3), activation='relu'))  #20\n",
        "model.add(Convolution2D(16, (3, 3), activation='relu'))  #19\n",
        "           \n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))                #9\n",
        "model.add(Convolution2D(20, (3, 3), activation='relu'))  #7\n",
        "model.add(Convolution2D(20, (3, 3), activation='relu'))  #5\n",
        "model.add(Convolution2D(20, (3, 3), activation='relu'))  #3\n",
        "\n",
        "model.add(Convolution2D(32, 1, activation='relu'))\n",
        "model.add(Convolution2D(10, 3))\n",
        "model.add(Flatten())\n",
        "model.add(Activation('softmax'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TzdAYg1k9K7Z",
        "colab_type": "code",
        "outputId": "222eafd3-a454-41cf-d379-f1c03200ca67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 595
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 377,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_28\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_236 (Conv2D)          (None, 26, 26, 16)        160       \n",
            "_________________________________________________________________\n",
            "conv2d_237 (Conv2D)          (None, 24, 24, 16)        2320      \n",
            "_________________________________________________________________\n",
            "conv2d_238 (Conv2D)          (None, 22, 22, 16)        2320      \n",
            "_________________________________________________________________\n",
            "conv2d_239 (Conv2D)          (None, 20, 20, 16)        2320      \n",
            "_________________________________________________________________\n",
            "conv2d_240 (Conv2D)          (None, 18, 18, 16)        2320      \n",
            "_________________________________________________________________\n",
            "max_pooling2d_28 (MaxPooling (None, 9, 9, 16)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_241 (Conv2D)          (None, 7, 7, 32)          4640      \n",
            "_________________________________________________________________\n",
            "conv2d_242 (Conv2D)          (None, 5, 5, 32)          9248      \n",
            "_________________________________________________________________\n",
            "conv2d_243 (Conv2D)          (None, 3, 3, 32)          9248      \n",
            "_________________________________________________________________\n",
            "conv2d_244 (Conv2D)          (None, 3, 3, 10)          330       \n",
            "_________________________________________________________________\n",
            "conv2d_245 (Conv2D)          (None, 1, 1, 10)          910       \n",
            "_________________________________________________________________\n",
            "flatten_27 (Flatten)         (None, 10)                0         \n",
            "_________________________________________________________________\n",
            "activation_27 (Activation)   (None, 10)                0         \n",
            "=================================================================\n",
            "Total params: 33,816\n",
            "Trainable params: 33,816\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zp6SuGrL9M3h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "             optimizer='adam',\n",
        "             metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4xWoKhPY9Of5",
        "colab_type": "code",
        "outputId": "40d91f01-689e-4a88-9ab3-4c77793c4e64",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        }
      },
      "source": [
        "model.fit(X_train, Y_train, batch_size=128, nb_epoch=20, verbose=1,validation_data=(X_test, Y_test))"
      ],
      "execution_count": 379,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/20\n",
            "60000/60000 [==============================] - 15s 249us/step - loss: 0.4158 - acc: 0.8612 - val_loss: 0.1212 - val_acc: 0.9611\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 10s 162us/step - loss: 0.1004 - acc: 0.9687 - val_loss: 0.0801 - val_acc: 0.9754\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0680 - acc: 0.9798 - val_loss: 0.0518 - val_acc: 0.9830\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0551 - acc: 0.9827 - val_loss: 0.0530 - val_acc: 0.9833\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0484 - acc: 0.9845 - val_loss: 0.0410 - val_acc: 0.9871\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0417 - acc: 0.9871 - val_loss: 0.0367 - val_acc: 0.9878\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0366 - acc: 0.9886 - val_loss: 0.0361 - val_acc: 0.9894\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 10s 163us/step - loss: 0.0327 - acc: 0.9896 - val_loss: 0.0315 - val_acc: 0.9894\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0295 - acc: 0.9907 - val_loss: 0.0377 - val_acc: 0.9879\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 10s 163us/step - loss: 0.0275 - acc: 0.9912 - val_loss: 0.0346 - val_acc: 0.9892\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: 0.0246 - acc: 0.9917 - val_loss: 0.0356 - val_acc: 0.9882\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: 0.0237 - acc: 0.9920 - val_loss: 0.0275 - val_acc: 0.9913\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: 0.0200 - acc: 0.9933 - val_loss: 0.0412 - val_acc: 0.9870\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 10s 161us/step - loss: 0.0194 - acc: 0.9939 - val_loss: 0.0323 - val_acc: 0.9908\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: 0.0180 - acc: 0.9941 - val_loss: 0.0350 - val_acc: 0.9894\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 10s 160us/step - loss: 0.0152 - acc: 0.9948 - val_loss: 0.0411 - val_acc: 0.9890\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 10s 159us/step - loss: 0.0178 - acc: 0.9942 - val_loss: 0.0284 - val_acc: 0.9912\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 10s 158us/step - loss: 0.0145 - acc: 0.9951 - val_loss: 0.0368 - val_acc: 0.9893\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 10s 164us/step - loss: 0.0136 - acc: 0.9955 - val_loss: 0.0474 - val_acc: 0.9859\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 10s 159us/step - loss: 0.0116 - acc: 0.9963 - val_loss: 0.0293 - val_acc: 0.9905\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f22c5cb36d8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 379
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AtsH-lLk-eLb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "score = model.evaluate(X_test, Y_test, verbose=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mkX8JMv79q9r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "86f070c2-d44e-49ae-e985-ff579901b4bf"
      },
      "source": [
        "print(score)"
      ],
      "execution_count": 381,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.029271746694766853, 0.9905]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCWoJkwE9suh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_pred = model.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ym7iCFBm9uBs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        },
        "outputId": "f9d8a366-f7c1-4e90-9893-f7305dcfbeb3"
      },
      "source": [
        "print(y_pred[:9])\n",
        "print(y_test[:9])"
      ],
      "execution_count": 383,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[2.76192541e-10 7.80452069e-07 4.26619593e-08 1.36068401e-09\n",
            "  1.50781854e-09 6.17843943e-10 2.33475912e-12 9.99999166e-01\n",
            "  3.64308445e-11 2.19488849e-09]\n",
            " [7.30411953e-10 1.52818677e-06 9.99998331e-01 2.19706666e-11\n",
            "  1.08934942e-12 1.28144123e-07 6.73944456e-09 5.26798827e-10\n",
            "  2.47288012e-09 1.74243791e-14]\n",
            " [8.57063766e-15 1.00000000e+00 9.80431556e-13 4.77353888e-14\n",
            "  4.63965710e-09 3.87694854e-09 2.62388361e-10 4.28664571e-09\n",
            "  3.54080265e-10 1.50828839e-13]\n",
            " [9.99999166e-01 2.59092862e-18 1.37903196e-11 4.42229634e-15\n",
            "  2.30888034e-11 8.81384032e-09 7.61906961e-07 1.20188012e-14\n",
            "  5.82100803e-08 1.75155379e-09]\n",
            " [5.28382642e-14 6.40037611e-13 1.68681407e-13 1.60502441e-15\n",
            "  9.99952435e-01 1.97891053e-12 1.66136668e-12 1.04500523e-11\n",
            "  1.10537302e-09 4.75455199e-05]\n",
            " [8.50552171e-17 1.00000000e+00 1.48350611e-13 3.31771392e-15\n",
            "  1.14476040e-10 9.77978920e-10 5.77921695e-12 1.29385889e-08\n",
            "  2.54463395e-11 2.77332288e-15]\n",
            " [1.36924120e-12 1.12487257e-07 4.59174032e-09 3.23531445e-11\n",
            "  9.98167157e-01 2.39002329e-09 1.27877865e-11 3.95984188e-07\n",
            "  1.67381589e-03 1.58519571e-04]\n",
            " [1.98717423e-16 8.10581000e-18 3.99295695e-13 1.88356442e-10\n",
            "  5.02941013e-08 8.75618120e-11 1.09043320e-14 3.80420112e-10\n",
            "  1.81126808e-10 1.00000000e+00]\n",
            " [7.88913290e-09 4.49550908e-18 1.16903286e-13 1.42547932e-10\n",
            "  4.70379583e-20 9.99994397e-01 3.00321017e-06 8.27417377e-21\n",
            "  2.46655736e-06 1.56220409e-07]]\n",
            "[7 2 1 0 4 1 4 9 5]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CT--y98_dr2T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "layer_dict = dict([(layer.name, layer) for layer in model.layers])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GY4Upv4dsUR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "outputId": "a08c3035-8bab-4031-cb71-01171d8c0442"
      },
      "source": [
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "from keras import backend as K\n",
        "%matplotlib inline\n",
        "# util function to convert a tensor into a valid image\n",
        "def deprocess_image(x):\n",
        "    # normalize tensor: center on 0., ensure std is 0.1\n",
        "    x -= x.mean()\n",
        "    x /= (x.std() + 1e-5)\n",
        "    x *= 0.1\n",
        "\n",
        "    # clip to [0, 1]\n",
        "    x += 0.5\n",
        "    x = np.clip(x, 0, 1)\n",
        "\n",
        "    # convert to RGB array\n",
        "    x *= 255\n",
        "    #x = x.transpose((1, 2, 0))\n",
        "    x = np.clip(x, 0, 255).astype('uint8')\n",
        "    return x\n",
        "\n",
        "def vis_img_in_filter(img = np.array(X_train[2]).reshape((1, 28, 28, 1)).astype(np.float64), \n",
        "                      layer_name = 'conv2d_14'):\n",
        "    layer_output = layer_dict[layer_name].output\n",
        "    img_ascs = list()\n",
        "    for filter_index in range(layer_output.shape[3]):\n",
        "        # build a loss function that maximizes the activation\n",
        "        # of the nth filter of the layer considered\n",
        "        loss = K.mean(layer_output[:, :, :, filter_index])\n",
        "\n",
        "        # compute the gradient of the input picture wrt this loss\n",
        "        grads = K.gradients(loss, model.input)[0]\n",
        "\n",
        "        # normalization trick: we normalize the gradient\n",
        "        grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)\n",
        "\n",
        "        # this function returns the loss and grads given the input picture\n",
        "        iterate = K.function([model.input], [loss, grads])\n",
        "\n",
        "        # step size for gradient ascent\n",
        "        step = 5.\n",
        "\n",
        "        img_asc = np.array(img)\n",
        "        # run gradient ascent for 20 steps\n",
        "        for i in range(20):\n",
        "            loss_value, grads_value = iterate([img_asc])\n",
        "            img_asc += grads_value * step\n",
        "\n",
        "        img_asc = img_asc[0]\n",
        "        img_ascs.append(deprocess_image(img_asc).reshape((28, 28)))\n",
        "        \n",
        "    if layer_output.shape[3] >= 35:\n",
        "        plot_x, plot_y = 6, 6\n",
        "    elif layer_output.shape[3] >= 23:\n",
        "        plot_x, plot_y = 4, 6\n",
        "    elif layer_output.shape[3] >= 11:\n",
        "        plot_x, plot_y = 2, 6\n",
        "    else:\n",
        "        plot_x, plot_y = 1, 2\n",
        "    fig, ax = plt.subplots(plot_x, plot_y, figsize = (12, 12))\n",
        "    ax[0, 0].imshow(img.reshape((28, 28)), cmap = 'gray')\n",
        "    ax[0, 0].set_title('Input image')\n",
        "    fig.suptitle('Input image and %s filters' % (layer_name,))\n",
        "    fig.tight_layout(pad = 0.3, rect = [0, 0, 0.9, 0.9])\n",
        "    for (x, y) in [(i, j) for i in range(plot_x) for j in range(plot_y)]:\n",
        "        if x == 0 and y == 0:\n",
        "            continue\n",
        "        ax[x, y].imshow(img_ascs[x * plot_y + y - 1], cmap = 'gray')\n",
        "        ax[x, y].set_title('filter %d' % (x * plot_y + y - 1))\n",
        "\n",
        "vis_img_in_filter()"
      ],
      "execution_count": 385,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-385-19229f66b51a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0max\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'filter %d'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mplot_y\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m \u001b[0mvis_img_in_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-385-19229f66b51a>\u001b[0m in \u001b[0;36mvis_img_in_filter\u001b[0;34m(img, layer_name)\u001b[0m\n\u001b[1;32m     22\u001b[0m def vis_img_in_filter(img = np.array(X_train[2]).reshape((1, 28, 28, 1)).astype(np.float64), \n\u001b[1;32m     23\u001b[0m                       layer_name = 'conv2d_14'):\n\u001b[0;32m---> 24\u001b[0;31m     \u001b[0mlayer_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlayer_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m     \u001b[0mimg_ascs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mfilter_index\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'conv2d_14'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9tvptcn8dxvp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}