notebooks/05_resnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Users\\Yue\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.datasets import fashion_mnist\n",
    "from tensorflow.contrib.eager.python import tfe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# enable eager mode\n",
    "tf.enable_eager_execution()\n",
    "tf.set_random_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('weights/'):\n",
    "    os.makedirs('weights/')\n",
    "\n",
    "# constants\n",
    "image_size = 28\n",
    "batch_size = 128\n",
    "epochs = 10\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x train (60000, 28, 28, 1)\n",
      "y train (60000, 10)\n",
      "x test (10000, 28, 28, 1)\n",
      "y test (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# dataset loading\n",
    "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n",
    "x_train = x_train.reshape((-1, image_size, image_size, 1))\n",
    "x_test = x_test.reshape((-1, image_size, image_size, 1))\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "\n",
    "# one hot encode the labels. convert back to numpy as we cannot use a combination of numpy\n",
    "# and tensors as input to keras\n",
    "y_train_ohe = tf.one_hot(y_train, depth=num_classes).numpy()\n",
    "y_test_ohe = tf.one_hot(y_test, depth=num_classes).numpy()\n",
    "\n",
    "print('x train', x_train.shape)\n",
    "print('y train', y_train_ohe.shape)\n",
    "print('x test', x_test.shape)\n",
    "print('y test', y_test_ohe.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a basic Conv layer helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3x3 convolution\n",
    "def conv3x3(channels, stride=1, kernel=(3, 3)):\n",
    "    return tf.keras.layers.Conv2D(channels, kernel, strides=(stride, stride), padding='same', use_bias=False,\n",
    "                                  kernel_initializer=tf.variance_scaling_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ResNet block builder\n",
    "\n",
    "This can be either an Identity block or a Convolution block, and I am using the pre-activation variant of ResNets without the BottleNeck variant, since this is an example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResnetBlock(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, channels, strides=1, residual_path=False):\n",
    "        super(ResnetBlock, self).__init__()\n",
    "        self.channels = channels\n",
    "        self.strides = strides\n",
    "        self.residual_path = residual_path\n",
    "\n",
    "        self.conv1 = conv3x3(channels, strides)\n",
    "        self.bn1 = tf.keras.layers.BatchNormalization()\n",
    "        self.conv2 = conv3x3(channels)\n",
    "        self.bn2 = tf.keras.layers.BatchNormalization()\n",
    "\n",
    "        if residual_path:\n",
    "            self.down_conv = conv3x3(channels, strides, kernel=(1, 1))\n",
    "            self.down_bn = tf.keras.layers.BatchNormalization()\n",
    "\n",
    "    def call(self, inputs, training=None, mask=None):\n",
    "        residual = inputs\n",
    "\n",
    "        x = self.bn1(inputs, training=training)\n",
    "        x = tf.nn.relu(x)\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn2(x, training=training)\n",
    "        x = tf.nn.relu(x)\n",
    "        x = self.conv2(x)\n",
    "\n",
    "        if self.residual_path:\n",
    "            residual = self.down_bn(inputs, training=training)\n",
    "            residual = tf.nn.relu(residual)\n",
    "            residual = self.down_conv(residual)\n",
    "\n",
    "        x = x + residual\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Configurable Network\n",
    "\n",
    "This network is adaptive, in that it can have many layers, and therefore we cannot determine the layers before hand.\n",
    "\n",
    "To remedy this, we use the convenient `setattr` (and optinally `getattr`) to dynamically \"register\" and \"call\" sublayers.\n",
    "\n",
    "# Note on why this is needed\n",
    "\n",
    "Eager Models *will* automatically register all variables that have been bound to an identifier inside that class - \n",
    "\n",
    "- Using `self.layer_name = tf.keras.layers.***`\n",
    "- Using `self.block = ClassWhichInheritsModel(...)`\n",
    "\n",
    "However. **it will not register variables that have not been bound directly to the class itself or are custom variables.**\n",
    "\n",
    "- Using `self.layers = [layer1, layer2]`\n",
    "- Using `self.layers = {'l1':layer1, 'l2':layer2}`\n",
    "- Using `self.variable = tf.get_variable(...)`\n",
    "\n",
    "Special case : \n",
    "\n",
    "- Using `self.cells = [LSTMCell(), LSTMCell()]` and then wrapping it around an RNN as : `self.rnn = RNN(self.cells)` **will work as expected**. The weights of the LSTMCell will be registered and the RNN itself is registered as well.\n",
    "\n",
    "**`setattr` and `getattr` bypasses the above issues, and sets the layers or models to the class itself, so it is registered by Keras.**\n",
    "\n",
    "# Note 2\n",
    "\n",
    "This registration of layers is important only for convenience of using Model methods - when using Model.compile(), Model.fit(), Model.predict() and Model.evaluate().\n",
    "\n",
    "If there is no need for these utilities, you can write the class as you want, extract all the variables in a list, get the gradients using `tf.GradientTape()` and then update the parameters by hand using `Optimizer.apply_gradients()`. In such a scenario, even the **Model._set_input(...)** fix need not be applied, since you will be doing batch level training anyways and the first update will use that small batch to determine the shape of the model. Such an example is shown in `10_custom_model.ipynb`\n",
    "\n",
    "However, it is far too convenient to use Keras' inbuilt methods for general use-cases such as classification and regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNet(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, block_list, num_classes, initial_filters=16, **kwargs):\n",
    "        super(ResNet, self).__init__(**kwargs)\n",
    "        self.num_blocks = len(block_list)\n",
    "        self.block_list = block_list\n",
    "\n",
    "        self.in_channels = initial_filters\n",
    "        self.out_channels = initial_filters\n",
    "        self.conv_initial = conv3x3(self.out_channels)\n",
    "\n",
    "        self.blocks = []\n",
    "\n",
    "        # build all the blocks\n",
    "        for block_id in range(len(block_list)):\n",
    "            for layer_id in range(block_list[block_id]):\n",
    "                key = 'block_%d_%d' % (block_id + 1, layer_id + 1)\n",
    "                if block_id != 0 and layer_id == 0:\n",
    "                    block = ResnetBlock(self.out_channels, strides=2, residual_path=True)\n",
    "                else:\n",
    "                    if self.in_channels != self.out_channels:\n",
    "                        residual_path = True\n",
    "                    else:\n",
    "                        residual_path = False\n",
    "                    block = ResnetBlock(self.out_channels, residual_path=residual_path)\n",
    "\n",
    "                self.in_channels = self.out_channels\n",
    "\n",
    "                # \"register\" this block to this model ; Without this, weights wont update.\n",
    "                setattr(self, key, block)\n",
    "\n",
    "                self.blocks.append(block)\n",
    "\n",
    "            self.out_channels *= 2\n",
    "\n",
    "        self.final_bn = tf.keras.layers.BatchNormalization()\n",
    "        self.avg_pool = tf.keras.layers.GlobalAveragePooling2D()\n",
    "        self.fc = tf.keras.layers.Dense(num_classes)\n",
    "\n",
    "    def call(self, inputs, training=None, mask=None):\n",
    "        out = self.conv_initial(inputs)\n",
    "\n",
    "        # forward pass through all the blocks\n",
    "        # build all the blocks\n",
    "        for block in self.blocks:\n",
    "            out = block(out, training=training)\n",
    "\n",
    "        out = self.final_bn(out)\n",
    "        out = tf.nn.relu(out)\n",
    "\n",
    "        out = self.avg_pool(out)\n",
    "        out = self.fc(out)\n",
    "\n",
    "        # softmax op does not exist on the gpu, so always use cpu\n",
    "        with tf.device('/cpu:0'):\n",
    "            output = tf.nn.softmax(out)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fashion MNIST\n",
    "Here, we try a harder dataset than the basic MNIST, where it is very easy to get 99% with even small networks. The basic average of small models on Fashion MNIST on the other hand is close to 90-92%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of variables in the model : 77\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            multiple                  144       \n",
      "_________________________________________________________________\n",
      "resnet_block_1 (ResnetBlock) multiple                  4736      \n",
      "_________________________________________________________________\n",
      "resnet_block_2 (ResnetBlock) multiple                  4736      \n",
      "_________________________________________________________________\n",
      "resnet_block_3 (ResnetBlock) multiple                  14592     \n",
      "_________________________________________________________________\n",
      "resnet_block_4 (ResnetBlock) multiple                  18688     \n",
      "_________________________________________________________________\n",
      "resnet_block_5 (ResnetBlock) multiple                  57856     \n",
      "_________________________________________________________________\n",
      "resnet_block_6 (ResnetBlock) multiple                  74240     \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc multiple                  256       \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_1 ( multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              multiple                  650       \n",
      "=================================================================\n",
      "Total params: 175,898\n",
      "Trainable params: 174,874\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 33s 544us/step - loss: 0.5047 - acc: 0.8231 - val_loss: 1.9352 - val_acc: 0.4728\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 32s 534us/step - loss: 0.2930 - acc: 0.8942 - val_loss: 0.6433 - val_acc: 0.7962\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 32s 530us/step - loss: 0.2464 - acc: 0.9108 - val_loss: 0.3460 - val_acc: 0.8735\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 32s 536us/step - loss: 0.2140 - acc: 0.9226 - val_loss: 0.3475 - val_acc: 0.8778ETA: 18s - loss: 0.2153 - acc: 0.9216 - ETA: 13s - loss: 0.2141 - acc: 0.9217\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 32s 526us/step - loss: 0.1945 - acc: 0.9295 - val_loss: 0.2785 - val_acc: 0.8994\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 31s 522us/step - loss: 0.1728 - acc: 0.9371 - val_loss: 0.2572 - val_acc: 0.9102\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 32s 528us/step - loss: 0.1592 - acc: 0.9417 - val_loss: 0.2814 - val_acc: 0.9021\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 33s 556us/step - loss: 0.1462 - acc: 0.9467 - val_loss: 0.3024 - val_acc: 0.8996\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 33s 549us/step - loss: 0.1348 - acc: 0.9512 - val_loss: 0.3417 - val_acc: 0.8887\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 33s 557us/step - loss: 0.1186 - acc: 0.9574 - val_loss: 0.3065 - val_acc: 0.8981\n",
      "10000/10000 [==============================] - 2s 157us/step\n",
      "Final test loss and accuracy : [0.3064836109161377, 0.8981]\n"
     ]
    }
   ],
   "source": [
    "device = '/cpu:0' if tfe.num_gpus() == 0 else '/gpu:0'\n",
    "\n",
    "with tf.device(device):\n",
    "    # build model and optimizer\n",
    "    model = ResNet([2, 2, 2], num_classes)\n",
    "    model.compile(optimizer=tf.train.AdamOptimizer(0.001), loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    # TF Keras tries to use entire dataset to determine shape without this step when using .fit()\n",
    "    # Fix = Use exactly one sample from the provided input dataset to determine input/output shape/s for the model\n",
    "    dummy_x = tf.zeros((1, image_size, image_size, 1))\n",
    "    model._set_inputs(dummy_x)\n",
    "\n",
    "    print(\"Number of variables in the model :\", len(model.variables))\n",
    "    model.summary()\n",
    "\n",
    "    # train\n",
    "    model.fit(x_train, y_train_ohe, batch_size=batch_size, epochs=epochs,\n",
    "              validation_data=(x_test, y_test_ohe), verbose=1)\n",
    "\n",
    "    # evaluate on test set\n",
    "    scores = model.evaluate(x_test, y_test_ohe, batch_size, verbose=1)\n",
    "    print(\"Final test loss and accuracy :\", scores)\n",
    "\n",
    "    saver = tfe.Saver(model.variables)\n",
    "    saver.save('weights/05_resnet/weights.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}