TensorFlow VGG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape:  (49000, 32, 32, 3)\n",
      "Train labels shape:  (49000,)\n",
      "Validation data shape:  (1000, 32, 32, 3)\n",
      "Validation labels shape:  (1000,)\n",
      "Test data shape:  (10000, 32, 32, 3)\n",
      "Test labels shape:  (10000,)\n"
     ]
    }
   ],
   "source": [
    "from data_utils import load_CIFAR10\n",
    "\n",
    "def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=10000):\n",
    "    \"\"\"\n",
    "    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare\n",
    "    it for the two-layer neural net classifier. These are the same steps as\n",
    "    we used for the SVM, but condensed to a single function.  \n",
    "    \"\"\"\n",
    "    # Load the raw CIFAR-10 data\n",
    "    cifar10_dir = 'cifar-10-batches-py'\n",
    "    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)\n",
    "\n",
    "    # Subsample the data\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = X_train[mask]\n",
    "    y_val = y_train[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = X_train[mask]\n",
    "    y_train = y_train[mask]\n",
    "    mask = range(num_test)\n",
    "    X_test = X_test[mask]\n",
    "    y_test = y_test[mask]\n",
    "\n",
    "    # Normalize the data: subtract the mean image\n",
    "    mean_image = np.mean(X_train, axis=0)\n",
    "    X_train -= mean_image\n",
    "    X_val -= mean_image\n",
    "    X_test -= mean_image\n",
    "\n",
    "    return X_train, y_train, X_val, y_val, X_test, y_test\n",
    "\n",
    "\n",
    "# Invoke the above function to get our data.\n",
    "# X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data(num_training=5000, num_validation=1000, num_test=2000)\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model(writer, merged, session, predict, loss_val, Xd, yd,\n",
    "              epochs=1, batch_size=64, print_every=100,\n",
    "              training=None, plot_losses=False):\n",
    "    # have tensorflow compute accuracy\n",
    "    correct_prediction = tf.equal(tf.argmax(predict,1), y)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    tf.summary.scalar('accuracy', accuracy)\n",
    "    \n",
    "    # shuffle indicies\n",
    "    train_indicies = np.arange(Xd.shape[0])\n",
    "    np.random.shuffle(train_indicies)\n",
    "\n",
    "    training_now = training is not None\n",
    "    \n",
    "    # setting up variables we want to compute (and optimizing)\n",
    "    # if we have a training function, add that to things we compute\n",
    "    variables = [merged, mean_loss,correct_prediction,accuracy]\n",
    "    if training_now:\n",
    "        variables[-1] = training\n",
    "    \n",
    "    # counter \n",
    "    iter_cnt = 0\n",
    "    for e in range(epochs):\n",
    "        # keep track of losses and accuracy\n",
    "        correct = 0\n",
    "        losses = []\n",
    "        # make sure we iterate over the dataset once\n",
    "        for i in range(int(math.ceil(Xd.shape[0]/batch_size))):\n",
    "            # generate indicies for the batch\n",
    "            start_idx = (i*batch_size)%Xd.shape[0]\n",
    "            idx = train_indicies[start_idx:start_idx+batch_size]\n",
    "            \n",
    "            # create a feed dictionary for this batch\n",
    "            feed_dict = {X: Xd[idx,:],\n",
    "                         y: yd[idx],\n",
    "                         is_training: training_now }\n",
    "            # get batch size\n",
    "            actual_batch_size = yd[idx].shape[0]\n",
    "            \n",
    "            # have tensorflow compute loss and correct predictions\n",
    "            # and (if given) perform a training step\n",
    "            summary, loss, corr, _ = session.run(variables,feed_dict=feed_dict)\n",
    "            writer.add_summary(summary, i*e)\n",
    "            # aggregate performance stats\n",
    "            losses.append(loss*actual_batch_size)\n",
    "            correct += np.sum(corr)\n",
    "            \n",
    "            # print every now and then\n",
    "            if training_now and (iter_cnt % print_every) == 0:\n",
    "                print(\"Iteration {0}: with minibatch training loss = {1:.3g} and accuracy of {2:.2g}\"\\\n",
    "                      .format(iter_cnt,loss,np.sum(corr)/actual_batch_size))\n",
    "            iter_cnt += 1\n",
    "        total_correct = correct/Xd.shape[0]\n",
    "        total_loss = np.sum(losses)/Xd.shape[0]\n",
    "        print(\"Epoch {2}, Overall loss = {0:.3g} and accuracy of {1:.3g}\"\\\n",
    "              .format(total_loss,total_correct,e+1))\n",
    "        if plot_losses:\n",
    "            plt.plot(losses)\n",
    "            plt.grid(True)\n",
    "            plt.title('Epoch {} Loss'.format(e+1))\n",
    "            plt.xlabel('minibatch number')\n",
    "            plt.ylabel('minibatch loss')\n",
    "            plt.show()\n",
    "    return total_loss,total_correct\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_model(X,y,is_training):\n",
    "    #32x32x3\n",
    "    net = tf.layers.conv2d(X, filters=32, kernel_size=[3,3], padding='same',\n",
    "                           activation=tf.nn.leaky_relu)\n",
    "    #32x32x32\n",
    "    net = tf.layers.conv2d(X, filters=32, kernel_size=[3,3], padding='same',\n",
    "                           activation=tf.nn.leaky_relu)\n",
    "    net = tf.layers.max_pooling2d(net, pool_size=[2,2], strides=2)\n",
    "        \n",
    "    #16x16x32\n",
    "    net = tf.layers.batch_normalization(net, training=is_training)\n",
    "    net = tf.layers.conv2d(net, filters=64, kernel_size=[3,3], padding='same',\n",
    "                          activation=tf.nn.leaky_relu)\n",
    "    #16x16x64\n",
    "    net = tf.layers.conv2d(net, filters=64, kernel_size=[3,3], padding='same',\n",
    "                          activation=tf.nn.leaky_relu)\n",
    "    net = tf.layers.max_pooling2d(net, pool_size=[2,2], strides=2)\n",
    "    \n",
    "    #8x8x64\n",
    "    net = tf.layers.batch_normalization(net, training=is_training)\n",
    "    net = tf.layers.conv2d(net, filters=128, kernel_size=[3,3], padding='same',\n",
    "                          activation=tf.nn.leaky_relu)\n",
    "    #8x8x128\n",
    "    net = tf.layers.conv2d(net, filters=128, kernel_size=[3,3], padding='same',\n",
    "                          activation=tf.nn.leaky_relu)\n",
    "    net = tf.layers.conv2d(net, filters=128, kernel_size=[3,3], padding='same',\n",
    "                          activation=tf.nn.leaky_relu)\n",
    "    net = tf.layers.max_pooling2d(net, pool_size=[2,2], strides=2)\n",
    "    \n",
    "    #4x4x128\n",
    "    net = tf.reshape(net, [-1, 4*4*128])\n",
    "    # nx(4*4*32) = nx2048\n",
    "    net = tf.layers.dense(net, units=1024, activation=tf.nn.relu)\n",
    "    net = tf.layers.dropout(net, rate=0.7, training=is_training)\n",
    "    # nx1024\n",
    "    net = tf.layers.dense(net, units=512, activation=tf.nn.relu)\n",
    "    net = tf.layers.dropout(net, rate=0.5, training=is_training)\n",
    "    # nx512 (note: no activation in the last layer) \n",
    "    net = tf.layers.dense(net, units=10)\n",
    "    # nx10 \n",
    "    return net\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, 32, 32, 3])\n",
    "y = tf.placeholder(tf.int64, [None])\n",
    "is_training = tf.placeholder(tf.bool)\n",
    "\n",
    "y_out = my_model(X,y,is_training)\n",
    "loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=y_out)\n",
    "mean_loss = tf.reduce_mean(loss)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)\n",
    "tf.summary.scalar(\"cross_entropy_loss\", mean_loss)\n",
    "\n",
    "# batch normalization in tensorflow requires this extra dependency\n",
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(extra_update_ops):\n",
    "    train_step = optimizer.minimize(mean_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Iteration 0: with minibatch training loss = 2.83 and accuracy of 0.078\n",
      "Epoch 1, Overall loss = 1.48 and accuracy of 0.472\n",
      "Iteration 1000: with minibatch training loss = 1.1 and accuracy of 0.66\n",
      "Epoch 2, Overall loss = 1.09 and accuracy of 0.623\n",
      "Iteration 2000: with minibatch training loss = 0.847 and accuracy of 0.72\n",
      "Epoch 3, Overall loss = 0.956 and accuracy of 0.674\n",
      "Iteration 3000: with minibatch training loss = 0.933 and accuracy of 0.62\n",
      "Epoch 4, Overall loss = 0.856 and accuracy of 0.712\n",
      "Epoch 5, Overall loss = 0.792 and accuracy of 0.737\n",
      "Iteration 4000: with minibatch training loss = 0.575 and accuracy of 0.77\n",
      "Epoch 6, Overall loss = 0.741 and accuracy of 0.754\n",
      "Iteration 5000: with minibatch training loss = 1.05 and accuracy of 0.7\n",
      "Epoch 7, Overall loss = 0.696 and accuracy of 0.771\n",
      "Iteration 6000: with minibatch training loss = 0.929 and accuracy of 0.77\n",
      "Epoch 8, Overall loss = 0.644 and accuracy of 0.788\n",
      "Epoch 9, Overall loss = 0.603 and accuracy of 0.801\n",
      "Iteration 7000: with minibatch training loss = 0.42 and accuracy of 0.81\n",
      "Epoch 10, Overall loss = 0.556 and accuracy of 0.816\n",
      "Iteration 8000: with minibatch training loss = 0.426 and accuracy of 0.81\n",
      "Epoch 11, Overall loss = 0.531 and accuracy of 0.824\n",
      "Iteration 9000: with minibatch training loss = 0.517 and accuracy of 0.83\n",
      "Epoch 12, Overall loss = 0.489 and accuracy of 0.838\n",
      "Epoch 13, Overall loss = 0.461 and accuracy of 0.848\n",
      "Iteration 10000: with minibatch training loss = 0.224 and accuracy of 0.94\n",
      "Epoch 14, Overall loss = 0.431 and accuracy of 0.857\n",
      "Iteration 11000: with minibatch training loss = 0.49 and accuracy of 0.8\n",
      "Epoch 15, Overall loss = 0.405 and accuracy of 0.868\n",
      "Iteration 12000: with minibatch training loss = 0.313 and accuracy of 0.91\n",
      "Epoch 16, Overall loss = 0.381 and accuracy of 0.876\n",
      "Iteration 13000: with minibatch training loss = 0.347 and accuracy of 0.86\n",
      "Epoch 17, Overall loss = 0.356 and accuracy of 0.884\n",
      "Epoch 18, Overall loss = 0.349 and accuracy of 0.887\n",
      "Iteration 14000: with minibatch training loss = 0.341 and accuracy of 0.83\n",
      "Epoch 19, Overall loss = 0.328 and accuracy of 0.893\n",
      "Iteration 15000: with minibatch training loss = 0.221 and accuracy of 0.89\n",
      "Epoch 20, Overall loss = 0.31 and accuracy of 0.898\n",
      "Iteration 16000: with minibatch training loss = 0.373 and accuracy of 0.91\n",
      "Epoch 21, Overall loss = 0.305 and accuracy of 0.903\n",
      "Epoch 22, Overall loss = 0.293 and accuracy of 0.905\n",
      "Iteration 17000: with minibatch training loss = 0.0869 and accuracy of 1\n",
      "Epoch 23, Overall loss = 0.281 and accuracy of 0.911\n",
      "Iteration 18000: with minibatch training loss = 0.403 and accuracy of 0.89\n",
      "Epoch 24, Overall loss = 0.268 and accuracy of 0.915\n",
      "Iteration 19000: with minibatch training loss = 0.279 and accuracy of 0.89\n",
      "Epoch 25, Overall loss = 0.269 and accuracy of 0.916\n",
      "Epoch 26, Overall loss = 0.255 and accuracy of 0.92\n",
      "Iteration 20000: with minibatch training loss = 0.102 and accuracy of 0.97\n",
      "Epoch 27, Overall loss = 0.248 and accuracy of 0.923\n",
      "Iteration 21000: with minibatch training loss = 0.273 and accuracy of 0.91\n",
      "Epoch 28, Overall loss = 0.236 and accuracy of 0.925\n",
      "Iteration 22000: with minibatch training loss = 0.203 and accuracy of 0.92\n",
      "Epoch 29, Overall loss = 0.234 and accuracy of 0.929\n",
      "Epoch 30, Overall loss = 0.234 and accuracy of 0.927\n",
      "Iteration 23000: with minibatch training loss = 0.287 and accuracy of 0.91\n",
      "Epoch 31, Overall loss = 0.24 and accuracy of 0.928\n",
      "Iteration 24000: with minibatch training loss = 0.164 and accuracy of 0.95\n",
      "Epoch 32, Overall loss = 0.233 and accuracy of 0.93\n",
      "Iteration 25000: with minibatch training loss = 0.158 and accuracy of 0.95\n",
      "Epoch 33, Overall loss = 0.223 and accuracy of 0.933\n",
      "Iteration 26000: with minibatch training loss = 0.131 and accuracy of 0.94\n",
      "Epoch 34, Overall loss = 0.228 and accuracy of 0.932\n",
      "Epoch 35, Overall loss = 0.22 and accuracy of 0.934\n",
      "Iteration 27000: with minibatch training loss = 0.279 and accuracy of 0.91\n",
      "Epoch 36, Overall loss = 0.212 and accuracy of 0.937\n",
      "Iteration 28000: with minibatch training loss = 0.351 and accuracy of 0.92\n",
      "Epoch 37, Overall loss = 0.215 and accuracy of 0.937\n",
      "Iteration 29000: with minibatch training loss = 0.246 and accuracy of 0.94\n",
      "Epoch 38, Overall loss = 0.211 and accuracy of 0.941\n",
      "Epoch 39, Overall loss = 0.2 and accuracy of 0.942\n",
      "Iteration 30000: with minibatch training loss = 0.207 and accuracy of 0.94\n",
      "Epoch 40, Overall loss = 0.215 and accuracy of 0.938\n",
      "Iteration 31000: with minibatch training loss = 0.144 and accuracy of 0.98\n",
      "Epoch 41, Overall loss = 0.198 and accuracy of 0.944\n",
      "Iteration 32000: with minibatch training loss = 0.0176 and accuracy of 0.98\n",
      "Epoch 42, Overall loss = 0.199 and accuracy of 0.944\n",
      "Epoch 43, Overall loss = 0.211 and accuracy of 0.94\n",
      "Iteration 33000: with minibatch training loss = 0.0608 and accuracy of 0.98\n",
      "Epoch 44, Overall loss = 0.196 and accuracy of 0.945\n",
      "Iteration 34000: with minibatch training loss = 0.35 and accuracy of 0.92\n",
      "Epoch 45, Overall loss = 0.195 and accuracy of 0.944\n",
      "Iteration 35000: with minibatch training loss = 0.198 and accuracy of 0.97\n",
      "Epoch 46, Overall loss = 0.191 and accuracy of 0.947\n",
      "Iteration 36000: with minibatch training loss = 0.164 and accuracy of 0.92\n",
      "Epoch 47, Overall loss = 0.199 and accuracy of 0.946\n",
      "Epoch 48, Overall loss = 0.194 and accuracy of 0.946\n",
      "Iteration 37000: with minibatch training loss = 0.245 and accuracy of 0.91\n",
      "Epoch 49, Overall loss = 0.196 and accuracy of 0.947\n",
      "Iteration 38000: with minibatch training loss = 0.136 and accuracy of 0.95\n",
      "Epoch 50, Overall loss = 0.209 and accuracy of 0.942\n",
      "Validation\n",
      "Epoch 1, Overall loss = 0.795 and accuracy of 0.806\n"
     ]
    }
   ],
   "source": [
    "# from tensorflow.python import debug as tf_debug\n",
    "merged=tf.summary.merge_all()\n",
    "sess = tf.Session()\n",
    "# sess = tf_debug.TensorBoardDebugWrapperSession(sess, \"e9a3475a5764:6005\")\n",
    "writer = tf.summary.FileWriter(logdir=\"logs/50-epochs\", graph=sess.graph)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print('Training')\n",
    "run_model(writer, merged, sess,y_out,mean_loss,X_train,y_train,50,64,1000,train_step)\n",
    "print('Validation')\n",
    "run_model(writer, merged, sess,y_out,mean_loss,X_val,y_val,1,64)\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test your model here, and make sure \n",
    "# the output of this cell is the accuracy\n",
    "# of your best model on the training and val sets\n",
    "# We're looking for >= 70% accuracy on Validation\n",
    "# print('Training')\n",
    "# run_model(sess,y_out,mean_loss,X_train,y_train,1,64)\n",
    "# print('Validation')\n",
    "# run_model(sess,y_out,mean_loss,X_val,y_val,1,64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'run_model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-fa1fe81cff4f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrun_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_out\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmean_loss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'run_model' is not defined"
     ]
    }
   ],
   "source": [
    "print('Test')\n",
    "run_model(sess,y_out,mean_loss,X_test,y_test,1,64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}