CIFAR10/performance-small.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR-10 dataset - Model (c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model directory in an env variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "#change the directory is needed\n",
    "os.environ['CIFAR10_DIR']='logdir/1527069102'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform operations on the graph to obtain the quantized model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Freeze the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 12 variables to const ops.\r\n"
     ]
    }
   ],
   "source": [
    "!freeze_graph --input_saved_model_dir=$CIFAR10_DIR --output_graph=$CIFAR10_DIR/frozen.pb \\\n",
    "    --output_node_names=final_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m tensorflow.python.tools.optimize_for_inference --input=$CIFAR10_DIR/frozen.pb \\\n",
    "    --output=$CIFAR10_DIR/optimized.pb --input_names=\"input\" --output_names=\"final_result\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the optimized graph to Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m scripts.graph_pb2tb $CIFAR10_DIR $CIFAR10_DIR/optimized.pb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the quantized model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/emanuele/Desktop/notebooks/Tensorflow-Neural-Network-Quantization/CIFAR10/scripts/quantize_graph.py:299: quantize_v2 (from tensorflow.python.ops.array_ops) is deprecated and will be removed after 2017-10-25.\r\n",
      "Instructions for updating:\r\n",
      "`tf.quantize_v2` is deprecated, please use `tf.quantize` instead.\r\n"
     ]
    }
   ],
   "source": [
    "!python -m scripts.quantize_graph --input=$CIFAR10_DIR/optimized.pb --output=$CIFAR10_DIR/eightbit.pb \\\n",
    "    --output_node_names=final_result --mode=eightbit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23M\tlogdir/1527069102/optimized.pb\n",
      "5,6M\tlogdir/1527069102/eightbit.pb\n"
     ]
    }
   ],
   "source": [
    "!du -h $CIFAR10_DIR/optimized.pb\n",
    "!du -h $CIFAR10_DIR/eightbit.pb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy, Average prediction time and cache utilization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### not-quantized model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060340089s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060484042s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060390645s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060484988s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060461859s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060534921s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060559824s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060505603s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060547477s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.694000000\n",
      "Average prediction time: 0.060533311s\n",
      "\n",
      " Performance counter stats for 'python evaluate.py --graph logdir/1527069102/optimized.pb' (10 runs):\n",
      "\n",
      "    45.616.217.287      cache-references                                              ( +-  0,10% )\n",
      "    18.251.882.939      cache-misses              #   40,012 % of all cache refs      ( +-  0,17% )\n",
      "\n",
      "     621,121758162 seconds time elapsed                                          ( +-  0,05% )\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!perf stat -r 10 -e cache-references,cache-misses python evaluate.py --graph $CIFAR10_DIR/optimized.pb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### quantized model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036642184s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036481655s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036675648s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036602279s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036577716s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036631594s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036575865s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036633570s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036636642s\n",
      "Evaluating model with 10000 images\n",
      "               Accuracy: 0.688200000\n",
      "Average prediction time: 0.036547622s\n",
      "\n",
      " Performance counter stats for 'python evaluate.py --graph logdir/1527069102/eightbit.pb' (10 runs):\n",
      "\n",
      "    29.970.937.353      cache-references                                              ( +-  0,05% )\n",
      "     3.969.561.036      cache-misses              #   13,245 % of all cache refs      ( +-  0,15% )\n",
      "\n",
      "     379,374719992 seconds time elapsed                                          ( +-  0,05% )\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!perf stat -r 10 -e cache-references,cache-misses python evaluate.py --graph $CIFAR10_DIR/eightbit.pb"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}