int8

Author: vinhngx

TFTRT-int8-calibration-from-saved-model.ipynb

@@ -503,7 +503,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [
     {
@@ -514,7 +514,7 @@
       "INFO:tensorflow:The specified SavedModel has no variables; no checkpoints were restored.\n",
       "INFO:tensorflow:Running against TensorRT version 5.1.2\n",
       "Calibrate model on calibration data...\n",
-      "Calibration accuracy: 82.03%\n"
+      "Calibration accuracy: 75.38%\n"
      ]
     }
    ],
@@ -535,7 +535,7 @@
     "        precision_mode='INT8')\n",
     "\n",
     "#Then calibrate it with 2 batchs of examples\n",
-    "N_runs=2\n",
+    "N_runs=100\n",
     "with tf.Session(graph=tf.Graph()) as sess:\n",
     "    \n",
     "    output_node = tf.import_graph_def(\n",
@@ -558,7 +558,7 @@
     "    for _ in range(N_runs):\n",
     "            image_data = sess.run(next_element)    \n",
     "            img = image_data[0]\n",
-    "            label = image_data[1]\n",
+    "            label = image_data[1].squeeze()\n",
     "            output = sess.run(output_node[0].outputs[0], feed_dict={\"trt/input_tensor:0\": img})\n",
     "            prediction = np.argmax(output, axis=1)\n",
     "            num_hits += np.sum(prediction == label)\n",
@@ -588,7 +588,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
@@ -598,7 +598,7 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mFailedPreconditionError\u001b[0m                   Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-35-37c230aa2655>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#Create Int8 inference model from the calibration graph and write to a saved session\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrt_int8_calibrated_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalib_graph_to_infer_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrt_int8_calib_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m output_node = tf.import_graph_def(\n\u001b[1;32m      4\u001b[0m         \u001b[0mtrt_int8_calibrated_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mreturn_elements\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'softmax_tensor:0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m<ipython-input-38-774a4b39c75b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#Create Int8 inference model from the calibration graph and write to a saved session\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrt_int8_calibrated_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalib_graph_to_infer_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrt_int8_calib_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m output_node = tf.import_graph_def(\n\u001b[1;32m      4\u001b[0m         \u001b[0mtrt_int8_calibrated_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mreturn_elements\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'softmax_tensor:0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/tensorrt/python/trt_convert.py\u001b[0m in \u001b[0;36mcalib_graph_to_infer_graph\u001b[0;34m(calibration_graph_def, is_dynamic_op)\u001b[0m\n\u001b[1;32m    415\u001b[0m     \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    416\u001b[0m     raise _impl._make_specific_exception(None, None, \";\".join(msg[1:]),\n\u001b[0;32m--> 417\u001b[0;31m                                          int(msg[0]))\n\u001b[0m\u001b[1;32m    418\u001b[0m     \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    419\u001b[0m   \u001b[0moutput_graph_def\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgraph_pb2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphDef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mFailedPreconditionError\u001b[0m: Need to run graph with calibration data first!"
      ]
@@ -638,7 +638,7 @@
     "        while True:        \n",
     "            image_data = sess.run(next_element)    \n",
     "            img = image_data[0]\n",
-    "            label = image_data[1]\n",
+    "            label = image_data[1].squeeze()\n",
     "            output = sess.run(['softmax_tensor:0'], feed_dict={\"input_tensor:0\": img})\n",
     "            prediction = np.argmax(output[0], axis=1)\n",
     "            num_hits += np.sum(prediction == label)\n",