11_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 11 – Deep Learning**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 11._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Vanishing/Exploding Gradients Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sigmoid_saturation_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFX3wPHvDaQXOiIKBikCkabSFDWIWJEm+FroYHtV\nlPKzIIooKIo0ewWkCFghSBEFIoKACEpRXkAlIhA6CYT05Pz+mE1INrshCbvZ3eR8nmceMrN3Z85e\nJjl7y8wYEUEppZTyNn6eDkAppZRyRBOUUkopr6QJSimllFfSBKWUUsoraYJSSinllTRBKaWU8kqa\noFSJGWNWG2Pe8HQcULRYjDHbjTHPl1ZMeY47wxgTUwrHud4Yk2WMqVoKx3rAGPOPMSbTE3VqF0t/\nY8wpT8ag3MPodVDKEWNMdeBF4FbgQiAB2A5MEJGVtjKVgQwROeOxQG2KEosxZjvwuYi86KYYrgdW\nA9VF5ESe7eFYv2su+yNqjNkLvCkik/NsqwhUFZEjrjqOk2NXBo4ATwBfAEkikuzOY+Y5djbQS0S+\nyrMtEAgXkWOlEYMqPRU9HYDyWl8BQcBA4C+gJnA9UC2ngIgkeCa0grwkFgOI7d9cInK6NA4uIplY\nicPdIoEKwBJ3J8OiEJE0IM3TcSg3EBFddMm3AJWAbOCGc5RbDbyRZ70mEAMkA3uBAVitrufzlMkG\nHgIWAmeAXUA0cBGwHEgCfgVa2R2rJ7ANSAX2AaPOEUsNYFGeWAbax+Lg81xqiyveFsdm4Ha7Mv7A\ny0CcLZY/gUeBS2yfLSvPv9Nt75kJxNh+fgA4BPjZ7fdT4OuixGH7rPmOZdsebVuvWox62ws8C7wH\nJAL/AiMLqaP+Dj5nXWAMsN1B2dN51sfY/g/+Y6u3U8DXeePN876cmA/lqce9eY6bDfxt2z4g73Fs\n2x4E9mAlrj3AELvXs4H7gc9sdfwXcJ+nf/d0yb/oGJRyJMm2dLV1nxTVLKAO1h/KbkAfrD9e9p7F\n+oPcHNgEzAM+At4GWgIHgRk5hY0xV2L9IfkCuBx4CnjGGPNIIbF8gvWH/gagO9APK4kUJgxYCnSy\nxfYF8KUxppHdZ+yD1b3VGBiM1f25D7jTVqYJVrfo47b1vP3on2F9Abgxz+cLAboCs4sYR09gPzAW\nqGU7Vs5xco9VjHp7AishtAJeBV4zxrR1UkfzgVtsP19lO/Z+B58TJ9sigbuwzo/OtmOOzxPzg1jJ\n8mOgGVYX8++2l1tjtU4H2z53ayefuwfwJjAZiAKmAe8YY263i+U5rATZHFgATDfG1HHyuZUneDpD\n6uKdC9ADOAakAD8BE4E2dmVyWy3AZVjfSlvnef1iIJOCLahxedajbNsez7PteqxvylVt63OA7+2O\nPQbY5ySWRrZ9tsvzel37WIpYD+uxtTqAhrb9dnZSNl/cebbPwNaCsq1/BXySZ70PcBIIKEoctvW9\nwPDCjl/EetsLzLUrsxu7lpbd61fajlPXbr/b7Mr1B07ZlUkGwvJsGwXszrP+LzC+kGNnAz3PcZy1\nwIcO/g/WFHIeVsBq0d9b2r9rujhftAWlHBKRr4HaQBesb/PtgQ3GmKedvOUyrD9am/PsYz9Wa8je\n9jw/H7b9u8PBtpq2f5sA6+z2sRa4yBgT5mD/jW2xbMoTyz4nseQyxoQYY14zxvxujDlhjDmN9cc4\npxXY0rbf2ML2UwRzgO7GmCDb+r3AFyKSXsQ4iqqo9bbNrsxBzta9q/0jIkmOjmWMqYHV1bvqPI/R\nBOtLVV5rgaZ223LPQxHJAo7ivs+tSkATlHJKRNJFZKWIjBORDljdLi/YZovZMw62OZOR9zBOthnO\nnp8Gx91HONlenFjymoTVTfcscB3QAivJBZznfu19g5Xoutn+KN/I2e69osRRVEWttwwHrxX3b0M2\nBevH30G5wo7lqvrN2e+5trnicys30v8MVRw7sWZ+Bjl5zQ/rmz4AxpiLsVphJZH3j8kfQAe7168F\n9ovjaeU5seSMUWCMqVuEWK4BZonIQhHZgfXtvn6e17fY9tvRyfvTbf9WKOwgtpbSF1hde/8B4kVk\nTTHiyDlWoceh+PV2Po4CF9hta1WcHYg1I/AA1tibMxmc+3PvxPHn/qM48SjP0wSlCjDGVDXGrDTG\n3GeMaWaMiTTG9Ab+D2tMI8n+PSKyG1gBvG+MaWuMaQlMx+rXL8nFdnm/TU8CrjfGjDHGNDTG3AcM\nxxrQL8AWy7e2WNrZYpmBNf5RmN1AD2NMK2NMM6xWTe4kERH5E/gc+MgY09NWLx2MMX1sRf7B+qy3\nG2OqG2NCCznWHOBmrBmNnxYnDps44FpjTG1jTLU820tcb8Vk39qJBaoaY0YZYy41xgzm7KSR4hgP\nPGGMecIWc0tjzPA8r8cBnYwxF9iux3JkItDXGPNfY0wDY8xjwD245nOrUqQJSjmShDUoPxTrD88O\nYBzWH9W785SzTzz9sQa5V2NNk56D9c06tZD3nHObiPwK9MaavbYda5r3yyLyzjli2QusxJpuPhfr\nj1thhmNdR7QGWIJVBz/alemLlVCmYX1TnwFE2OI8iDURYDzW9Og3nR3I1mI6gDVeNqcEcTyPNWPy\nL/Jf+3S+9eZsW6FlROR/wMNYU7e3YrWCxjt4X+E7FXkPeAQYYot5KfnHjkZgtWD/xWrROtrHIuAx\nrNmJv9t+flhEljqLv5BtyoP0ThLKbWzf7A8Cd9smXSilVJFpglIuY4zpCIRjffO9AOsbdCOgkYik\neDI2pZTv0VsdKVfyx+oKrIc13rMBuF6Tk1KqJLQFpZRSyit5fQvKGKMZVCmlyjgRKXAdnE/M4vP0\n7TYcLf379/d4DL6yaF0Vb/HWc94bF0+cW5lZmfx+5HePf3Zfqa+iLM74RIJSSilvUcGvAk1r2N81\nSbmDJqgSioyM9HQIPkPrSrmLnlvF42v1pQmqhKKjoz0dgs/QulLuoudW8fhafWmCUkopJ9Kz0hm2\nfBgnU056OpRyyetn8SmllCdkSzb9F/YnNTOV8MBwT4dTLnn9dVDGGPH2GJVyJWNMoTOblPuJCE8s\nf4JfD/3Kt32+Jdg/2NMhlWm2c77ANHNtQSmllJ1X173K6rjVrBm4RpOTB+kYVAnFxsZ6OgSfoXWl\n3MUd59YPcT/w/ub3Wd5nOZWDnD3Rwzf52u+itqCUUiqPay+5lvWD11MrrJanQyn3dAxKKS+jY1Cq\nvHE2BqVdfEoppbySJqgS8rW+XE/SulLuoudW8fhafWmCUkqVWydSTtD7894kpSd5OhTlgI5BKeVl\ndAyqdCRnJHPjrBvpULcDr3V+zdPhlGvOxqA0QSnlZTRBuV9GVgY9FvSgWkg1ZnSbgZ/RziRP0kkS\nLuZrfbmepHWl3KUk55aI8MA3D5At2Xx0x0flKjn52u+iXgellCpXYnbFsPPoTlb2W4l/BX9Ph6MK\noV18SnkZ7eJzLxEhOSOZ0IBQT4eibHQMSikfoQlKlTc6BuVivtaX60laV8pd9NwqHl+rL01QSqky\nTVujvku7+JTyMtrF5zq7j+9mcMxgvu/7PYEVAz0djnJCu/iUUuVK/Ol4bplzC/1b9Nfk5KM0QZWQ\nr/XlepLWlXIXZ+dWYmoit869lUGtBjHkiiGlG5QX87XfRU1QSqkyJTUzle4LutOhbgeevfZZT4ej\nzoOOQSnlZXQM6vzM2z6Pr//3NfPunEcFvwqeDkcVgV4HpZSP0AR1/rKyszQ5+ZBSmSRhjHnEGLPJ\nGJNqjJl+jrLDjDHxxpiTxpiPjDE+dc8RX+vL9SStK+Uuzs4tTU6O+drvoqvHoA4ALwEfF1bIGHMz\n8CTQEYgE6gNjXRyLUkopH+aWLj5jzEvARSIyyMnrc4G9IjLatn4DMFdELnRQVrv4VLmiXXzFk5GV\noTd99XHedh1UFLA1z/pWoKYxpoqH4lFK+aA1/6yh7UdtycrO8nQoyg08laDCgMQ864mAAcI9E07x\n+VpfridpXSl32HZ4G90mdGNi54k65lRE3vC7KAIpKXDiBBw4AH/+WWhhcfmCNQ41vZDXfwN65Vmv\nCmQBVRyUFUfLmDFjRERk9erVsnr1asnRv39/La/lfbq89WvpPfF4Y/m9J/fKRZMukuY3N/eKeMpm\n+UCBCIEL5cEH35U1a0Refnm1PPfcavngA5HXXxdp0WKKwEMC7wnMEnhB4BWJjPxb2rYVufTS1VK7\n9mq56CKRqlVF/Py+E1htW8YI9LctiDjIFZ4cg/pbRJ6zrd8AzBGR2g7KijtiVMpb6RhU4Y6eOUqH\nGR14tPWjPNb2MU+H49VErJbKsWPWvznL8ePO1xMS4NQpyMx0X1yBgRAcfHb5+2/HY1AufaKuMaYC\n4A9UACoaYwKBTBGx7yCeBcwwxnwKHAKeBWa4MhalVNm07M9l9G7au1wnp4wMq3ts/344dMj5cuSI\nVbYk/P0hIgLCw60l78/262FhEBp6NuGEhORPQHnXg4Kggl2PrCmQmmzbXflNzRgzBhiD1UTMMRYr\n+fwBNBGR/bayTwBPA0HAF8DDIlKgKr21BRUbG0t0dLSnw/AJWlfFoy2ooiur51ZiIsTFwb591vLP\nP2d/3rcPDh60WkdFEREBNWpAtWogEkvDhtFUqwZVq1pL3p+rVoUqVaykE+jG++v+9ttvDBs2jEWL\nFhEREeF0Fp9LW1AiMhbn1zOF25WdCkx15fGVUspXnDljTRDYvRv27Dn77549cPRo4e/184OLLoKL\nL4YLL4RatazlggvO/pyzHhR09n2xseDJfC4ivPnmmzz99NOkp6eTmJhIRESE0/J6qyOlvIy2oMqW\n9HTYtQu2b8+/7Nvn/D3BwVCvHtStay2XXHL257p1reTk72OXfiUkJHDPPfewZs0akpOTCQkJYdeu\nXVx88cWl04JSSilXS0xNpFJQJU+HUSQpKfDrr/Dzz7BpE2zbZiUnR+NA/v5Qvz40bAiNGuX/t3Zt\nq5VUVmzYsIFu3bqRkJBAeno6YLWm/M7xITVBlVBZ7ft2B60rVVKzts7izZ/f5OchP2McjKR78tzK\nyoKdO61klLNs22Ztt3fppdCsWf6lYUOoWMp/gUu7vrKzs5kwYQLjxo0jJSWlwOuaoJRSPmnpnqU8\n+d2TrO6/2mFyKm0ZGbBlC/zwg7WsXWtNx87Lzw+aN4fWra2lZUuIirJmuZU3R48epVevXmzevNlh\nchKRc/6/6hiUUl5Gx6Bgw/4NdJ3XlZh7Ymh3cTuPxCACv/8Oy5fD999bCenMmfxlLrkE2raFNm2s\n5YorrOnW5V1sbCw9e/YkKSmJDCfz3IOCgti3bx81atTQMSillG/YeXQn3ed3Z2b3maWenBISYOVK\nKyktX25dZ5RXw4Zw/fXWTLjrr7dm0amzsrKyeP7555kyZYrDVlNeRWlBaYIqIR1XKTqtK1UcPx/4\nmVdvfJXbGt52zrKuOLfi42HhQvjqK2sadt47KFxwAdx8s7VER1uTF3yZu38Xx4wZwyuvvFLkHgAd\ng1JK+ZT+Lfu7/Rh798KXX1pJacOGsxe9VqgA110Ht9xiLS1alK3ZdO42YsQI0tPTefvtt8nKyiIt\nLc1p2aLM4tMxKKW8jI5BuceJE/DZZzBnDqxbd3Z7UJDVQurRA+64w7qbgjo/x44d45FHHmHx4sVO\nu/oCAgI4cuQIlSpV0jEopVT5k54OS5bA7NnWv7ZLcAgJga5d4c47rZZSeZxl507Vq1fn8OHDpKam\nOi1TlBaUNl5LyBueq+IrtK6UMyLCvsRCbqlwDs7Orbg4eOYZqFMHevaEr7+2xpY6d4ZZs+DwYZg3\nD3r1Kl/JqbR+Fzdu3MimTZvy9QSEhobi7++Pv+0WGJqglFJe7fnVzzM4ZrBL9pWVBd98A7ffbl0Y\nO2GCdTfvyy+HiRPh339hxQro27d8JSVPGDFiBMnJyfm2hYSEsHv3bvr160dQUBCZmZl6HZRSvqa8\njEG99fNbvLHxDdYOWkvN0Jol3s/p0/DhhzBt2tn72wUEwF13wUMPwdVXO3+cg3K9devWcdNNN+VL\nUKGhoUydOpUhQ4YAsG/fPr7++muGDh2KMcbpGJQmKKW8THlIUJ/9/hnDvx3OjwN/pF6VeiXax6FD\n8MYb8M471uMpwLq33YMPwsCBUL26CwNWRdauXTs2btyYb1utWrXYt29fbveePWcJSrv4SkjHVYpO\n60rltfLvlTy69FGW3LukRMlp92544AHrLg6vvBJLYiJcey3ExFiv/d//aXJyxt2/iz/88AM7duzI\nty00NJQJEyY4TU6F0Vl8SqlSdfjMYT7r/RktarUo1vv++gtefNGaJp6dbXXbdegAr70G7du7KVhV\nZCLCiBEjOGN3P6jKlStz3333lWif2sWnlJcpD118xbFvH4wbBzNmWDPxKlaEAQNg5Ei47DJPR6dy\nrFq1iq5du+ZLUKGhobz//vvnTFA6BqWUj9AEZTl0CF5+Gd5/37p+yc8P+vWD55+3HuanvIeI0KpV\nK7Zu3Zpve506ddi7dy8VKlQo9P06BuViOq5SdFpXqjhSU+GVV6wbs775pvWYi3vugT/+sFpReZOT\nnlvF4676+u677/jzzz/zbQsNDWXixInnTE6F0QSllHKbtMw0th3eVqSyItb98Zo2hVGjICnJuvXQ\n1q3w6afaneetRIThw4cXGHuqWbMmvXv3Pq99axefUl6mrHTxZUs293x5DwEVApjdY3ahZX/7DZ54\nwnoQIFgP+Zs6FW68sRQCVedl6dKl3HXXXQXGnj755BPuvPPOIu1Dx6CU8hFlIUGJCEOXDWX7ke0s\n77OcoIpBDsudPg3PPgtvvWW1oKpVg5degvvvL/3HoaviExGaNGnCrl278m1v2LAhu3btKvKTkHUM\nysW077votK7Kn1fWvsKP+35k0d2LnCanmBirO+/NN60JEE88AXv2wMMPFz056blVPK6ur8WLF3Pg\nwIF820JDQ5k0aVKRk1NhXJqgjDFVjDFfG2OSjDF7jTH3OCkXYIx5zxhzyBhzzBizyBhzoStjUUp5\nxsdbPuajLR+x7L5lVAqqVOD1+HjrJq3dullPrG3dGjZvhilToEoVDwSsSiQ7O5vhw4eTlJSUb3ud\nOnXo0qWLS47h0i4+Y8w824+DgCuAJUB7EdlpV+5J4B6gM3AK+AgIEZFeDvapXXyqXPH1Lr5F/1tE\nkxpNaFStUb7tIvDBB/DUU9atiUJDYfx4ePRR60GByrd8+eWX9O/fv8DY0+eff86tt95arH25fQzK\nGBMCnASaishftm2zgP0iMsqu7DvAKRF52rZ+GzBJRJo42K8mKFWu+HqCcuTgQRg0CL791lq//Xbr\nHnp163o2LlUy2dnZ1K9fn7i4uHzbo6Ki2L59e7G790pjDKoRkJmTnGy2AlEOyn4MdDDGXGhLbPcB\nS10Yi9tp33fRaV2Vb19+Cc2aWcmpalXrOUyLF7smOem5VTyuqq+//vqLffv25XueU2hoKFOmTHHJ\n2FMOVyaoMCDRblsiEO6g7G5gH3AASAAaAy+5MBallIclJkL//tZ404kT1mPVt2+Hu+/Wx1/4uoYN\nG/L777/TpUsXgoODMcbQoEEDbnTxdQGunMiZBETYbYsATjso+x4QCFQBkoGngOVAO0c7HjBgAJGR\nkYB148GWLVsSHR0NnP1GoOvevZ7DW+Lx9vUc3hKPs/VFyxfxT8I/DL17aL7XK1aMpk8f+OefWPz9\nYcqUaP77X/jhh1h273bd8XO2eUt9ePt6zjZX7K9x48YMGzaM7t27s2zZMoYPH84PtgvZinJ+x8bG\nFugitOfqMagTQFSeMahPgAMOxqC2A6NEZLFtvRLW+FV1ETlhV1bHoFS54itjUGfSz9BpVic61evE\n+E7jAesu46+9Zl3blJ0NV15p3X28cWMPB6u8mtvHoEQkGfgKeNEYE2KMuQboCji6hHwT0M8YE2GM\n8QcewUpkJxyU9Ur233SVc1pXZU9GVga9P+9N4+qNGXfDOMDqxuvWDZ55xkpOTz0F69e7NznpuVU8\nvlZfrr5Q9xEgBDgCzAUeEpGdxpgOxphTecqNBNKAPcBh4Bagh4tjUUq5gYgwZPEQ/IwfH97xIcYY\nfv4ZrrgCvvnGupZp8WKYMAFK8Iw6pXLprY6U8jLe3sU3auUoVsetZmW/lQRXDOHtt2H4cOuu461b\nw2efgW3IWKki0XvxKeUjvD1BxcbF0qxmM0L9qnH//dYYE1gX3L7+OgQGejY+5Xv0Xnwu5mt9uZ6k\ndVW2REdGk5ZQjeuvt5JTaCjMn2/dU6+0k5OeW8Xja/WlCUopVSy//GJ15f38M1xyCfz0E/znP56O\nSjnTsWNHhg4d6ukwSkS7+JTyMt7cxTd/PgwcaD319tprrbtE1Kjh6ahc79ixYzz//PMsW7aM+Ph4\nKleuTLNmzXj66afp1KnTOd//ww8/0LFjR44dO0bVqlVLIWL45JNPePTRRzl9Ov+lpwkJCfj7+xMa\nGloqcZSEsy4+feKKUsqp3cd3s+f4Hm5tcDvPP2/d3BVgyBB4+20ICPBsfO7Ss2dPUlNTmTFjBvXr\n1+fIkSP88MMPHD9+vEjvFxGXfdHIyMjAvwjTIXOOaa9y5crnHYPHiIhXL1aI3mf16tWeDsFnaF0V\nj7ec8wdOHZDIqZHy7vrpcvfdIiDi5ycybZpIdrano7O449xKSEgQY4ysXLnSaZk5c+ZI69atJTw8\nXGrWrCm9e/eWAwcOiIhIXFycGGPEz88v99+BAweKiEh0dLQ89thj+fY1YMAAueOOO3LXo6Oj5eGH\nH5aRI0dKjRo1pE2bNiIiMnnyZGnevLmEhobKRRddJEOGDJHExEQREYmNjS1wzLFjxxY45urVqyUy\nMlLGjRsnDz74oERERMjFF18sEydOzBfT7t275brrrpOgoCBp3LixLF26VMLCwuSTTz45n6p1ynbO\nF/j7r2NQSqkCElITuHXurfRp+CgLnhnI/PkQFgZLl8LQoWX7XnphYWGEhYURExNDWlqawzIZGRm8\n+OKLbNu2jSVLlnD8+HHuvfdewHoe0pdffgnAzp07iY+PZ9q0acWKYe7cuQCsXbuWWbNmAVChQgWm\nTZvGH3/8wbx589i0aROPPfYYAFdffTVTp04lJCSEw4cPEx8fz8iRI53uf+rUqTRv3pxff/2Vp556\niieffJKNGzcCVqOle/fuBAQE8PPPPzNz5kzGjh1Lenp6sT6DSzjKWt604CXfJpUqLZ4+51MyUuS6\nGdfJwFnPyeWXZwuI1Kol8uuvHg2rVH311VdSrVo1CQoKkvbt28vIkSNl48aNTsvv3LlTjDG5rajY\n2Fjx8/OT48eP5ytX1BZUixYtzhnj8uXLJSgoKHd95syZEh4eXqCc/TEjIyPl3nvvzVemYcOGMn78\n+Nz9+vv7S3x8fO7rP/30kxhjtAWllPKsITFDCD7Rmu9GjWXHDkPjxrBhA7Rs6enISk+PHj04ePAg\n33zzDbfddhvr16+nXbt2TJgwAYAtW7bQvXt3IiMjiYiIoHXr1hhj2Ldvn0uOf+WVVxbYtmrVKm66\n6Sbq1KlDREQEPXv2JD09nUOHDhV7/82bN8+3Xrt2bY4cOQLArl27qF27NrVq1cp9vXXr1vkerVFa\nNEGVkK9dT+BJWle+5XoZw4YXJ7J/v6FDB1i3zppO7o3ceW4FBATQqVMnRo8ezdq1axk8eDAvvPAC\np06d4pZbbiEsLIw5c+bwyy+/sHz5ckTknN1gfn5+BSZOZGRkFChnP+Nu3759dOnShaioKL744gu2\nbNnC9OnTAYrV9ZZTX/aTLowxZGdnA84nW3iCzuJTSuVatAgevbch6elw553WhbhBQZ6Oyjs0adKE\nzMxMfvvtN44dO8b48eO5xJa5d+zYke+PeoBtemNWVla+fdSoUYP4+Ph827Zu3Uq9evUKPfYvv/xC\nRkYGkydPzj1OTExMvjIBAQEFjlcSTZo04cCBAxw6dCi3FbVp06bcBFaatAVVQnmfr6IKp3XlG+bO\ntZJSejr897+wYIH3Jyd3nFsnTpygU6dOzJ07l+3btxMXF8fnn3/OxIkTufHGG2natCmBgYG8+eab\n7N27lyVLlvD888/n28cll1yCMYYlS5Zw7Ngxzpw5A8ANN9zAsmXLWLx4Mbt372bEiBH8+++/54yp\nYcOGZGdnM2XKFOLi4pg3b16BiReRkZGkpqby/fffc/z4cVJSUgrspyj11blzZxo1akS/fv3Ytm0b\nGzZsYMSIEfj7+5d6y0oTlFKK996Dvn0hKwtGjYK33oIKFTwdlWeEhYXRvn173njjDaKjo7n88ssZ\nPXo0ffr0Yf78+VSvXp1Zs2axaNEioqKieOmll5gyZUq+fdSuXZuxY8fy7LPPUqtWrdzZdoMGDWLQ\noEEMHjyYDh06EB4eTs+ePfO911ESaNasGdOmTWPKlClERUUxffp0Jk2alK9M+/bteeihh7jnnnuo\nWbMmEydOdPj5HO0/7zZjDAsXLiQ9PZ22bdsycOBARo8eDUBQKX9j0TtJlFDep1KqwmldFU9p3kki\nNi6WD9+swqeTWwDWIzKeeqpUDu0Sem4VT0nra+vWrbRq1YrNmzfTqlUrl8eld5JQSuXzW/xWbh/8\nK8mrhmGMdWeIhx/2dFTKGyxcuJDQ0FAaNmzI3r17GTFiBK1atXJLciqMtqCU8jKl0YL6+8Remndf\nyZkfh1ChAsycCX36uPWQyofMnj2bcePGsX//fqpUqULHjh2ZPHkyNdx040V9HpRSPsLdCepw0hEu\nu305iWv6ERBgTYbo3t1th1PqnPR5UC6m1/YUndaV9xCBK7v/lJucvvrKt5OTnlvF42v1pWNQSpUT\nItZTbw+s7E5goPD114Zbb/V0VEo5p118SnkZd3TxZWdbyendd62n3i5cCLfc4tJDKFViOotPqXIq\nO9u68Pb9963ktGgR3Hyzp6NS6tx0DKqEfK0v15O0rjxHxHo8xvvvW3eFiIkpW8lJz63i8bX60haU\nUmWUCNw+cDvLPmlGYKCVnDp39nRUShWdjkEp5WVcNQZ139BdfPrmZVSsaE2I6NLFBcEp5QalMs3c\nGFPFGPMcdMVYAAAgAElEQVS1MSbJGLPXGHNPIWWvMMb8YIw5bYyJN8Y85spYlCrPHn32Hz598zL8\n/IRPP9XkpHyTq8eg3gFSgRpAH+BdY0wT+0LGmGrAMuBdoArQAFjh4ljcytf6cj1J66p0jZ4Qz9sv\nW4+BmDHD0Lu3hwNyIz23isfX6stlCcoYEwL0BEaLSIqIrANigL4Oig8HlovIfBHJFJEzIrLLVbEo\nVV59/LEw/pkLAesO5f36eTggpc6Dy8agjDEtgXUiEppn2wjgOhHpZld2JbAdaI3VetoAPCoiBR6M\nomNQqrwp6RjUvHlw333W5IjJk2HYMDcEp5QblMZ1UGFAot22RCDcQdmLgVbAjcAOYCIwD+jgaMcD\nBgwgMjISgMqVK9OyZcvcW8bnNFl1XdfL0nqOopZPSIimb18QiWXQIBg2zLs+j67ret71nJ/j4uIo\njKtbUGtFJCzPtuHA9Q5aUL8Bm0VksG29KnAMqCQip+3KemULKlafQ1NkWlfFU9wW1OrV1l0h0tPh\nmWfg5ZfdGJyX0XOreLy1vkpjFt9uoKIxpn6ebS2A3x2U3QbY/wYKULrPE1bKx23ZAt26WcnpkUdg\n/HhPR6SU67j0OihjzKdYieZ+rC68b4CrRWSnXbmOwBdAR2An8BpwhYhc72CfXtmCUspditqC2rMH\nWrZJIjkhjLvvhrlzwU/vDaN8UGk9buMRIAQ4AswFHhKRncaYDsaYUzmFRGQ1MApYChwCLgXudXEs\nSpVZBw9Cu+sTSU4II7pTOp98oslJlT0uPaVF5KSI9BCRMBGJFJEFtu1rRSTCruz7InKxiFQTkW4i\ncsCVsbib/WC2ck7ryrUSEqBddAIn4ivR4so0Fi8MICDA01F5hp5bxeNr9aXfuZTyISkpcG3nk/y7\npzKRDVL5fnkgYWHnfp9SvkjvxaeUl3E2BpWZCT16ZPPNN37UqJXGLxsDqVvXAwEq5WL6yHelfJgI\n3H8/fPONH1WqCLErNTmpsk8TVAn5Wl+uJ2ldnb+nn4aZMyEkBJYsMTRt6umIvIOeW8Xja/WlCUop\nL/f66/Daa1CxInzxBbRv7+mIlCodOgallJfJOwY1Y2Y2gwZa3yPnzLHutadUWVMa9+JTSrnQopgs\nBg+2fp46VZOTKn+0i6+EfK0v15O0rorvxx+FO3tnIdkVePLpTB5/3NMReSc9t4rH1+pLE5RSXieS\nm25LJSs9gH4D05nwsnZ0qPJJx6CU8iJxcVCv0UHIqM2td6QS81UQFTU/qTJOr4NSyssdOQKdb8qG\njNq0uSaFrz7T5KTKN01QJeRrfbmepHV1bqdOwa23wp97/IAPWbEkmKAgT0fl/fTcKh5fqy9NUEp5\nWFoa9OhhPdupfn2AJ6lUydNRKeV5OgallAdlZcHdd1sX4NaqBevWQf36xXuirlK+Tq+DUsrLiMCD\nD2fwxRf+VKoE334Ll15asNyJEyeYPXs2DRo0oGnTplxyySX46cOfVDmgLagSio2NJTo62tNh+ASt\nK8eeGZ3OhPEBBARm8f13Fbj2Wmu7/d3M9+/fT2RkJEFBQYgImZmZXHLJJTRr1ow2bdoQFRVFVFRU\nuUxcem4Vj7fWl7aglPIiU6ZmMWF8AMYvi88W+OUmJ0cuvvhi+vXrx5w5c8jIyABgz5497Nmzh5iY\nGEJCQsjKyiIjI4O6devmJq5rr72Wa665ppQ+kVKupy0opUrZ7DnZ9OtrtXQ+/CiLIYMr5Hvd0fOg\nDh48SP369UlNTS3SMfz8/GjZsiWbN292TdBKuZFeB6WUF1i6FAYMsJLP+AnpBZKTM7Vr12bIkCEE\nBgYWqXxQUBBz584tcZxKeQNNUCXka9cTeJLWlWXdOujVS8jOqsDQESmMeiqgWO8fM2YMFSqcO6GF\nhoYybdo0GjduXNJQfYaeW8Xja/WlCUqpUrBtG3TpAikphsGDYerE4GLvo3r16jz22GMEneMK3szM\nTK666qqShqqU19AxKKXc7O+/4Zpr4NAh6NkTFiyg0FsYORqDypGYmMjFF19MUlJSoccMDg5m3Lhx\nDBs2DGMKdO0r5VV0DEopDzh0CDp3tv694QaYO7fw5HQulSpV4qmnniI4uPAWWEpKCs899xw33HAD\nR48eLfkBlfIglyYoY0wVY8zXxpgkY8xeY8w95yjvb4z5nzFmnyvjKA2+1pfrSeW1rhISoFPnDP7+\nG666ChYuxCX31xs2bBgBAfnHr4KCggq0lJKTk1m3bh2XXXYZK1euPP8De6Hyem6VlK/Vl6tbUO8A\nqUANoA/wrjGmSSHlnwQOuTgGpTwuORluvi2NP3b4U/fSZJYuhfBw1+w7NDSUMWPGEBISAkDFihVp\n0qQJl156aYGWVUZGBidPnuSOO+5g5MiRuddRKeULXDYGZYwJAU4CTUXkL9u2WcB+ERnloHw94Btg\nOPChiNR1sl8dg1I+JSMD7uiewbdL/alU8xTbNkVQ1+HZ7VhhY1A50tLSuOiiizh+/DgRERHs3LmT\nqlWrMnToUObOnUtycnKB94SEhHDppZcSExNDvXr1ivuxlHKb0hiDagRk5iQnm61AlJPybwDPYLW4\nlCoTsrLgvj6ZfLvUn6CIJH5aHV6s5FRUgYGBjB8/HoDZs2dTu3ZtgoKC+OCDD5g3bx4RERFUtBvs\nSk5O5o8//qBZs2Z6jZTyCa5MUGFAot22RKBAx4YxpgdQQURiXHj8UuVrfbmeVF7qKjsbhgzJ5vPP\nKlIxOJnY70Jo2tR9M+gGDx7M6tWr6dq1a77tXbt25Y8//qBVq1a53YBnY8zmzJkzPPDAA9xzzz3n\nnA3o7crLueUqvlZfrrwXXxIQYbctAjidd4OtK/BV4NacTefa8YABA4iMjASgcuXKtGzZMveGhzkV\nXtrrOTx1fF9a/+2337wqHnesX399NEOHwsyZa/DzT2fFso60bePn1vMrp4WU9wageV9fv349Q4YM\nYd68eaSlpeXbb3JyMgsXLuT7779n/PjxPPDAA6VaX65a/+2337wqHm9f95b6yvk5Li6Owrh6DOoE\nEJVnDOoT4EDeMShjTAvgZ+A4VnIKACoBR4B2IrLPbr86BqW8mgg89RRMnAiBgfDNN3DjjSXfX1HG\noIpj/fr1dO/encTExAKJCqxrpsaOHcvIkSP1minlEc7GoFx6oa4x5lNAgPuBVliTIK4WkZ15yvgB\n1fO87RrgTVv5Y/bZSBOU8nZjx8ILL1jXN339tXXHiPPh6gQFkJCQQN++fVm1apXTCRStW7fms88+\no2bNmi49tlLnUloX6j4ChGC1huYCD4nITmNMB2PMKQARyRaRIzkLVqsrW0SO+lImsu+KUc6V5bqa\nONFKTn5+8Omn55+c3KVy5crExMQwbdo0QkJCHF4z9dNPP3HZZZfx3XffeSjK4ivL55Y7+Fp9uTRB\nichJEekhImEiEikiC2zb14qI/fhUznt+cDbFXClv9vbb8OST1s8zZkDv3p6N51yMMQwZMoTNmzdT\nv359h9dMJSQk0K1bN5544gm9Zkp5nN6LT6kSmDEDBg2yfh792n5e+r+LXbZvd3Tx2UtNTeWJJ55g\n9uzZDrv8goODufTSS1m0aBH169d3ayxK6b34yrCOHTsydOhQT4dRbsycCYMHWwmk/5NbXZqcSktQ\nUBDvvfce8+fPd3jNVEpKCjt37qRFixbMnj3bQ1Gqck9EvHqxQnS9o0ePysMPPyyRkZESGBgoF1xw\ngdx4443y/fffF+n9U6ZMEWOMHD9+3C3xOTJz5kwJCwsrsP3kyZOSlJRUanEU1+rVqz0dgst8/LGI\nMdkCIt3+u8Etx3DXOe/MgQMHpG3bthIaGipYk5zyLSEhIdK7d285depUqcZVFGXp3CoN3lpftnO+\nwN//ctuC6tmzJ7/88gszZsxgz549LFmyhFtvvZXjx48XeR+u6oopal+/iDicBly5cmVCQ0PPOw5V\nuI8/hiFDBBFDpwe+Y+HbbT0dkkvUrl2bdevWOb1LenJyMosXL6Zx48b6CHlVuhxlLW9acMO3yYSE\nBDHGyMqVK52WmTNnjrRu3VrCw8OlZs2a0rt3bzlw4ICIiMTFxYkxRvz8/HL/HThwoIiIREdHy2OP\nPZZvXwMGDJA77rgjdz06OloefvhhGTlypNSoUUPatGkjIiKTJ0+W5s2bS2hoqFx00UUyZMgQSUxM\nFBGR2NjYAsccO3asw2NGRkbKuHHj5MEHH5SIiAi5+OKLZeLEifli2r17t1x33XUSFBQkjRs3lqVL\nl0pYWJh88sknJa3WMu2DD0SsK55Eujzyg2RnZ7vtWO4454tq/fr1csEFF0hgYKDD1lRwcLBMmDBB\nsrKyPBajKnvQFtRZYWFhhIWFERMT4/DCRbBaNS+++CLbtm1jyZIlHD9+nHvvvReAOnXq8OWXXwKw\nc+dO4uPjmTZtWrFiyLkX2tq1a5k1axYAFSpUYNq0afzxxx/MmzePTZs28dhjjwFw9dVXM3XqVEJC\nQjh8+DDx8fGMHDnS6f6nTp1K8+bN+fXXX3nqqad48skn2bhxI2B9KenevTsBAQH8/PPPzJw5k7Fj\nx5Kenl6sz1BefPAB2G60wMSJsPit68rsBa3t2rVj165d3HzzzQVukwTW2NSLL75IdHQ0hw8f9kCE\nqlxxlLW8acFN3ya/+uorqVatmgQFBUn79u1l5MiRsnHjRqfld+7cKcaY3FbUlClTxM/Pr8AYVFFb\nUC1atDhnjMuXL5egoKDc9ZkzZ0p4eHiBco5aUPfee2++Mg0bNpTx48fn7tff31/i4+NzX//pp5/E\nGOOWFpS39nsXxTvvnG05TZpUOsd01zlfHNnZ2fLxxx9LSEiIGGMKtKT8/f2lcuXKsnz5co/G6cvn\nlid4a32hLaj8evTowcGDB/nmm2+47bbbWL9+Pe3atWPChAkAbNmyhe7duxMZGUlERAStW7fGGMO+\nfa55tuKVV15ZYNuqVau46aabqFOnDhEREfTs2ZP09HQOHSr+I7OaN2+eb7127docOXIEgF27dlG7\ndm1q1aqV+3rr1q3x8yu3p4NDr74K//2v9fPkyTB8uGfjKU3GGAYNGsSWLVto0KCB02umevToweOP\nP66tb+UW5fovUkBAAJ06dWL06NGsXbuWwYMH88ILL3Dq1CluueUWwsLCmDNnDr/88gvLly9HRHJ/\nEVu1auVwn35+fgUmTjiaBGE/qWHfvn106dKFqKgovvjiC7Zs2cL06dMBSvTL7+/vn2/dGEN2djbg\nfLKFu+TcKNJXiMCzz8LTT4MxwltvZzFsmKej8ozLLruM7du3M2DAAIcTKFJSUvjoo49o2bIlf/75\nZ6nH52vnlqf5Wn2V6wRlr0mTJmRmZvLbb79x7Ngxxo8fT4cOHWjUqBGHDx/O90c955HbWVlZ+fZR\no0YN4uPj823bunXrOY/9yy+/kJGRweTJk2nbti0NGjTgwIED+coEBAQUOF5JNGnShAMHDuRrmW3a\ntCk3gZVn2dkwdCi8/DL4Vcgm/O5HuP3efz0dlkcFBgbyzjvv8Nlnn1GpUiWHz5natWsXLVq0YMeO\nHR6KUpVF5TJBnThxgk6dOjF37ly2b99OXFwcn3/+ORMnTuTGG2+kadOmBAYG8uabb7J3716WLFnC\n888/n28fBw4cwBjDkiVLOHbsGGfOnAHghhtuYNmyZSxevJjdu3czYsQI/v333H/gGjZsSHZ2NlOm\nTCEuLo558+YVmHgRGRlJamoq33//PcePHyclJaVEn79z5840atSIfv36sW3bNjZs2MCIESPw9/d3\nS8vKV+7/lZkJAwfCW2+Bf0A2oX36893EAURWjvR0aF6hS5cu7Ny5k6uuusrhc6bCw8OpU6dOqcbk\nK+eWt/C1+iqXCSosLIz27dvzxhtvEB0dzeWXX87o0aPp06cP8+fPp3r16syaNYtFixYRFRXFSy+9\nxJQpU/Lto3r16owdO5Znn32WWrVq5c62GzRoEIMGDWLw4MF06NCB8PBwevbsme+9jpJAs2bNmDZt\nGlOmTCEqKorp06czadKkfGXat2/PQw89xD333EPNmjWZOHGiw8/naP95txljWLhwIenp6bRt25aB\nAwcyevRowLrDQHmUlgb/+Q/MmgXBIVmEDriLz0bfR5uL2ng6NK9y4YUXsm7dOp555pl8XX7BwcEs\nWrSISpUqeTA6VdbovfgUYHVDtmrVis2bNzsdXyurEhKgRw+IjYVKlbIJ7N+TSYN70ad5H4/EUxr3\n4nOFjRs30r17d06cOMHo0aN57rnnPB2S8lGl8jwod9AE5R4LFy4kNDSUhg0bsnfvXkaMGIExptzd\nKeDff+G222DHDrjwQvhi0RkOhi6jV9NeHovJVxIUQGJiIvPnz+f+++/XWaCqxPRmsS7ma3259k6f\nPs2jjz5KVFQUffv2JSoqiuXLl7vlWN5aV9u3Q/v2VnJq0gTWr4erW4d6NDn5mkqVKvHggw96LDl5\n67nlrXytviqeu4gqi/r27Uvfvn09HYbHrF4N3bvDqVNw7bWwcCFUrerpqJRSeWkXnyp35s2D/v0h\nIwN69YLZs8Gb5ob4UhefUq6gXXyq3MvOhjFj4N57reT0+OPQbfSnUDHV06EppRzQBFVCvtaX60ne\nUFdnzsBdd8GLL4KfH0yZApF3T2X82pc4k37G0+EpoF69ekyePLlY7/GGc8uX+Fp96RiUKvP++Qe6\ndYOtW6FSJZg/H05eNI8nv5/EukHrqBZSzdMhlhsDBw7k+PHjxMTEFHjtl19+0eeaqXzK7BjUqlWr\n2LBhA/fffz81atRwQ2TKF6xbBz17wpEj0LAhLF4M/1RcQd+v+7Ky30our3m5p0MsoCyPQRWWoLxF\nRkZGgXtZKvcqV2NQIsJDDz3E2LFjqVOnDr1792bTpk2eDkuVIhH46CPo2NFKTp07w8aNkF55O/d9\ndR9f9P7CK5NTeWbfxefn58eHH37IXXfdRVhYGPXr1899jlqOgwcPcvfdd1O1alWqVq1Kly5d8t20\n9u+//6Z79+5ceOGFhIWFceWVV7JkyZICxx07diyDBw+mSpUq9OnjmQu0VUFlMkFt3LiRgwcPkp6e\nTlpaGl9++SVt2rQpcPPV8+FrfbmeVNp1lZxs3VPv/vvPToZYuhSqVIEGVRuw6O5FXHvJtaUakyqZ\nl156iR49erBt2zb+85//MGjQoNx7W6akpNCuXTtCQ0P58ccf2bBhA7Vr1+bGG28kNdWa+JKUlMRt\nt93GypUr2bZtG7169eLOO+9k9+7d+Y4zZcoUmjRpwubNm3n55ZdL/XOWFp/7u+XoIVElXYAqwNdA\nErAXuMdJuZHAduAU8BcwspB9FvvhV927dy/wkLUmTZoUez+F8dYHf3mj0qyr//1P5PLLrQcMBgeL\n+OIT7EtyzvsK+4d35hUZGSmT8jwV0hgjzz77bO56ZmamhISEyNy5c0VE5OOPP5Y6derk20dmZqZU\nq1ZNPv/8c6cxtGvXLvfhnTnH7dq1a4k+j6/x1r9bOHlgoasnSbwDpAI1gCuAJcaY30Rkp4OyfYFt\nQANghTFmn4h8dr4BHD58OPfZTTnCwsJ4+umnz3fX+fjac1U8qbTqasECGDIEkpLgssvgiy/gcu3F\n82nNmjXL/blChQrUqFEj98GbW7ZsIT4+nvDw8HzvSUlJ4a+//gKsR4G88MILLFmyhPj4eDIyMkhL\nS6NFixb53nPVVVe5+ZN4B1/7u+WyBGWMCQF6Ak1FJAVYZ4yJwUpEo/KWFZHX86zuNsYsAq4BzjtB\nvfvuuwUGmI0x3HXXXee7a+Wl0tJgxAh4+21r/e674YMPwO7vlvJBhT14Mzs7m1atWrFgwYICv/NV\nbbcFGTFiBCtWrGDSpEk0aNCAkJAQ+vbtW+AhoDp70Du5cgyqEZApIn/l2bYViCrCe68Ffj/fADIy\nMnjjjTdIS0vL3RYQEMCDDz7o8sdI+Fxfrge5s662bYPWra3kFBBg/fvpp1ZyysrO4pUfXyEpPclt\nx1eec8UVV/C///2PatWqcemll+ZbKleuDMC6devo168f3bt35/LLL6d27dq5ravyyNf+brkyQYUB\niXbbEoFCv8caY8YCBphxvgEsWrSIzMxM+/0zdOjQ89218jLZ2fD661Zy2r4dGjSwppT/979gjDW2\n+ujSR/l+7/f4++mUYW9y6tQptm7dmm+Ji4sr9n7uu+8+qlSpQrdu3VizZg1xcXGsWbOGkSNH5iah\nRo0a8fXXX/Prr7+yfft2+vbtm+8LrPJurhyDSgIi7LZFAKedvcEY8yjQB+ggIhnOyg0YMIDIyEgA\nKleuTMuWLXP7UnO+EURHR/PKK69w+nT+wzVv3py//vor90mfecvreumt53DF/g4fhvfei8Z6KZY7\n7oB586IJDT1bfo1Zw8YDG3mp3kusX7ve45/fk/XlTeuHDh3ixx9/5Iorrsj3Oe+8806MMfz555/E\nxsYSHR2NMYYdO3ZQtWrV3PenpaXlTiMPDg5m2rRpfPDBB9x1110kJiZStWpVWrZsSZUqVQDo3bs3\nr7/+Otdddx1VqlShS5cuNG3aNPe4sbGx+RKWp+vH3es52zwdT87P5/pi4rILdW1jUCeAqJxuPmPM\nJ8ABERnloPwg4AXgWhH5p5D9SlFi3LFjB23atMn3GPSwsDBiYmLo2LFjcT+O8kIiMHcuPPooJCZC\nzZowfTrcfnv+ch9s/oBX173KT4N+4oKwCzwT7HkoyxfqKuWI2y/UFZFk4CvgRWNMiDHmGqArMNtB\nMPcB44HOhSWn4nj99dcLDHzm/eblavbfdJVzrqiruDgrEfXtayWnbt2s5zjZJ6cf//mRF2Jf4Ns+\n3/pkclLFo7+HxeNr9eXqC3UfAUKAI8Bc4CER2WmM6WCMOZWn3EtAVWCTMea0MeaUMeadkh40ISGB\nBQsWkJWVlbstJCSEJ598EmMKJGXlQzIzrRu7RkXBsmVQuTJ8/DF8/TU4uoNVu4vbsWbgGhpUbVD6\nwSqlXKpM3Itv8uTJPPfccyQnJ+duCw4O5vDhwwWukVC+49dfrbtB5DyF/q67YNo0qFXLs3G5m3bx\nqfKmzN6LLzs7m9dffz1fcqpYsSJ9+vTR5OSjEhJg+HBrht7mzVCnjnWT1wULyn5yUkqd5fMJ6rvv\nviswc69ixYoMHz7crcf1tb5cTypqXWVlwfvvW3cdnzLFmkr++OPw++/QpYt7Y1S+SX8Pi8fX6svn\nE9SECRNISsp/IWbLli1p3LixhyJSJbFqFbRqBQ89BMeOQYcOsGkTTJ3q/I4QSelJPLLkEX3goFJl\nlE+PQf39999ERUXl3rkYIDw8nDlz5tC1a9fSClGdh1274OmnYeFCa/2SS2DiROjVy7rg1pn0rHTu\nmHcHdSLq8OEdH5apyTA6BqXKG2djUD79RN2pU6fmm7kHEBgYyO32c4+V1/n7b+vx67NnW115oaHw\nzDPW2FNwcOHvzZZsBi0aRGCFQN7r8l6ZSk5KqbN8tosvOTmZ6dOnk5Fx9gYUwcHBDBs2jAoVKrj9\n+L7Wl+tJeetq/36rG++yy+CTT8DPz5qpt3s3PPvsuZOTiDByxUjiEuKY32s+Ff18+juWOk/6e1g8\nvlZfPvvb/emnnxbYJiI8+OCDHohGncu+fTBpkjUJIi3NSkz9+sHzz0P9+kXfz6Jdi1jx1wrWDFxD\niH+I+wJWSnmcT45BiQgNGzbMd1diPz8/evXqxYIFC0o7RFWIbdusMaX5862LbsG6numFF6BJk+Lv\nLys7i5OpJ6keUt2lcXoTHYNS5U2ZGoNav349hw4dyrctKCiIJ5980kMRqbxEIDYWXnsNli+3tvn5\nWc9pevppsHtWXLFU8KtQppOTUuosnxiDOnMm/zTiV199Nd+FuQD16tXjyiuvLLWYfK0vtzScOQMf\nfghXXgk33GAlp+Bg6NEjlj//hHnzzi85KWVPfw+Lx9fqyycSVOXKlenduzebNm0iPj6eFStWuP2R\n7qrofv/dusN47drwwAPWLYqqV4exY62xp6FDoV49T0eplPI1PjEGFRYWxpkzZwgODiYwMJDk5OR8\nz3CJiIjgyJEjBAYGejDS8uX0afjqK+vGrT/+eHb71VfDww9b1zGd70OM95/az2PLHmNBrwUEVAg4\nv535EB2DUuWNT49B+fn5ISIkJycX6NoLCAjg4Ycf1uRUCjIzYcUKmDPHurA259FbYWHWYzAeegia\nN3fNsU6mnOSWObfQr0W/cpWclFJn+UQXn/1j3PPy8/Nj3bp1rFq1qlS/dfpaX25JZWVZLaTHH4eL\nLrKevzRvnpWcrrsOPvgADh6Ed95xnpyKW1cpGSl0nd+Vm+rfxP9d/X/n/yFUmVVefg9dxdfqyyda\nUPYPIswrNTWVtWvX0q1bN6pUqcK0adPo0aNHKUZX9qSnw+rVVhfewoVw5MjZ1y67zGot3XcfREa6\n/tiZ2Znc/eXd1K1Ul9dvel3vEqFUOeYTCSo7O/ucZZKSkkhLS2P//v2lEBFue1Kvpxw+DN9+a828\nW7rUemptjksvhZ49reuXrrqq8HvkOVKcupq7bS6pmal83vtz/IxPNPCVB5W130N387X68plJEvZ3\nLLcXEhLCjBkzuOuuu0opMt+Wng4bN1oJadkya+ZdXpdfbiWlnj2trrvSasiICGlZaQRVPM8ZFj5M\nJ0mo8sanJ0kUlqD8/PwIDw/n22+/pW3btqUWU2xsrE99G0lLsxLSDz9Yy08/nZ3kANaMu44d4eab\n4bbbrGcyuUpx6soYU66TkyoeX/s99DRfqy+fSFAREREF7hwB1gy+Cy+8kNjYWCLdMSDio0Ssm7Ju\n2gQ//wwbNlhLnpn5gHWroZtvhltvhWuvPfeNWpVSqjT5RBdf27Zt2bhxY77twcHBtGzZkqVLl1K5\ncmUPRecdTpyALVusFtLPP1uLg3xOs2Zw/fXWct11ULNm6cdqT0R0IoQd7eJT5Y1Pd/FVrVo133pI\nSA094oIAAAoJSURBVAg9e/Zk+vTp+Pv7eyiq0peaCjt3wvbt+ZeDBwuWrVIFWreGNm2sf6++2rq7\ngzfZEr+F4d8OZ2W/lVTwc/8jUpRSvsUnElSNGjVyfw4ODmbUqFGMGjXKo9+83dWXK2IlnN27Yc+e\ns8uuXda/ds9nBCAkxJrI0LatlZDatLEeYeEtDRNHdfXXib/o8mkX3rrtLU1OqsR8bUzF03ytvnwi\nQdW09UWVhZl6InD0qHWPun374J9/zv7855/WYnezjFx+ftC4sdVVl3epV896zVccTjrMzXNuZsz1\nY+jZpKenw1FKeSmfGIN6/fXXefHFF1mxYkWpztQrDhFISrLGfg4ftv7NWeLjzyahffusrrrCVK8O\njRpZM+kaNjz7c+PG539/O087lXaK6JnRdLusG2Oix3g6HK+kY1CqvHE2BuXSBGWMqQJMBzoDR4FR\nIjLPSdlXgcGAANNF5Ckn5eT06dOkpaVRrVo1l8VaGBGrFXPihLUcP+7856NHzyYiZy0fe1WqQN26\n1nLJJWd/rlfPSkRVqrj383nSu5veZdvhbbxz+zs6OcIJTVCqvCmtBJWTjAYBVwBLgPYistOu3IPA\nE8ANtk3fA9NE5AMH+yzwRF172dlWqyQlJf9y5ox11+3Tp+HUqYI/O9t28mTBKdkFxQLR+bYEBcGF\nF0KtWmeXCy6w/s1JRnXqQHj4ufZdtuTt9xYRBNG7RBRCE1TR+dqYiqd5a325fRafMSYE6Ak0FZEU\nYJ0xJgboC4yyK94PmCQi8bb3TgKGAAUSFFjX6eRNPMnJ+dfP1WVWEoGBUK2atVStenbJWT96FNq3\nhxo1ziaj8HDvmZjgrYwxGLSSlFLn5rIWlDGmJbBORELzbBsBXCci3ezKJgCdRWSTbf1KYJWIVHKw\nX7F6AQsXFGRdaBocbM1qy/k3PNxaIiKK/nPVqnrRqvIcbUGp8qY0roMKAxLttiUCjjq07Msm2rY5\ncRHgD2QCWVhPCTFACpAEpJOaarWkTp4sYfTKPfyAc9/rV9nR8TmlXJugkoAIu20RwOkilI2wbXNI\n5MB5B+dq3tqX601idsXw6rpXGVdvHB07dvR0OD5DW1BFp7+HxeOt9eXsC5krR6p3AxWNMfXzbGsB\n/O6g7O+213K0dFJO+ai1+9YyOGYwU2+eqq0BpVSJuHoW36dYA0b3A62Ab4CrncziG4o1HR1gBdYs\nvg8d7POcs/iUd9lxZAedZnVido/Z3FT/Jk+H43O0BaXKG2djUK6e6/sIEAIcAeYCD4nITmNMB2PM\nqZxCIvI+sBjYDmwDFjtKTsr3/JPwD7fOvZUpN0/R5KSUOi8uTVAiclJEeohImIhEisgC2/a1IhJh\nV/ZpEakmItVF5BlXxlEaYmNjPR2CV1q8ezHD2w3n3mb35m7TulLuoudW8fhaffnEvfiU73i0zaOe\nDkEpVUb4xL34vD1GpVxJx6BUeVNaY1BKKaWUS2iCKiFf68t1l5Mp574yWutKuYueW8Xja/WlCUqV\n2JT1U+j9eW9Ph6GUKqN0DEqVyNxtc3lm5TOsHbSWupXqejqcMkXHoFR5Uxr34lPlxIq/VjB8xXBW\n9VulyUkp5TbaxVdCvtaX6yqbDmyiz1d9+Oqur4iqGVWk95TXulLup+dW8fhafWmCUsWy6eAmPur6\nEdfUvcbToSilyjgdg1LKy+gYlCpv9DoopZRSPkUTVAn5Wl+uJ2ldKXfRc6t4fK2+NEEpp7Kys/gn\n4R9Ph6GUKqd0DEo5JCI89M1DJKQlsKDXAk+HU67oGJQqb/Q6KFUsY38Yy+b4zazuv9rToSilyint\n4ishX+vLLY53N73L3O1zWXrfUsIDw897f2W5rpRn6blVPL5WX9qCUvl88ccXvLTmJX4c+CM1Q2t6\nOhylVDmmY1Aqn4+3fMwVF15BqwtbeTqUckvHoFR542wMShOUUl5GE5Qqb/RCXRf7//buJbSuKgrj\n+P/TVEws0YiPmUKlgqVYRUfSEoUi6kQwRSpVKIJaaFU01ZEIrcNm4ECqYHwgaqATtShUqBSEOlCU\niIhYKElbiiiizYM2LSbLwU3xtqY393Fyz9653w8yyM1JzmJnZa97dvZZJ7e13DJ5rGypOLcak9t4\nuUCZmVmSvMTXwabOTnFs4hhrb1hbdihWxUt81mm8xGcXODd7joF9Awz/MFx2KGZmCyqkQEnqk/SJ\npGlJY5Ieq3HsTkk/SZqUdFTSziJiaLfc1nKrzcUcWz/dSs+KHobuH1ry8+U8VpY251Zjchuvoq6g\n9gIzwPXA48Cbkm6rcfwTwDXAg8AOSY8WFEfbjI6Olh1CUyKCwS8HOTF5gpGBEbouW/pb4XIdK0uf\nc6sxuY1XywVKUg/wCPBKRJyJiMPAfipF6H8iYigiRiNiLiKOAJ8B2T397tSpU2WH0JQ93+zh4NhB\n9m/eT/eK7racM9exsvQ5txqT23gVcQV1K/BPRByteu1HoL7ngcMG4OcC4rA6rOpbxYEtB+jr7is7\nFDOzmopY31kJTFz02gSwaBM3SbsAAe8VEEdbjY+Plx1CUzat2dT2c+Y6VpY+51ZjchuvRbeZSzoE\n9AMLHXgYeA44HBFXVX3Pi0B/RDxc4+fuAF4A1kfEbzWO835bM7NlrqnHbUTEfbW+Pv8/qMsl3VK1\nzLeOGst2kp4EXgY21CpOlwrazMyWv0Ju1JX0MZUrrKeAO4HPgXsi4pcFjt0CDAH3RsSvLZ/czMyW\npaK2mW8HeoA/gI+AbeeLk6T1kiarjn0NuBb4TtLU/P1QewuKw8zMlonkWx2ZmVlncqsjMzNLkgtU\nASStlnRG0gdlx5IqSVdIGpY0LmlC0veSHig7rpQ00jKskzmXmpfbXOUCVYw3gG/LDiJxXcBxKjs3\nrwZeBfZJuqncsJLSaMuwTuVcal5Wc5ULVIskbQb+Br4qO5aURcTpiNgdESfmP/8CGAPuKjeyNDTa\nMqyTOZeak+Nc5QLVAkm9wC5gkEpHDKuTpBuB1bjN1XmttgzrWM6lxeU6V7lAtWY38HZEnCw7kJxI\n6gI+BN6fbxhsLbQM62TOpbplOVe5QF2CpEOS5iTNLvDxtaR1wEbg9bJjTcFi41V1nKhMKGeBZ0sL\nOD3TQO9Fr/UCUyXEkgXnUn0k3UGmc9XSPwwoU3W0eHoeuBk4Pv+HspJKy6c1EXF3O2JMyWLjVeUd\n4DrgoYiYXcKQcnME6GqkZZg5l+rUT6ZzlW/UbZKkK7nwHe9LVJJgW0T8VU5UaZP0FnA7sDEiTpcd\nT2oaaRnW6ZxL9ct5rvIVVJMiYobKlmAAJE0DM6n/wssyvwX4aSpj9nvljRwBPBMRI2XGlpDtwLtU\nWob9SVXLMPuPc6kxOc9VvoIyM7MkeZOEmZklyQXKzMyS5AJlZmZJcoEyM7MkuUCZmVmSXKDMzCxJ\nLlBmZpYkFygzM0vSvwqOP117EqfvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f732b9f8c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "plt.plot(z, logit(z), \"b-\", linewidth=2)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"sigmoid_saturation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Xavier and He Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function. The main differences relevant to this chapter are:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* it does not support `tensorflow.contrib.framework.arg_scope()` (introduced later in chapter 11).\n",
    "* it does not support regularizer params (introduced later in chapter 11)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                          kernel_initializer=he_init, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Nonsaturating Activation Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Leaky ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha*z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure leaky_relu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNW5x/HvCyPLsGowikZBMKIigkjUSMRRvC64sIgB\nV8AoLkEFr3uMiEbQixo3ogIRriImCIJKCC7AYIS4oIIrEhHCVSMigggM28y5f5wGh6Fn6enprqqu\n3+d5+pmu7pqut8+c7nfqPaeqzDmHiIhI2NQKOgAREZFklKBERCSUlKBERCSUlKBERCSUlKBERCSU\nlKBERCSUlKCkXGY2x8weCjqOXGBmx5tZsZntkYVtLTOza7OwnTZmNt/Miszs80xvrwrxlJhZr6Dj\nkJqjBBVRZjbOzF4IOo5UmVlh4oukxMw2m9lnZjbczOqk+Dr9zOyHCp5Pmlwr+72aUE6CmAc0d859\nV4PbGWpmHyR5qhPwp5raTgX+AGwADgJ+kYXtARX2/b2BF7MVh2ReXtABSOw44AngZqAu/ottfOLx\n31XjtaobQ1Y557YB32TipZNsa3UGtpPMgcA059z/ZWl7FXLOZaJ9JUDag8pRZtbYzEab2UozW5fY\noziy1PN7mNlEM/s/M9toZh+aWf9KXrOrma0xs0vN7Dgz22JmPy2zzl1mtrCS8DY651Y5575wzk0F\nXgFOLvM6+5jZX8zsu8RtupkdmForVI+ZjTCzxYl2WWZm95TdwzOz083sjcQ635rZ82ZW18zmAC2A\nkYm9xOLE+gWJ5T0Sf5uNZnZ6mdc8OdGmzSqLw8z6AUOBttu3Y2YXJZ7baQ/OzPYzs6mJfrDOzKaY\n2b6lnh9qZh+YWZ/EHu26xPrlliPNrAQ4HBia2PZtZtYiEUvHsutuL72VWqeXmb1sZhvM7CMzO6nM\n77RJtOlaM/vBzOaZWVszGwr0A04v9b67lN1OYvkwM3sl0X6rE3tejUs9P87MXjSzq83si0Q/e8LM\n6pX3viW7lKBy1wx8yaMb0AF4DZhlZnslnq8HvJN4/lDgAeAxMzsh2YuZ2dnAc8Alzrkxzrl/AJ8B\nF5Vax4ALgbFVDdLM2gOdga2lHqsPzMGXj44DjgG+Al7N0pfHeqA/cDBwBdCHUnt3ZnYqMA14CegI\nFABzAQN6AV8Aw/Dt3zzxay5xwzm3DpgOnF9mu+cBLznnvq1CHH8F7gM+BfZKbOev5byf54E9E3EW\nAPsAU8us0xL4NdAd+C/gCOCucl6PxHtbAtyb2Pa9pd5nVfwB3+cOB94GnjGzfAAzaw68DhQDXROx\njAJqAyOBScCr/Pi+55d98UQfmgmsw5c8ewDHAn8us+pxQNvEdn4N9ASuqeJ7kExzzukWwRswDnih\nnOdOxH8w65Z5/D3gugpe8xlgdKnlOcBDwKXAGqBrmfX/G/io1PJpQBGwewXbmANsBn4ANgEl+OTU\no9Q6FwOflvm92sC3QO/Ecj9gXSXbeSjJ4xX+XjmvdRmwpNTy68DTFay/DLi2zGPH479w90gsn4VP\nQA0Sy/WA74FfpxDHUOD9iraPTzZbgf1KPX9AIpYTS73ORqBhqXVuKb2tcuL5ALit1HKLxN+zY5n1\nSoBeZda5pNTz+yQeOzaxfFfiPdROpe+X2c72Pptf5m9QArQq9Tr/BmqVWmc08HKqn0fdMnPTHlRu\n6gg0AL5NlEd+MD8xoC3QGsDMapnZ78xsUaJE9QP+v8f9y7xWD+AR4FTn3Kwyz/0v0NrMjkksD8CP\nSaypJL6/4P9zPgb/X/8Y59y0MvG3KhP7WqDp9vgzycx6m9k/zOw/iW3/kZ3b5QhgdpqbmYFP5j0T\ny90TP3cM/lchjqo4GPjKlRoncs4tw++RHlpqvX8759aXWv4K2Kl8W8N2TO5wzn2VuLt9ex2A151z\nxWm8/sH45L2x1GPz8Qmq9Pv+2DlXUmo50+9bUqBJErmpFvA18Ct82am0dYmf1wNDgKuBD/H/zY/A\nl4JKWwS0Ay4B3iz9hHPuW/OzqS42syX4vYLTqdz3iS9JzOxC4CMzu8g592Sp+N/Dl7TKxl/VWXDr\ngCZJHm+K31NJysyOxu9JDsWX8Nbik8fIKm63Spxz28zsWXyZbwK+vPecc25TDcdhlF92K/341iTP\npfoP7PYv+h1/MzMr7zum7PYotb2yf/PqyOb7lgxRgspN7+Lr8257IkiiM/Cic27i9gfM7CB8WaS0\nZcBVwFwzG+2cG1jm+THA5MR6XyfZy6pQ4ot6OHC3mU1KfEG/C/QFVjs/XlMdn+JLjmUdmXiuPJ2B\nL5xzw7c/YGYty6zzHn7Moux4xnZb8CXJykwACs3sEODUMvFWJY6qbOdjYF8z2985tyLxOq3wZbWP\nqhBjKlYlfjYv9dgR1Xidd4HzzSzP+dmPZVX1fQ8wswbOuQ2JxzrjE9cn1YhJAqD/FKKtsZm1L3Nr\n4Zx7FV/OeN7MTjWzlmb2SzO73cw6J353CdDVzDqb2cFm9gh+bGIXzrnlwAnAqWY2usxzrwCr8f/p\nP1HN9zER/5/rVYnlp4GVifi7JOLvYmb3mlnpEl/tJO+/beK5R/FlwofM7HAzO8jMhuD3yiraC1mC\n/0I/z8wOMLMr8MmytLuAc8zsTjM7JDG7bHCpCRzLgePMz0T8Sanf22nPwDk3H1iReP+r8ONmqcSx\nHGhhZkeY2U8sybFkib7wPvC0mXU0s074xLjAOVdYQTukLPHPxRvAjWZ2qJkdi2/rVKf1/wloCDxr\nZp3MrLWZ9TWzwxPPLwcOS/xNf2JmyZLV0/hJNk8mZvN1AR4DpjjnAj+oWKpGCSrajsP/t1n6tv3L\n9zT8OMloYDF+3OcgfI0d/Cyqt/BjIYX4Et+EMq+/44sl8aEuAE4xs8fKrDcOvzc+vgoxJztuZyt+\nnOv6xH+8RUAX4HP8jK1PEttoys57ePXY9f3PSbzmssRr/BxfInsTP0urt3PupXKDc246vg3/iC9v\ndgV+X2adv+PHjk4ttc0Cfixx3QbsByxl52Ofkn1RP40fj5vonCvd3pXGAUzB//1mJbazPYGV3U53\nfkyAs/B9oCfpS/Z+BiR+voX/JyHZsW3Jfq/0e/8K/7fbDd+H3wUGAdv3psbg+8QC/PvunOQ1ioBT\ngMb4v/1U/MHSv6n8bUlYWKnPhEi1mNmfgNbOuVOCjkVEcofGoKTaEgc9tsUfC9U74HBEJMcoQUk6\nnsefqmisc25m0MGISG5RiU9EREKpRvegzEzZTkREUuac2+X4txqfxRf0qTEquvXr1y/wGKJ6U9ul\ndwv7ZyPsN/W/mm+/ESP86SEbN3YsXhxsfOXRNHMRkZj5+9/hllv8/QkToE2bYOMpT6wSVMuWLYMO\nIbLUdhIk9b/0lG6/f/0LzjsPnIM77oAzzwwursrEKkEVFBQEHUJkqe0kSOp/6dnefj/8AD16wNq1\n/ufvUr1EaJbFKkGJiMRVSQn06wcffwyHHgpPPgm1Qp4BUgrPzH5uZkVm9mTla4uISFgMHw5Tp0KT\nJjBtGjRqFHRElUvpOCgzewl//rN/O+cuSvK8S+X1ROLCzCqcrSSSSdOnw1ln/Xi/W7dg4ykr8fmo\n/jRzM+uLP1FnSpdTEBGR4Hz6KZx/vp8Ucddd4UtOFalSgkqcc20Y/hLfNXExsUAUFhYGHUJkqe0k\nSOp/1bNunZ8MsW5dIb17w003BR1Raqp6Jok78Jfl/tKs4vzUv3//HVMamzZtSocOHXbMINneyYJa\nXrhwYaDb17KWtazlbC3Pnl3I738PixcX0LIlXHxxIXPnhiO+wsJCxo8fD1R8CEGlY1Bm1gF/naAO\nzl/9dCj+0goagxKpIo1BSbbdfjsMGwa77w5vvw2tW1f6K4EpbwyqKntQxwMtgBXmd58a4q9keqhz\nrlMNxykiIml6/nmfnGrVgmeeCXdyqkhVxqAeB1oDHYD2+MsmTwdOzmBcGbF9F1NSp7aTIKn/Vd3H\nH8MFF/j7I0bAKadEt/0q3YNyzm0CNm1fNrP1wCbn3HeZDExERFKz/QwR69dDnz5w/fVBR5SeGr0e\nlMagRJLTGJRkWnGxP9Zpxgxo3x7mzYMGDYKOqmrSPg5KRETCa+hQn5z22MOfMSIqyakisUpQUa3D\nhoHaToKk/lexKVP8Qbi1asGkSXDAATs/H9X2i1WCEhHJNR9+6E8CCzByJHTtGmw8NUljUCJZoDEo\nyYQ1a+AXv4ClS/3pjJ56Cio5l0IoaQxKRCSHFBfDuef65HTEETB6dDSTU0VilaCiWocNA7WdBEn9\nb1e/+x289BI0a+YnReTnl79uVNsvVglKRCQXTJoE99wDtWvDs89CixZBR5QZGoMSyQKNQUlNWbQI\njj0WNm6EBx+Eq68OOqL0lTcGpQQlkgVKUFITVq/2kyKWLfMz98aNy41xJ02SILp12DBQ20mQ1P9g\n2zbo29cnp06d4LHHqp6cotp+sUpQIiJRdfPN8Oqr8NOfwnPPQb16QUeUeSrxiWSBSnySjokT/XFO\neXkwezYcd1zQEdUslfhERCLovffgN7/x9x98MPeSU0VilaCiWocNA7WdBCmu/W/VKn/5jE2bfJK6\n4orqvU5U2y9WCUpEJCq2bfPXdFqxAo4+GkaNyo0Ze6nQGJRIFmgMSlI1ZAg88ADsvTcsWAD77ht0\nRJmjMSgRkYh46imfnHbbzV9KI5eTU0VilaCiWocNA7WdBClO/W/BArj0Un//kUf8WSPSFdX2i1WC\nEhEJs5UroWdP2LwZBg70tzjTGJRIFmgMSiqzdSucdBK89prfa5o9G+rWDTqq7NAYlIhIiF17rU9O\n++wDkyfHJzlVJFYJKqp12DBQ20mQcr3/jRvnx5vq1PGnMWrevGZfP6rtF6sEJSISNm++CZdf7u8/\n+qg/5kk8jUGJZIHGoCSZr7+GI4+Er76CK6/0B+PGka4HJRIgJSgpa8sWOPFEmDfPn1/v1Vd9iS+O\nNEmC6NZhw0BtJ0HKxf53zTU+Of3sZ/6y7ZlMTlFtv1glKBGRMBgzxl9wsG5dPylir72CjiicVOIT\nyQKV+GS7+fOhoMAf9/S//wsXXRR0RMFTiU9EJGBffQVnn+2T09VXKzlVJlYJKqp12DBQ20mQcqH/\nbd7sk9PXX/s9qHvvzd62o9p+sUpQIiJBcA4GDYI33oD994dJk/yZyqViGoMSyQKNQcXbY4/5q+HW\nq+dn7nXsGHRE4aIxKBGRAPzjH3DVVf7+2LFKTqmIVYKKah02DNR2EqSo9r8vvoDevf3l26+9Fs4/\nP5g4otp+sUpQIiLZsmkT9OoF33wDXbvCPfcEHVH0aAxKJAs0BhUvzsHFF8P48dCypb9K7k9+EnRU\n4aUxKBGRLBk1yien+vVh2jQlp+qKVYKKah02DNR2EqQo9b+5c2HwYH//iSegfftg44FotV9psUpQ\nIiKZtGIFnHMOFBfDDTdA375BRxRtVRqDMrOngK5AA+A/wEjn3J+TrKcxKJEkNAaV+4qK4Fe/gnff\nhZNPhhkzoHbtoKOKhrSuB2VmhwCfOee2mtlBwFygm3PuvTLrKUGJJKEElduc8+fVmzABWrWCt9+G\nPfYIOqroSGuShHPuE+fc1u2vBTigdQ3GlxVRrcOGgdpOghT2/vfggz45NWjgJ0WELTmFvf3KU+Ux\nKDMbZWYbgE+Ar4AZGYtKRCQiZs+G667z98ePh3btAg0np6R0HJSZGfBLoAC4xzlXXOZ5lfhEklCJ\nLzctXw6dOsHq1XDLLXDXXUFHFE3llfjyUnmRRPaZb2YXAlcAj5Rdp3///rRs2RKApk2b0qFDBwoK\nCoAfdzO1rGUtaznqyzNnFjJoEKxeXcBpp8GJJxZSWBie+MK8XFhYyPjx4wF25ItkqnUmCTMbA6x3\nzg0p83io96AKCwt3NJakRm2XHu1BpSds/c85f169Z56Bn/8c3noLmjYNOqryha39yqr2JAkz29PM\n+phZAzOrZWanAH2BWZkIVEQk7O67zyenhg39pIgwJ6coq3QPysyaAZOBw/EJ7d/Ag865J5KsG+o9\nKJGgaA8qd7zyCpx6KpSUwHPPQc+eQUcUfWkdB5XCRpSgRJJQgsoNn3/uJ0WsWQO33QbDhgUdUW7Q\nyWKJ7rEAYaC2kyCFof9t2AA9evjkdOaZMHRo0BFVXRjarzpilaBERKrDORgwAD74ANq0gaeeglr6\n9sw4lfhEskAlvmi7+264+WZo1MjP2Dv44KAjyi0agxIJkBJUdM2cCd26+b2oF17w5T2pWRqDIrp1\n2DBQ20mQgup///oXnHuuT07DhkU3OUX18xurBCUiUlU//OAnRaxd63/eemvQEcWPSnwiWaASX7SU\nlEDv3jB1Khx6KLzxhh9/ksxQiU9EpIqGD/fJqUkTf6YIJadgxCpBRbUOGwZqOwlSNvvf9On+IFwz\nmDjRn2sv6qL6+U3pbOYiIrns00/9SWCd85fO6NYt6IjiTWNQIlmgMajwW7cOjj4aFi/240+TJvm9\nKMk8jUGJiJSjpAQuvNAnp8MOg3HjlJzCIFYJKqp12DBQ20mQMt3/7rjDH4TbtKmfFNGwYUY3l3VR\n/fzGKkGJiJT1/PP+INxateAvf4HWrYOOSLbTGJRIFmgMKpw++QSOOgrWr4d77oEbbgg6onjSufhE\nAqQEFT5r1/rk9K9/QZ8+/gq5GncKhiZJEN06bBio7SRINd3/Skrgggt8cjr8cPjzn3M7OUX18xur\nBCUiAv5ig3/7G+yxh58U0aBB0BFJMirxiWSBSnzhMWWKP86pVi146SU46aSgIxKV+EQk9j78EPr1\n8/dHjlRyCrtYJaio1mHDQG0nQaqJ/rdmjb9sxoYN/nRGQ4akH1dURPXzG6sEJSLxVFzsLzy4dCkc\ncQSMHp3bkyJyhcagRLJAY1DBuvlmuPtuaNYMFiyAFi2CjkhK03FQIgFSggrOpEn+OKfateHVV6Gg\nIOiIpCxNkiC6ddgwUNtJkKrb/xYtggED/P37749vcorq5zdWCUpE4mP1aujZEzZuhIsugquuCjoi\nSZVKfCJZoBJfdm3bBqed5kt6nTrBa69B/fpBRyXlUYlPRGLj5pt9cvrpT+G555ScoipWCSqqddgw\nUNtJkFLpfxMnwr33Ql4ePPss7Ldf5uKKiqh+fmOVoEQkt733Hlxyib//4IPQpUuw8Uh6NAYlkgUa\ng8q8Vav8eNOKFXDxxTB2rA7GjQodByUSICWozNq2DU4+GebMgaOPhsJCqFcv6KikqjRJgujWYcNA\nbSdBqqz/XX+9T0577+3PVq7ktLOofn5jlaBEJPc89RQ88ADsthtMngz77ht0RFJTVOITyQKV+DJj\nwQL41a9g82Z47DG47LKgI5LqUIlPRHLKN9/4M0Vs3gwDByo55aJYJaio1mHDQG0nQSrb/7ZuhXPO\ngS++gGOPhYceCiauqIjq5zdWCUpEcsO11/rTF+2zjx93qls36IgkEzQGJZIFGoOqOePG+eOc6tSB\nuXPhmGOCjkjSpTEoEYm8N9+Eyy/39//0JyWnXFdpgjKzOmY21syWm9n3ZvaOmZ2ajeBqWlTrsGGg\ntpMgFRYW8vXX0KsXbNkCV14Jv/lN0FFFR1Q/v1XZg8oDVgDHOeeaALcBk8xs/4xGJiKSsHUr9O4N\nX33lp5X/8Y9BRyTZUK0xKDNbBNzunJta5nGNQYkkoTGo9FxxhT/Oad994Z13YK+9go5IalKNjUGZ\n2V7Az4GPaiIwEZGKjBnjk1PdujB1qpJTnKSUoMwsD5gAjHfOLclMSJkT1TpsGKjtJAjz58NvfwtQ\nyOOPwy9+EXRE0RTVz29eVVc0M8Mnp83AVeWt179/f1q2bAlA06ZN6dChAwUFBcCPjRTU8sKFCwPd\nvpa1rOWqL0+eXMjAgbB1awG9ekGLFoUUFoYnPi1Xf7mwsJDx48cD7MgXyVR5DMrMngD2B7o557aU\ns47GoESS0BhUajZvhoICeOMNOP54eOUVfzJYyU3ljUFVaQ/KzB4DDgZOKi85iYjUBOdg0CCfnPbf\n31+2XckpnqpyHNT+wECgA7DSzH4ws3Vmdm7Go6th23cxJXVqO8mWxx/3V8OtV89PithzT/W/dEW1\n/Srdg3LOrUBnnBCRLHj9dbgqMcI9Zgx07BhsPBIsnYtPJAs0BlW5L76ATp1g5Up/Mtj77gs6IsmW\n8saglKBEskAJqmKbNkGXLvD229C1K8ycCXlVnmMsUaeTxRLdOmwYqO0kU5zzZ4p4+21o2RL+8pdd\nk5P6X3qi2n6xSlAiEj6jRsH48VC/PkybBs2aBR2RhIVKfCJZoBJfcnPn+pJecTE88wz07Rt0RBIE\nlfhEJFRWrPCXbS8uhuuvV3KSXcUqQUW1DhsGajupSUVF0LMnrFoFJ58MI0ZUvL76X3qi2n6xSlAi\nEjznYOBAePddaNXKl/Zq1w46KgkjjUGJZIHGoH70wAMwZAg0aAD//Ce0axd0RBI0HQclEiAlKG/2\nbF/SKy7259jr3TvoiCQMNEmC6NZhw0BtJ+lavhx+/WufnG6+ObXkpP6Xnqi2X6wSlIgEY+NGPyli\n9Wo47TS4886gI5IoUIlPJAviXOJzDs4/30+GOPBAf8aIpk2DjkrCRCU+EQnEfff55NSwoT9ThJKT\nVFWsElRU67BhoLaT6njlFbjxRn//ySehbdvqvY76X3qi2n6xSlAikj2ffw59+kBJCfz+934MSiQV\nGoMSyYK4jUFt2AC//CV88AGccQY8/zzU0r/DUg6NQYlIVjgHAwb45NSmDUyYoOQk1ROrbhPVOmwY\nqO2kqu65xx+E26iRnxTRpEn6r6n+l56otl+sEpSIZNbMmXDLLf7+hAlw8MHBxiPRpjEokSyIwxjU\nZ5/BL34Ba9fCsGFw221BRyRRoXPxiQQo1xPUDz/4SREffQQ9esCUKRp3kqrTJAmiW4cNA7WdlKek\nBPr188npkEP88U41nZzU/9IT1faLVYISkZo3fDhMneonQ0yb5idHiNQElfhEsiBXS3zTp8NZZ/14\nv1u3YOORaCqvxJcXRDAiEn2ffupPAusc3HWXkpPUvFiV+KJahw0DtZ2Utm6dnwyxbh2cfba/vlMm\nqf+lJ6rtF6sEJSLpKymBCy+ExYvhsMNg/HiwXYozIunTGJRIFuTSGNSwYXD77f6yGQsWQOvWQUck\nUafjoEQClCsJ6vnnfWmvVi2YMQNOOSXoiCQX6DgooluHDQO1nXzyCVxwgb8/YkR2k5P6X3qi2n6x\nSlAiUj1r10L37rB+vb/G0/XXBx2RxIFKfCJZEOUSX0mJP9bpb3+Dww+H+fOhQYOgo5JcohKfiFTL\n0KE+Oe2xhz9ThJKTZEusElRU67BhoLaLp+eegz/8wU+K+Otf4YADgolD/S89UW2/WCUoEam6Dz+E\niy7y90eOhJNOCjYeiR+NQYlkQdTGoNas8dd2WrrUn87oqad0MK5kjsagRKRKiovh3HN9cjriCBg9\nWslJghGrBBXVOmwYqO3i49Zb4aWXoFkzfxmN/PygI1L/S1dU2y9WCUpEKjZpEtx9N9Su7e+3aBF0\nRBJnVRqDMrPfAv2BdsBE59zF5aynMSiRJKIwBvX++/6y7Rs3wgMPwDXXBB2RxEW614P6ErgTOAWo\nX5OBiUjwVq/259jbuNHP3Lv66qAjEqliic85N8059wLwXYbjyaio1mHDQG2Xu7Zt85Mili2DTp3g\nscfCNylC/S89UW0/jUGJxNzNN8Mrr8BPf+oPzK2vGomERI1f8r1///60bNkSgKZNm9KhQwcKCgqA\nH7N4UMvbHwtLPFFaLigoCFU8Wq6Z5VdfhXvvLSAvD265pZClS2G//cIT3/Zl9b/car/CwkLGjx8P\nsCNfJJPSgbpmdiewryZJiKQmjJMk3nsPOneGoiJ45BH47W+DjkjiSgfqEt06bBio7XLLt99Cz54+\nOV18MVx5ZdARVUz9Lz1Rbb8qlfjMrDawG1AbyDOzusA251xxJoMTkZq3bRv8+tfw73/DUUfBqFHh\nmxQhAlU/DmooMBQovfIw59wdZdZTiU8kiTCV+IYM8cc57bUXvPMO7Ltv0BFJ3JVX4tPJYkWyICwJ\n6qmn/HFOu+0Gc+b4MSiRoGkMiujWYcNAbRd9CxbApZf6+w8/HK3kpP6Xnqi2X6wSlEhcffONnxSx\neTMMHAiXXRZ0RCKVU4lPJAuCLPFt3eovNvjaa/5ce3PmQN26gYQikpRKfCIxde21Pjk1bw5Tpig5\nSXTEKkFFtQ4bBmq7aBo3zh+EW6eOP41R8+ZBR1Q96n/piWr7xSpBicTJW2/B5Zf7+6NGwTHHBBuP\nSKo0BiWSBdkeg/r6a39m8i+/9GeJGDUqa5sWSZmOgxIJUDYT1JYtcOKJMG8e/OpXMGuWL/GJhJUm\nSRDdOmwYqO2i45prfHLad1+YPDk3kpP6X3qi2n6xSlDZdsIJJ3C1Lk0qWTRmjL/gYN26MHWqP52R\nSFTFusQ3YMAAVq9ezQsvvJCR1z/hhBNo164dDz30UEZeX6IjGyW+f/4Tjj/eH/c0bhz075/RzYnU\nGJX4RHLYV1/B2Wf75HTVVUpOkhtilaBSqcOuW7eOgQMHstdee9G4cWNOOOEE3nnnnR3Pf/fdd5x3\n3nnst99+5Ofnc9hhh+24QmR5Zs2axe67786YMWOq+Q6CE9Uadhxs3uyT03/+4/eg7rsv6Ihqnvpf\neqLafrFKUKno1q0bX3/9NTNmzGDhwoV06dKFrl27snLlSgA2bdrEkUceyYwZM/j4448ZPHgwl19+\nOXPmzEn6elOmTKFXr16MHTuWS7efsVMkTc7BoEHwxhuw//7w7LP+TOUiuUBjUEnGoGbPnk2PHj1Y\ntWoVdUudF+aII47g/PPP57rrrkv6eueeey6NGjVi9OjRwI9jUO3ateOGG25g8uTJdO3aNXNvSEIr\nU2NQjz0GV1wB9er5mXsdO9b4JkQyrrwxqCpdUTdu3n33XTZs2ECzZs12enzz5s0sXboUgJKSEkaM\nGMGkSZP48ssv2bx5M1u3bqWgoGCn35k2bRqPP/44r732GkcffXS23oLEwOuv+/Em8LP3lJwk18Sq\nxFfVOmzzP6DYAAAMyUlEQVRJSQl7770377//PosWLdpxW7x4MXfeeScAI0eO5I9//CM33ngjs2fP\nZtGiRXTv3p0tW7bs9Frt27enefPmjB07tqbfTlZFtYadq774Anr39pdvHzIELrgg6IgyS/0vPVFt\nP+1BJdGxY0dWrlyJmXHAAQckXWfevHmceeaZnHfeeTseW7JkCbvvvvtO6x1wwAE8/PDDHH/88Qwc\nOHBH+U+kujZtgl69YOVKf8aI//mfoCMSyYxY7UGVLb+Bn61Xei9p0aJFHHjggXTu3Jnu3bszc+ZM\nli9fzj//+U9uv/125s2bB8BBBx3ErFmzmDdvHosXL2bQoEEsW7Ys6XZbtmzJnDlzmDlzJgMHDszk\nW8yYZG0n2eecH3N6+21o0QL++lfIi8G/mep/6Ylq+8UqQSXzj3/8g44dO+50u+GGG5gxYwYnnngi\nAwcO5OCDD6Zv374sWbKEffbZB4Bbb72Vo446im7dulFQUEDDhg25oEydxezHMb9WrVpRWFjISy+9\nxOXbTzEtkqJRo2D8eKhfH6ZNgzLDpCI5JVaz+AoLCyP7n0TQ1HbpqYlZfHPnQteuUFwMzzwDffvW\nUHARoP6XnrC3n84kIRJhK1bAOef45HT99fFKThJfsdqDEglKOntQRUX+shnvvgsnnwwzZkDt2jUc\noEiAtAclEkHOwcCBPjm1auVLe0pOEhexSlBRPRYgDNR2wXjwQZgwAfLz/aSIPfYIOqJgqP+lJ6rt\nl3MJau3atXTv3p1zzjknq5fYFqlps2fD9rNqjR8P7doFGo5I1uXUGNQbb7xBjx49WLt2LbVr1+b+\n++/nsssuCyweke1SHYNavhw6dYLVq+Hmm2H48MzFJhK08sagciJBlZSUMHz4cIYPH05RUdGOx/Pz\n83nrrbdo27Zt1mMSKS2VBLVxI3TuDAsXwmmnwYsvatxJclvOTpJYuXIlXbp0YcSIETslJ4CNGzfS\np0+fHctRrcOGgdouO5yDSy7xyenAA+Hpp5WcQP0vXVFtv0ifJOXll1+mT58+rF+/nm3btu30XK1a\ntahfvz633XZbQNGJpO6++/xMvYYN/aSIMqd2FImVSJb4tm7dyk033cSjjz66y14T+NJey5YtefHF\nF2nVqlXG4xGpTFVKfK+8AqeeCiUl8Nxz0LNnloITCVjOXA9q+fLlnHXWWSxdujRpcqpfvz6XXHIJ\nI0eOpE6dOgFEKJK6zz+HPn18cvr975WcRCBiY1CTJ0+mXbt2fPTRR2zcuHGn5/Ly8mjSpAlTpkzh\nwQcfTJqcolqHDQO1XeZs2AA9esCaNXDGGXD77UFHFD7qf+mJavtFYg+qqKiIK6+8kkmTJu2SmMCX\n9Nq1a8fUqVNp3rx5ABGKVI9zMGAAfPABtGnjD8qtFal/G0UyJ/RjUB9//DFnnHEG//nPf9i0adMu\nz9evX58bb7yRW2+9ldqa7iQhVd4Y1N13++OcGjWCt96Cgw8OIDiRgEXuOCjnHGPHjmXw4MEUFRXt\n8uGuU6cOjRs3Ztq0aXTu3LlGtimSKckS1MyZ0K2b34t6/nk466yAghMJWKSOg1q3bh09e/Zk8ODB\nbNy4cZcPdn5+PieccAKffvppSskpqnXYMFDb1azPPoNzz/XJadgwJafKqP+lJ6rtF7oxqAULFnDm\nmWeyZs0aNm/evMvz9evX5+6772bQoEE7XbFWJCp++MFPili71v+89dagIxIJp6yX+JYuXcqYMWMY\nMWLETgmmpKSEe++9l9tvvz3p9PF69erRrFkzpk+fTvv27WssZpFs2F7icw569/bHOR1yCLzxBjRu\nHHR0IsFKq8RnZrub2VQzW29my8zs3OoGMnjwYEaOHMnDDz+847FVq1Zx4oknMmzYsHIPvO3ZsyeL\nFy9WcpJIGz7cJ6cmTfyZIpScRMpX1TGoPwGbgD2BC4BHzeyQVDf2/vvvM2vWLEpKSrjppptYuHAh\nc+bMoU2bNsyfP3+XKeRmRoMGDXj88ceZOHEiDRo0SHWTO4lqHTYM1Hbpmz7dH4Rr5s+xd9BBQUcU\nHep/6Ylq+1U6BmVm+UAv4FDnXBEwz8xeAC4EbkllY9ddd92OqeJFRUUUFBSwZcuWcs8I0aJFC158\n8UUOPPDAVDYjEkrnn+8nRfzhD3D66UFHIxJ+lY5BmVkHYJ5zrkGpx/4b6OKc615m3XLHoN555x2O\nO+64pMmorPz8fAYMGMD999+v0xVJpDkHEyfCBRcY4Dj7bHj2Wb8XJSJeOufiawh8X+ax74FGqQQw\nZMiQSpNTXl4e+fn5PP3005xxxhmpvLxIxpSUwPr1ftZdKrfvv/enL1qzxr/OySf7K+MqOYlUTVX3\noF53zjUs9di1wPHJ9qAyEqWIiOS06u5BLQHyzKy1c25p4rH2wEflbGSXx4455hjefPPNCjfStGlT\nvvzyS/Lz86sQUvUUFhZSUFCQsdfPZUG2XUkJrFtXvT2Y7T/TPZqiYUNo2rTiW5Mm5T+3226pXfJd\ndqbPbnrC3n7lHdNapeOgzGwi4IBLgSOA6cCxzrlPyqy3yxhUYWEhp59+etKTvJZWv359+vXrx6OP\nPlppPBItxcXpJZh169JPMI0aVT/BNGkCeWke0p7KJd9F4iatc/GZ2e7AE8B/Ad8CNzrn/ppkvZ0S\nlHOOjh07snDhwioFmZ+fzzPPPMNZOu9LqGzbln6CSVfjxtVPMI0bp59g0qUEJVK+QE4W+/LLL9Or\nVy82bNhQ4e/ttttu5OfnU1RUxDHHHMPcuXNrLKbSwr6bmynbtv2YLFJNLmvX+lPzQCFQUO0YKip/\nVSXBRP1E9UpQ6YnrZ7emhL39sn5FXeccgwcP3ik51a5dmwYNGlBcXMyWLVto0aIF7du356ijjqJd\nu3Ycdthh/OxnP8tUSJG1dWt6CWb9+vS2bwb5+bDnntVLMI0aRT/BiEj2ZWwPav78+XTu3Jm8vDya\nN29Ou3btOProozn88MNp27YtrVq1is31m7ZsSS/BVLIDWqlatdLbg2nUSBfRS5f2oETKl/US39at\nW/nss89o3bp15A+23bKl+sll7VqoZH5IpWrVqjy5VJRgGjZUggmaEpRI+SJ3wcKatHmzTxQvvVRI\nmzYFKSeYKpz8okK1a6efYII+uDPsNeywU4JKj/pfesLeflkfg6pJmzaltweT5ErxKcnLSy/BNGgQ\nfIIREYmajO9BOZdagkk2VpPkuoUpycuD3XevfoLJz1eCkfRoD0qkfFnbgzrllF0TzJYt6b3mbrul\nl2Dq11eCERGJmhrfg/InnNhZnTrpJZh69WomwYS9Dhtmarv0aA8qPep/6Ql7+2VtD+rvf0+eYERE\nRFIRi1l8IkHTHpRI+crbg9LRMSIiEkqxSlCFhYVBhxBZajsJkvpfeqLafrFKUCIiEh0agxLJAo1B\niZRPY1AiIhIpsUpQUa3DhoHaToKk/peeqLZfrBJUVa/sK7tS20mQ1P/SE9X2i1WCWrt2bdAhRJba\nToKk/peeqLZfrBKUiIhER6wS1PLly4MOIbLUdhIk9b/0RLX9MnCyWBERkdRk/Iq6IiIiNSVWJT4R\nEYkOJSgREQklJSgREQklJSgREQml2CYoM/u5mRWZ2ZNBxxIVZlbHzMaa2XIz+97M3jGzU4OOK8zM\nbHczm2pm681smZmdG3RMUaH+VnOi+n0X2wQFPAK8FXQQEZMHrACOc841AW4DJpnZ/sGGFWp/AjYB\newIXAI+a2SHBhhQZ6m81J5Lfd7FMUGbWF1gDzAo6lihxzm10zt3hnPu/xPLfgGXAkcFGFk5mlg/0\nAm51zhU55+YBLwAXBhtZNKi/1Ywof9/FLkGZWWNgGPDfwC4HhknVmdlewM+Bj4KOJaQOArY555aW\nemwR0DageCJN/S11Uf++i12CAu4Axjjnvgw6kCgzszxgAjDeObck6HhCqiHwfZnHvgcaBRBLpKm/\nVVukv+9yKkGZ2RwzKzGz4iS318ysPXAS8EDQsYZRZe1Xaj3Df1lsBq4KLODwWw80LvNYY+CHAGKJ\nLPW36jGzDkT8+y4v6ABqknPuhIqeN7NrgBbAikSnbwjUNrNDnXOdshFjmFXWfqX8GWgGdHPOFWcw\npKhbAuSZWetSZb72qESVKvW36jmeiH/fxepcfGZWj53/o70e/we83Dn3XTBRRYuZPQYcDpzknNsY\ndDxhZ2YTAQdcChwBTAeOdc59EmhgEaH+Vn258H2XU3tQlXHObcJP+QXAzNYDm6LyxwpaYnrvQHwb\nrvT/lOGAy5xzzwQZW4j9FngC+Ab4Fv/loORUBepv6cmF77tY7UGJiEh05NQkCRERyR1KUCIiEkpK\nUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkr/DwSbeSK+TMmPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f731a665f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "save_fig(\"leaky_relu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Implementing Leaky ReLU in TensorFlow:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, name=None):\n",
    "    return tf.maximum(0.01 * z, z, name=name)\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=leaky_relu, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's train a neural network on MNIST using the Leaky ReLU. First let's create the graph:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=leaky_relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=leaky_relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's load the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Batch accuracy: 0.88 Validation accuracy: 0.9004\n",
      "5 Batch accuracy: 0.96 Validation accuracy: 0.9478\n",
      "10 Batch accuracy: 0.98 Validation accuracy: 0.9646\n",
      "15 Batch accuracy: 1.0 Validation accuracy: 0.9722\n",
      "20 Batch accuracy: 1.0 Validation accuracy: 0.9742\n",
      "25 Batch accuracy: 1.0 Validation accuracy: 0.9766\n",
      "30 Batch accuracy: 1.0 Validation accuracy: 0.9788\n",
      "35 Batch accuracy: 0.96 Validation accuracy: 0.9782\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        if epoch % 5 == 0:\n",
    "            acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "            acc_test = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "            print(epoch, \"Batch accuracy:\", acc_train, \"Validation accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### ELU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z<0, alpha*(np.exp(z)-1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure elu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXPP9x/HXJxu5SSJBpG4RJSrasKW0ImG10SSoFCVU\nkR+P4EdFabVuP3EtLSptUf2JUOIatI24RJGVVuqnRdStdUskbnHJRSLZ3Pbz++M7K2syuzs7OzPf\nc3bez8djHjtz9uycz3z3O/OZ8/2c8z3m7oiIiCRNh9gBiIiI5KIEJSIiiaQEJSIiiaQEJSIiiaQE\nJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJaljZjeZ2dR2tB0zs9+b2UdmttbM9i71NpuJpSyvObOt\nXmb2vpltW47ttcTMppjZ6bHjkHVMM0m0b2Z2E3As4IA1+tVT7j7YzG4GNnb3g5r4+xnAC+4+Lmv5\nscA17t6jNJE3u+0ehL77SZq208z2DwDuBfYB5gAL3X1NKbeZ2e56r7tcrzmzrSsIfe/4Um8rH2b2\nFeAJoL+7L40dj2gPqlL8BfhC1m3/zO/a8g0lyrcbd19ajg/Qcm0HGAC85+7/5+4flCM5NaVcr9nM\nugLHAxNLva0c2+5uZveY2VaNl7v7i8CbwA/KHZPkpgRVGVa6+4eZD7+G2+JSb9TMhpvZTDNbaGYf\nm9nDZrZj1jo/NrNXzazOzOaZ2aWZ5TcR9ihOMbP6zNBXv4bfmdn9ZnZCZoioQ9Zz3m5mf8wnjjy2\nM7XR83QyswmZba4ws7+b2V6Nfj/DzK41s0vN7EMzW5DZS2iujW4CfgX0y2z/zczyWjP7Tfa6DfHk\nu63Wtm9rX3Ohrxs4AFjr7n9vYb2iMrPjgZ8AB5P7828qcGQ5Y5KmKUFJKW0IXA18jfBhuBi438w6\nApjZZcC5wKXATsBhwPzM354G/B24CegLbN7odxD23u4CegHDGhaaWTfgIODWPONoaTuNXZGJcQxQ\nDbwAPGxmfRut831gNbAncArwIzMb3UwbjQMuAt7ObH/3Rq+vJc1uq4D2fbvA19xiLDkMAZ7J9Qsz\n28bMxprZ8WaWc+i5UO5+o7tfwOeHuxt7GtjDzDoXc7tSmI6xA5CyGGlmjcfUHbjW3c8u5Ubd/b7G\njzPfXpcQPgCeB34EjHP3P2RWeRP4v8zffmJmq4Dl7v5hE8+/xMweBI4CHsksPoTwQTktnzjcfVZL\n28n8TTfgJOA4d384s+wk4JuED+TzM6u+nPkABHjdzE4AvkVIprlew9LM/2Ztc9tvQpPbMrMNKaB9\nzdZ9brfiNbf6dQPbAO9lLzSzhuc+0t1Xmdm9ZrYSeBn4cqM4egNnNv7TzE9v9Hg1cKG7r20ihlze\nBTYAtiDUAyUiJajK8AQwls9/ayzHEN8XgUuAPYA+hD12A/oRPjw6AY+3cTOTgZvMrIu71xG+yd/j\n7qvyjGNWntvZjvB++Wx9d683s78T9k4a/Cvr794FNmvVK8pfc9vaiba3b76vuaVYcukKvN94gZl1\nB24HvtHo/1dL+ALyV6Ah0eLui4Bz8n0hrbCC0De6luC5pZWUoCrDcncv9NvgJ8BGOZb3IuyFNGca\nYUjpBOAdYA3wCuGD02h6mKU1pgFrgVFm9jhhuG9YjnWaiiNf2d/QG2u8bHWO3xUylF7P+u2zQdbj\n5rZVjLbN9zW3FEsuHwG9s5YdDLzh7nMbLfsE2BF4pPGXjhLamBB7a/dmpQSUoKQl/wFG5li+W+Z3\nOZnZxoQPlv929ycyy3ZlXZ97GVhJGAZ6o4mnWQVUNRdcZhjoHsKRV30IR8PNbEUceW0HeJ3wITwE\nmJt5ng6EmsvkFv62EB8S6kKN7UL+w04vE15XW9q3udd8W55xNOU5wukPjW0OvJS1zIEqd/9cG2f+\nrz9p5vmN8EXkglYO8X0FeLeA4VYpASWoytA5R1F7rbt/lLnf08x2yfr9Ynd/C/gd4Uiv3xAOCa4j\nHIE1mnAwQlMWEb4ljzWzt4GtgF+S+abt7svM7NfAZZlayExgE2A3d78+8xxzCfWqbYBl7v5xE9ua\nDDwKbEsYIso7jny34+7Lzex3wOVm9jEhUZxBGMa6rpl2KNTjwNVm9h3CF4ETga3JM0EV0r7Awqzn\nKOVrnp553t6Z4bqGZUMaVjCzHYCBwNJMzamLu7+XiW0hpRniGwo8XILnlQIoQVWGYYSaQAMjHLHV\nL/N4KPBs1t/cCxzu7nMszGxwCeEDpAvwb+B77j69qQ26u5vZ4cBvCEd+vQ78OPO8Dc4mfCieR0gc\nC4BbGv3+SuBmwt5AFzPb1t3n5djWTDN7h7CnNDrrd/nEkdd2gJ8RvtFPIgxxPgcMd/cPGjbXVHsU\nYBIwCLgx8/g64D5CksmLu59lZnm3LyHBZ2vqNS9ovKl8Y2oU24tm9jRwBOFLEO7+vJldmTnA4lPg\nLXc/28x+BIzi80dmFsTMvk9Igk5IkH9z9+syv+tMGGbcr63bkeLQTBIiEoWZDQcmADt5Aj6IzOxk\n4CB3HxE7Fgl0HpSIRJHZA7+WsHeXBKuAU2MHIetoD0pERBJJe1AiIpJIUQ6SMDPttomIyGfcfb1z\n96LtQbl7Im/jx4+PHkPabmqzwm5Jfh8k+Zam/jZrltO1qwPOOeeo3Zq6NUVDfCIiJfDyy3DAAbBi\nBRx3HFxySeyI0kcJKsvcuXNjh5A6ajMppzT0t/nzYfhwWLQIvvMd+P3vwYox+VQbpKHdshU1QZnZ\nrWb2rpktMbN/Z2aNTpXq6urYIaSO2kzKKen9beFCGDEC3n4bBg+GO++EjgmYEiHp7ZZLUQ8zN7OB\nwOvuvjozTckTwP7u/lzWel7M7YqkkZk1O/4u6bN8OXz72/Dkk7DTTvDXv8LGG8eOKvky74XSHiTh\n7q+4e8McZ0aYTmS7Ym5DRCSJ1qyBI44IyWnrrWH6dCWntip6DSpz6edPCZczeBd4sNjbKKXa2trY\nIaSO2kzKKYn9zR1OPBHuvz8kpenTYaukzI+RkcR2a0nRR0bd/RQz+yFhSv4awiUV1jNmzBj69+8P\nQK9evaiurqampgZY15B6nI7Hs2fPTlQ8aXncICnxpOVxEvvbxIlw2201dO0KF11Uy4IFMHBgcuJr\nLAnxzJ49m8WLwzVTmzt4o6RTHWWm6n/J3a/JWq4alFQ81aDah9/+FsaNg6oq+NOf4MADY0eUPmWp\nQeXQEdWgRKSduusuOO20cH/iRCWnYitagjKzPmY22sw2NLMOman0jwAeK9Y2yiF7d1hapjaTckpK\nf3v0UTj66FB/uvxyGDMmdkTNS0q7tUYxa1AO/Dfh4mMdgLeA09x9WhG3ISIS3bPPwsEHw+rVYQ/q\npz+NHVH7FOVyG6pBiagGlVZvvBFOwP3gAzjySJg8GTpoTp42aaoGpQQlEokSVPosWBCS05tvwrBh\n8MAD0KlT7KjSL9ZBEqmTxnHa2NRmUk6x+tsnn8DIkSE57bYb3HdfupJTGt+nSlAiIi1YuTLUnJ57\nDrbfHh58EHr0iB1V+6chPpFINMSXDvX1odZ0993Qty/MmgVf/GLsqNoXDfGJiLSSezhK7+67wx7T\nww8rOZWTElSWNI7TxqY2k3IqZ3+77DK45ppQa/rznyGFV6z4TBrfp0pQIiI53HgjnHtuuNDgbbfB\nvvvGjqjyqAYlEolqUMk1dWo4KKK+Hq69Fk4+OXZE7ZtqUCIieXjySRg9OiSn//kfJaeYlKCypHGc\nNja1mZRTKfvbSy+FCV/r6mDsWLjwwpJtquzS+D5VghIRAebNg+HDYfFiGDUKrrsu1J8kHtWgRCJR\nDSo5Pv4Yhg6FV16BIUPgkUega9fYUVUO1aBERHL49NMwrPfKK/CVr4QDJJSckkEJKksax2ljU5tJ\nORWzv61eHQ6IeOop6NcvnIjbu3fRnj5R0vg+VYISkYrkHg6EeOAB2GQTmD4dttwydlTSmGpQIpGo\nBhXX2WeHK+F26waPPw5f/3rsiCqXalAiIhkTJoTkVFUF99yj5JRUSlBZ0jhOG5vaTMqprf3tjjvg\n9NPD/UmTwjWeKkEa36dKUCJSMf7yFzj22HD/iivgmGPixiPNUw1KJBLVoMrrn/8ME74uWwZnnAFX\nXRU7ImnQVA1KCUokEiWo8nntNdhrL/jwQzjqKLjlFuig8aPE0EESeUrjOG1sajMpp9b2t/feC1MY\nffhh+DlpUmUmpzS+Tyvw3yQilWLJknAQxJw5sPvu4Yi9Tp1iRyX50hCfSCQa4iuturqQnGprYcCA\ncBmNPn1iRyW5aIhPRCrG2rVw9NEhOW2+eZj8VckpfZSgsqRxnDY2tZmUU0v9zR3GjQvDeT17hvn1\n+vcvS2iJlsb3qRKUiLQrl1wSruXUuXOYmXznnWNHJIVSDUokEtWgiu+GG+CEE8JRelOmwCGHxI5I\n8qEalIi0a3/6E5x0Urh/7bVKTu2BElSWNI7TxqY2k3LK1d9mzoQjjoD6erjggnWJStZJ4/tUCUpE\nUu2FF+Cgg2DlypCYzj8/dkRSLKpBiUSiGlTbvfUWDB4M774bhvTuvjtcQkPSRXPxiSSMElTbfPQR\nDBkC//kP7L13uCJuly6xo5JClPwgCTPrZGYTzWyumS0xs2fMbESxnr9c0jhOG5vaTMqptraWTz+F\nAw4IyWnnneHPf1Zyakka36fFrEF1BOYBQ919I+B84G4z61fEbYhIhVuzBg47DJ5+OpyA+9BD0KtX\n7KikFEo6xGdmzwMXuPsfs5ZriE8qnob4Wq++HsaMgVtvhU03DfPr7bBD7Kikrcp+HpSZ9QUGAC+V\nahsiUlnOOiskpw03hAceUHJq7zqW4knNrCMwGbjZ3V/Ntc6YMWPon5kgq1evXlRXV1NTUwOsGyuN\n8bjxOG0S4knD4wkTJiTm/5emxw2SEk/SHz/zTA1XXAFmEzj//Gr22CNZ8SX9ccOyJMQze/ZsFi9e\nDMDcuXNpStGH+MzMgDuA7sAod1+bY53EDvHV1tZ+1pCSH7VZYTTEl7/Jk8Ps5ADnnFPLpZfWRI0n\njZL8Pi3bYeZmNgnoB+zv7quaWCexCUqkXJSg8jN9Ohx4YDg44qqr4IwzYkckxdZUgirqEJ+ZXQ/s\nCAxrKjmJiOTr6afh0ENDcjrzTCWnSlPM86D6AScA1cACM1tqZp+Y2ZHF2kY5ZNcHpGVqMymF//wn\nnOv06adwzDFw+eVhufpbYdLYbkXbg3L3eWhuPxEpgnffheHDw2wRI0fCxInhEhpSWTTVkUgkqkHl\ntngx7LMP/Otf8PWvw2OPhcPKpf3S9aBEJPHq6mDUqJCcvvQlmDZNyamSKUFlSeM4bWxqMymGtWvh\n+98P13baYotw9N6mm66/nvpbYdLYbkpQIhKdO5xyCvzxj2FevenTYZttYkclsakGJRKJalDrXHhh\nuBJuly7wyCMwdGjsiKScVIMSkUS6/vqQnDp0gDvvVHKSdZSgsqRxnDY2tZkU6t574eSTw/3rrw8H\nSLRE/a0waWw3JSgRieKJJ8JBEe5w8cUwdmzsiCRpVIMSiaSSa1DPPx8u0/7JJ+HgiN/+Fmy9CoRU\nirJNFptnMEpQUvEqNUHNmQODB8P778P3vhfqTlVVsaOSmHSQRJ7SOE4bm9pM8vXhh2EKo/ffh333\nDZfRaG1yUn8rTBrbTQlKRMpi2bIw+etrr0F1dTjnqXPn2FFJkmmITySSShriW7UKvvOdcI7TttvC\nrFnwhS/EjkqSQkN8IhJFfT0cd1xITn36hJ9KTpIPJagsaRynjU1tJk1xh5/8BG67Dbp3h4cegu23\nb9tzqr8VJo3tpgQlIiVz5ZVw9dWwwQZw332w226xI5I0UQ1KJJL2XoO65RY49thw/4474Igj4sYj\nyaUalIiUzYMPhroTwIQJSk5SGCWoLGkcp41NbSaNPfUUHHZYuL7TWWfBaacV9/nV3wqTxnZTghKR\nonnllXCu0/Ll8F//BT//eeyIJM1UgxKJpL3VoN55B/bcE+bPhwMPDCfiduwYOypJA9WgRKRkFi2C\nESNCctpzT7jrLiUnaTslqCxpHKeNTW1W2VasgIMOghdfhIED4f77oVu30m1P/a0waWw3JSgRKdia\nNeEIvb/9DbbaCqZPh002iR2VtBeqQYlEkvYalDuccAJMnAi9e8Nf/wpf/nLsqCSNVIMSkaIaPz4k\np65dYdo0JScpPiWoLGkcp41NbVZ5rr02XKa9qiocEDF4cPm2rf5WmDS2mxKUiLTKlClw6qnh/v/+\nb7iMhkgpqAYlEkkaa1AzZoTDyVetCifhnn127IikPWiqBqUEJRJJ2hLUc8/BPvvA0qUwblyYY8/W\n+0gRaT0dJJGnNI7TxqY2a//efBNGjgzJafTocAmNWMlJ/a0waWw3JSgRadaCBfDtb4ef3/oW/OEP\n0EGfHFIGGuITiSQNQ3xLl0JNDTz7LOy6a6hB9ewZOyppb8oyxGdmp5jZP8yszswmFfO5RaS8Vq2C\nQw4JyWm77cI1npScpJyKvaP+DnAxcGORn7ds0jhOG5varP2prw9Xw330UdhsszCFUd++saMK1N8K\nk8Z2K+p8w+7+JwAz2x3YspjPLSLl4Q6nnw533gk9esDDD4c9KJFyK0kNyswuBrZ09+Oa+L1qUFLx\nklqDuvzycH5Tp07w0EPwzW/Gjkjau6ZqUNGu2DJmzBj69+8PQK9evaiurqampgZYtyuqx3rcnh83\nSEo8NTU13HQTnH12eHzrrTV885vJik+P28fj2bNns3jxYgDmzp1LU7QHlaW2tvazhpT8qM0Kk7Q9\nqGnT4LvfhbVr4be/hR/+MHZEuam/FSbJ7aYTdUWkSbNmweGHh+R07rnJTU5SWYq6B2VmVcAGwPnA\nVsBYYI27r81aL7F7UCLlkpQ9qJdfhiFDwmXbjz8ebrhBUxhJeZVrD+o8YDnwM+CozP1zi7wNESmS\n+fNh+PCQnA46CK6/XslJkqOoCcrdL3T3Du5e1eh2UTG3UWrZBWxpmdosnRYuDDOTv/027LVXOKy8\nY7TDpvKn/laYNLabalAiFWj58nAdp5dfDlfCnTo1XBlXJEk0F59IJLFqUGvWwMEHh6P2tt46HCCx\n1VZlD0PkMzqKT0RwhxNPDMlp443DFEZKTpJUSlBZ0jhOG5vaLD3OOw8mTQrDeQ88AAMHxo6o9dTf\nCpPGdlOCEqkQv/lNuEx7VRVMmQLf+EbsiESapxqUSCTlrEHddRcceWQY4rv55jBTuUhSqAYlUqEe\nfRSOPjokp1/8QslJ0kMJKksax2ljU5sl17PPhiP2Vq+GH/0IzjwzdkRtp/5WmDS2mxKUSDv1+usw\nciQsWxaG9666SrNESLqoBiUSSSlrUO+/H2aHePNN2G+/cFh5p04l2ZRIm6kGJVIhPvkE9t8/JKev\nfQ3uvVfJSdJJCSpLGsdpY1ObJcfKlaHm9NxzMGBAONepR4/YURWX+lth0thuSlAi7cTateFovccf\nhy98IcwSsdlmsaMSKZxqUCKRFLMG5Q7jxsE110DPnvDEE1BdXZSnFik51aBE2rHLLgvJqVMn+POf\nlZykfVCCypLGcdrY1GZxTZwYLtNuBrffDjU1sSMqLfW3wqSx3ZSgRFJs6tQwOznAtdfCoYfGjUek\nmFSDEomkrTWoJ5+EYcOgrg7OPx8uvLCIwYmUUVM1KCUokUjakqBeegmGDIHFi+GEE+D66zVLhKSX\nDpLIUxrHaWNTm5XXvHkwfHhITgcfDNddV1nJSf2tMGlsNyUokRT5+OOQnN55B4YODQdFVFXFjkqk\nNDTEJxJJa4f4Pv001JyeegoGDYKZM6FXrxIGKFImGuITSbHVq2H06JCcttkGHn5YyUnaPyWoLGkc\np41NbVZa7jB2bJhXb5NNwhRGW2wRO6p41N8Kk8Z2U4ISSbizz4Y//AG6dQtJ6ktfih2RSHmoBiUS\nST41qAkT4PTToWNHuP9+GDGiTMGJlJFqUCIpc8cdITkBTJqk5CSVRwkqSxrHaWNTmxXfI4/AsceG\n+1deGS6jIYH6W2HS2G5KUCIJ849/wCGHhCP3fvzjcBOpRKpBiUSSqwb12msweDB89BH84Afh4IgO\n+hop7Zzm4hNJmOwE9d57sNdeMGdOqDdNnQobbBAxQJEy0UESeUrjOG1sarO2W7IERo4MyWmPPWDK\nFCWnpqi/FSaN7aYEJRJZXR1897vw/POwww7hXKfu3WNHJRJfUYf4zKw3MAnYD/gQOMfd78ixnob4\npOKZGWvWOKNHw733wuabw6xZ0L9/7MhEyqupIb6ORd7OdUAd0AfYFXjAzGa7+ytF3o5IuzBuXEhO\nG20U5tdTchJZp2hDfGbWDTgEOM/dV7j7k8BUIFVncKRxnDY2tVnhrrsOOncOB0TsvHPsaNJB/a0w\naWy3YtagdgDWuPsbjZY9D3y5iNsQaReuuSb87NAhzBix995x4xFJomIO8XUHlmQtWwL0yLWyVdIl\nQEWaUF9vHHJI7ChEkqmYCWoZ0DNrWU9gaa6VdZCEVKL77w+XaV+7FqB1FywUaa+a2mEp5hDfq0BH\nM9uu0bJdgJeKuI2SS+M4bWxqs/zU1sJhh4XkdM45saNJL/W3wqSx3YqWoNx9OXAfcJGZdTOzvYCD\ngFuLtQ2RtHrsMTjgAFi5Ek46CS65JHZEIslXyvOgPgJ+5u535VhP50FJxZg+PZyIW1cHY8bAxIlQ\nVZXf9aBEKoHm4hOJYNo0OPRQWLUKTjwxHFbeMPmrEpRIoLn48pTGcdrY1Ga53XNPuGzGqlVw6qnw\nu99pZvJiUH8rTBrbTW8XkRKYMAEOP3zdNZ1+/WvQmRUiraMhPpEiqq+Hn/wErr46PL7sMvjZz3In\nJw3xiQTlmotPpGLV1cExx6y7VMZNN8FRR8WOSiS9NMSXJY3jtLGpzWD+/DBd0ZQp0LNnmPhVyak0\n1N8Kk8Z20x6USBvNnBlOwP3ggzAb+dSpMGhQ7KhE0k81KJECuYdJX884A9asgWHD4M47YZNN8vt7\n1aBEAh1mLlJEixbBEUeE6zmtWQNnngkPPZR/chKRlilBZUnjOG1sldZmf/sbVFfD3XeHS7PfeSf8\n8pfQUQPmZVFp/a1Y0thuSlAieVq9Gi64APbZB+bNgz32gNmzYfTo2JGJtE+qQYnk4bnn4Pjjw08z\nOOssuPDCcDh5oVSDEgl0HpRIAerq4OKL4Re/CJfJ6N8fJk2CffeNHZlI+6chvixpHKeNrb222WOP\nwVe/Cj//eZghYtw4eOEFJafY2mt/K7U0tpv2oESyvPVWmD/v3nvD4y99CW68EfbaK25cIpVGNSiR\njGXL4KqrwnDeihXQrRuce244z6lLl+JvTzUokUA1KJEmrFoFN9wQak0LFoRlo0fDFVfA1lvHjU2k\nkqkGlSWN47SxpbXN1q6F226DgQPhhz8MyWmPPaC2NpzbpOSUTGntb7Glsd20ByUVZ9UqmDwZLr8c\nXnstLNtxR7j0Ujj4YF23SSQpVIOSirFsWbgExpVXhhNtAb74xVBnOuaY8s8EoRqUSKAalFSsOXPC\npK433ghLloRlAweGxDR6tKYoEkkq1aCypHGcNrYkttnq1eGyF6NGwfbbw69+FZLTkCHh8PEXXwzX\na1JySp8k9rc0SGO76e0p7cq//x2G8W65Bd5/PyzbYIOQjE47DXbbLW58IpI/1aAk9T74AO67Lxz4\n8OST65bvuCMcd1yoL/XtGy++pqgGJRKoBiXtyoIFISlNmQJPPBGmIoJw+YvRo8PErt/4ho7IE0kz\n1aCypHGcNrZytJk7vPJKqCXV1MDmm8PJJ8OMGVBVBQccADffDO+9BxMnwp57Kjm1V3qPFiaN7aY9\nKEmsZcvg8cfDlWofeijMkdegUyf49rfhsMPgoIOgV694cYpIaagGJYmxdCnMmhWG7GbOhKefDkfj\nNdh0UxgxAkaODHtMG20UL9ZiUA1KJFANShLFPZws+49/wFNPhYT07LNh+qEGZqGONHJkuO22G3TQ\noLRIxVCCylJbW0tNTU3sMFKlpTZzh3ffhX/9C/75z7Bn9PTT4ei7xqqqwlx4++wTbnvtpaE7WZ/e\no4VJY7spQUlRLV0aToJ94YWQkF54IdwWLVp/3Y03Dglp993DCbSDB4ej8EREQDUoKcDKlfDmm2Gi\n1ddeg9dfX3e/YY67bL17w6BBsOuuISntsUeYB6+Sj7RTDUokUA1K8rZsWUg08+ev/3POnHA0XcN5\nR9k6dQrz3A0aBDvvHH4OGgRbbFHZyUhEWk8JKksax2nz4Q6LF4cTXHPd3nsvJKD583MPxzXWoQNs\nuy0MGBBuUMv++9cwYAD07x+mFhIplfb6Hi21NLZbURKUmZ0CjAEGAbe7+3HFeF5Zn3vYw1m0CBYu\n/PzPXMsWLgwHI3zwQbgOUj46dw4X6+vXb93PhvvbbBOSU+fO69avrQ0nz4qIFFNRalBm9l2gHhgO\ndG0pQbX3GlR9PdTVhduKFet+Zt9fvjwkm6VLwy3f+00Nr7WkR48wJ11Tt4Zk1KePhuPKQTUokaCk\nNSh3/1NmI7sDW+bzN59+Gs55aXyrr19/WVt+13j5mjXhpM9Vq8LPxvfbsixXAlq5shit2rRu3cIR\ncL17h1uu+42XbbZZSEBdu5Y2LhGRYopWg0ru4cS1QE2bn6VLl3Dr2nXdrfHjhvs9eqy7de/e/P3u\n3cMtaTWeNI5tS3qpvxUmle3m7kW7ARcDk/JYz3PdNtxwvG+xhXufPjO8b98Zvt127jvs4N6z57E5\n199qq/E+dKj7LrvM8K9+dYbvt5/7iBHuW26Ze/1ddhnvY8e6jxo1ww89dIafeab7Oee477JL7vVH\njRrvt9/ufsEFM/zii2f4gw+6/+Uv7sOH517/pz8d78uXuz/22AyfMWOGNzj22Nzrjx8/3t3dZ8xI\n9/pXX311ouLR+u17ffW39rm+58gVLdagzGwGsE/mSbI96e57N1r3YmBLr/AalEg+VIMSCQquQbn7\nvqUJSUREpGlFmXrTzKrMrAtQBXQ0s85mVlWM5y63NF4zJTa1mZST+lth0thuxZob+jxgOfAz4KjM\n/XOL9NwGele/AAAFTklEQVQiIlKBNBefSCSqQYkETdWgdHUdERFJJCWoLGkcp41NbSblpP5WmDS2\nmxKUiIgkkmpQIpGoBiUSqAYlIiKpogSVJY3jtLGpzaSc1N8Kk8Z2U4ISEZFEUg1KJBLVoEQC1aBE\nRCRVlKCypHGcNja1mZST+lth0thuSlAiIpJIqkGJRKIalEigGpSIiKSKElSWNI7TxqY2k3JSfytM\nGttNCUpERBJJNSiRSFSDEglUgxIRkVRRgsqSxnHa2NRmUk7qb4VJY7spQYmISCKpBiUSiWpQIoFq\nUCIikipKUFnSOE4bm9pMykn9rTBpbDclKBERSSTVoEQiUQ1KJFANSkREUkUJKksax2ljU5tJOam/\nFSaN7aYEJSIiiaQalEgkqkGJBKpBiYhIqihBZUnjOG1sajMpJ/W3wqSx3ZSgREQkkVSDEolENSiR\nQDUoERFJlTYnKDPrZGYTzWyumS0xs2fMbEQxgoshjeO0sanNpJzU3wqTxnYrxh5UR2AeMNTdNwLO\nB+42s35FeG4REalQJalBmdnzwAXu/scmfq8alFQ81aBEgrLVoMysLzAAeKnYzy0iIpWjYzGfzMw6\nApOBm9391ebWHTNmDP379wegV69eVFdXU1NTA6wbK43xuPE4bRLiScPjCRMmJOb/l6bHDZIST1oe\nq78V9rhhWRLimT17NosXLwZg7ty5NKXFIT4zmwHsA+Ra8Ul33zuzngF3AN2BUe6+tpnnTOwQX21t\n7WcNKflRmxVGQ3yFUX8rTJLbrakhvqLVoMxsEtAP2N/dV7WwbmITlEi5KEGJBE0lqKIM8ZnZ9cCO\nwLCWkpOIiEg+inEeVD/gBKAaWGBmS83sEzM7ss3RRZBdH5CWqc2knNTfCpPGdmtzgnL3ee7ewd27\nuXuPzK2nu99RjADLbfbs2bFDSB21mZST+lth0thumuooS8ORJZI/tZmUk/pbYdLYbkpQIiKSSEpQ\nWZo7Jl9yU5tJOam/FSaN7Rbtchtl36iIiCRWSc+DEhERKSYN8YmISCIpQYmISCIpQYmISCIpQYmI\nSCIpQTXDzAaY2QozuyV2LElnZp3MbKKZzTWzJWb2jJmNiB1XEplZbzP7o5ktM7M5aZ0WrJzUv9ou\njZ9nSlDNuwZ4OnYQKdERmAcMdfeNgPOBuzNzNcrnXQfUAX2AHwC/M7OBcUNKPPWvtkvd55kSVBPM\n7AhgEfBY7FjSwN2Xu/tF7j4/8/gBYA6wW9zIksXMugGHAOe5+wp3fxKYChwdN7JkU/9qm7R+nilB\n5WBmPYELgR8D6508Ji0zs77AAOCl2LEkzA7AGnd/o9Gy54EvR4onldS/8pfmzzMlqNwuAm5w93di\nB5JGZtYRmAzc7O6vxo4nYboDS7KWLQF6RIglldS/Wi21n2cVl6DMbIaZ1ZvZ2hy3mWa2CzAMmBA7\n1iRpqd0arWeED4+VwKnRAk6uZUDPrGU9gaURYkkd9a/WMbNqUvx5VpQr6qaJu+/b3O/N7DRgG2Be\n5s3QHagys53c/WvliDGJWmq3Rm4ENgX2d/e1JQwprV4FOprZdo2G+XZBQ1X5Uv9qnX1I8eeZ5uLL\nYmZd+Pw33DMJ/+CT3H1hnKjSwcyuB3YGhrn78tjxJJWZ3Q44MBb4KjANGOzur0QNLOHUv1ov7Z9n\nFbcH1RJ3ryMcAgyAmS0D6tLwz4wpc7jvCYS2WxC+rOHAiWm9unIJnQJMAj4APiJ8WCg5NUP9qzBp\n/zzTHpSIiCRSxR0kISIi6aAEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIi\nifT/9xF4HfJNR+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f73188cc898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"elu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Implementing ELU in TensorFlow is trivial, just specify the activation function when building each layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.elu, name=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Batch Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.batch_norm()` rather than `tf.layers.batch_normalization()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.batch_normalization()`, because anything in the contrib module may change or be deleted without notice. Instead of using the `batch_norm()` function as a regularizer parameter to the `fully_connected()` function, we now use `batch_normalization()` and we explicitly create a distinct layer. The parameters are a bit different, in particular:\n",
    "* `decay` is renamed to `momentum`,\n",
    "* `is_training` is renamed to `training`,\n",
    "* `updates_collections` is removed: the update operations needed by batch normalization are added to the `UPDATE_OPS` collection and you need to explicity run these operations during training (see the execution phase below),\n",
    "* we don't need to specify `scale=True`, as that is the default.\n",
    "\n",
    "Also note that in order to run batch norm just _before_ each hidden layer's activation function, we apply the ELU activation function manually, right after the batch norm layer.\n",
    "\n",
    "Note: since the `tf.layers.dense()` function is incompatible with `tf.contrib.layers.arg_scope()` (which is used in the book), we now use python's `functools.partial()` function instead. It makes it easy to create a `my_dense_layer()` function that just calls `tf.layers.dense()` with the desired parameters automatically set (unless they are overridden when calling `my_dense_layer()`). As you can see, the code remains very similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\")\n",
    "bn1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9)\n",
    "bn1_act = tf.nn.elu(bn1)\n",
    "\n",
    "hidden2 = tf.layers.dense(bn1_act, n_hidden2, name=\"hidden2\")\n",
    "bn2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9)\n",
    "bn2_act = tf.nn.elu(bn2)\n",
    "\n",
    "logits_before_bn = tf.layers.dense(bn2_act, n_outputs, name=\"outputs\")\n",
    "logits = tf.layers.batch_normalization(logits_before_bn, training=training,\n",
    "                                       momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "To avoid repeating the same parameters over and over again, we can use Python's `partial()` function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "my_batch_norm_layer = partial(tf.layers.batch_normalization,\n",
    "                              training=training, momentum=0.9)\n",
    "\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\")\n",
    "bn1 = my_batch_norm_layer(hidden1)\n",
    "bn1_act = tf.nn.elu(bn1)\n",
    "hidden2 = tf.layers.dense(bn1_act, n_hidden2, name=\"hidden2\")\n",
    "bn2 = my_batch_norm_layer(hidden2)\n",
    "bn2_act = tf.nn.elu(bn2)\n",
    "logits_before_bn = tf.layers.dense(bn2_act, n_outputs, name=\"outputs\")\n",
    "logits = my_batch_norm_layer(logits_before_bn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's build a neural net for MNIST, using the ELU activation function and Batch Normalization at each layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "batch_norm_momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    my_batch_norm_layer = partial(\n",
    "            tf.layers.batch_normalization,\n",
    "            training=training,\n",
    "            momentum=batch_norm_momentum)\n",
    "\n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=he_init)\n",
    "\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    bn1 = tf.nn.elu(my_batch_norm_layer(hidden1))\n",
    "    hidden2 = my_dense_layer(bn1, n_hidden2, name=\"hidden2\")\n",
    "    bn2 = tf.nn.elu(my_batch_norm_layer(hidden2))\n",
    "    logits_before_bn = my_dense_layer(bn2, n_outputs, name=\"outputs\")\n",
    "    logits = my_batch_norm_layer(logits_before_bn)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: since we are using `tf.layers.batch_normalization()` rather than `tf.contrib.layers.batch_norm()` (as in the book), we need to explicitly run the extra update operations needed by batch normalization (`sess.run([training_op, extra_update_ops],...`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.868\n",
      "1 Test accuracy: 0.8967\n",
      "2 Test accuracy: 0.9126\n",
      "3 Test accuracy: 0.9211\n",
      "4 Test accuracy: 0.9301\n",
      "5 Test accuracy: 0.9355\n",
      "6 Test accuracy: 0.9387\n",
      "7 Test accuracy: 0.945\n",
      "8 Test accuracy: 0.9461\n",
      "9 Test accuracy: 0.9497\n",
      "10 Test accuracy: 0.9518\n",
      "11 Test accuracy: 0.9551\n",
      "12 Test accuracy: 0.9567\n",
      "13 Test accuracy: 0.9589\n",
      "14 Test accuracy: 0.96\n",
      "15 Test accuracy: 0.9626\n",
      "16 Test accuracy: 0.9639\n",
      "17 Test accuracy: 0.9634\n",
      "18 Test accuracy: 0.9653\n",
      "19 Test accuracy: 0.9649\n"
     ]
    }
   ],
   "source": [
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run([training_op, extra_update_ops],\n",
    "                     feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "What!? That's not a great accuracy for MNIST. Of course, if you train for longer it will get much better accuracy, but with such a shallow network, Batch Norm and ELU are unlikely to have very positive impact: they shine mostly for much deeper nets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note that you could also make the training operation depend on the update operations:\n",
    "\n",
    "```python\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(extra_update_ops):\n",
    "        training_op = optimizer.minimize(loss)\n",
    "```\n",
    "\n",
    "This way, you would just have to evaluate the `training_op` during training, TensorFlow would automatically run the update operations as well:\n",
    "\n",
    "```python\n",
    "sess.run(training_op, feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "One more thing: notice that the list of trainable variables is shorter than the list of all global variables. This is because the moving averages are non-trainable variables. If you want to reuse a pretrained neural network (see below), you must not forget these non-trainable variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/kernel:0',\n",
       " 'hidden1/bias:0',\n",
       " 'batch_normalization/beta:0',\n",
       " 'batch_normalization/gamma:0',\n",
       " 'hidden2/kernel:0',\n",
       " 'hidden2/bias:0',\n",
       " 'batch_normalization_1/beta:0',\n",
       " 'batch_normalization_1/gamma:0',\n",
       " 'outputs/kernel:0',\n",
       " 'outputs/bias:0',\n",
       " 'batch_normalization_2/beta:0',\n",
       " 'batch_normalization_2/gamma:0']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.trainable_variables()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/kernel:0',\n",
       " 'hidden1/bias:0',\n",
       " 'batch_normalization/beta:0',\n",
       " 'batch_normalization/gamma:0',\n",
       " 'batch_normalization/moving_mean:0',\n",
       " 'batch_normalization/moving_variance:0',\n",
       " 'hidden2/kernel:0',\n",
       " 'hidden2/bias:0',\n",
       " 'batch_normalization_1/beta:0',\n",
       " 'batch_normalization_1/gamma:0',\n",
       " 'batch_normalization_1/moving_mean:0',\n",
       " 'batch_normalization_1/moving_variance:0',\n",
       " 'outputs/kernel:0',\n",
       " 'outputs/bias:0',\n",
       " 'batch_normalization_2/beta:0',\n",
       " 'batch_normalization_2/gamma:0',\n",
       " 'batch_normalization_2/moving_mean:0',\n",
       " 'batch_normalization_2/moving_variance:0']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.global_variables()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Gradient Clipping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's create a simple neural net for MNIST and add gradient clipping. The first part is the same as earlier (except we added a few more layers to demonstrate reusing pretrained models, see below):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_hidden3 = 50\n",
    "n_hidden4 = 50\n",
    "n_hidden5 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\")\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\")\n",
    "    hidden5 = tf.layers.dense(hidden4, n_hidden5, activation=tf.nn.relu, name=\"hidden5\")\n",
    "    logits = tf.layers.dense(hidden5, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now we apply gradient clipping. For this, we need to get the gradients, use the `clip_by_value()` function to clip them, then apply them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "threshold = 1.0\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "grads_and_vars = optimizer.compute_gradients(loss)\n",
    "capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "              for grad, var in grads_and_vars]\n",
    "training_op = optimizer.apply_gradients(capped_gvs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The rest is the same as usual:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.586\n",
      "1 Test accuracy: 0.8165\n",
      "2 Test accuracy: 0.8742\n",
      "3 Test accuracy: 0.8963\n",
      "4 Test accuracy: 0.9084\n",
      "5 Test accuracy: 0.9137\n",
      "6 Test accuracy: 0.924\n",
      "7 Test accuracy: 0.9276\n",
      "8 Test accuracy: 0.9309\n",
      "9 Test accuracy: 0.9374\n",
      "10 Test accuracy: 0.939\n",
      "11 Test accuracy: 0.9419\n",
      "12 Test accuracy: 0.9437\n",
      "13 Test accuracy: 0.9475\n",
      "14 Test accuracy: 0.9487\n",
      "15 Test accuracy: 0.9503\n",
      "16 Test accuracy: 0.9514\n",
      "17 Test accuracy: 0.9552\n",
      "18 Test accuracy: 0.9535\n",
      "19 Test accuracy: 0.9569\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Reusing Pretrained Layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Reusing a TensorFlow Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First you need to load the graph's structure. The `import_meta_graph()` function does just that, loading the graph's operations into the default graph, and returning a `Saver` that you can then use to restore the model's state. Note that by default, a `Saver` saves the structure of the graph into a `.meta` file, so that's the file you should load:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.import_meta_graph(\"./my_model_final.ckpt.meta\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next you need to get a handle on all the operations you will need for training. If you don't know the graph's structure, you can list all the operations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X\n",
      "y\n",
      "hidden1/kernel/Initializer/random_uniform/shape\n",
      "hidden1/kernel/Initializer/random_uniform/min\n",
      "hidden1/kernel/Initializer/random_uniform/max\n",
      "hidden1/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden1/kernel/Initializer/random_uniform/sub\n",
      "hidden1/kernel/Initializer/random_uniform/mul\n",
      "hidden1/kernel/Initializer/random_uniform\n",
      "hidden1/kernel\n",
      "hidden1/kernel/Assign\n",
      "hidden1/kernel/read\n",
      "hidden1/bias/Initializer/Const\n",
      "hidden1/bias\n",
      "hidden1/bias/Assign\n",
      "hidden1/bias/read\n",
      "dnn/hidden1/MatMul\n",
      "dnn/hidden1/BiasAdd\n",
      "dnn/hidden1/Relu\n",
      "hidden2/kernel/Initializer/random_uniform/shape\n",
      "hidden2/kernel/Initializer/random_uniform/min\n",
      "hidden2/kernel/Initializer/random_uniform/max\n",
      "hidden2/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden2/kernel/Initializer/random_uniform/sub\n",
      "hidden2/kernel/Initializer/random_uniform/mul\n",
      "hidden2/kernel/Initializer/random_uniform\n",
      "hidden2/kernel\n",
      "hidden2/kernel/Assign\n",
      "hidden2/kernel/read\n",
      "hidden2/bias/Initializer/Const\n",
      "hidden2/bias\n",
      "hidden2/bias/Assign\n",
      "hidden2/bias/read\n",
      "dnn/hidden2/MatMul\n",
      "dnn/hidden2/BiasAdd\n",
      "dnn/hidden2/Relu\n",
      "hidden3/kernel/Initializer/random_uniform/shape\n",
      "hidden3/kernel/Initializer/random_uniform/min\n",
      "hidden3/kernel/Initializer/random_uniform/max\n",
      "hidden3/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden3/kernel/Initializer/random_uniform/sub\n",
      "hidden3/kernel/Initializer/random_uniform/mul\n",
      "hidden3/kernel/Initializer/random_uniform\n",
      "hidden3/kernel\n",
      "hidden3/kernel/Assign\n",
      "hidden3/kernel/read\n",
      "hidden3/bias/Initializer/Const\n",
      "hidden3/bias\n",
      "hidden3/bias/Assign\n",
      "hidden3/bias/read\n",
      "dnn/hidden3/MatMul\n",
      "dnn/hidden3/BiasAdd\n",
      "dnn/hidden3/Relu\n",
      "hidden4/kernel/Initializer/random_uniform/shape\n",
      "hidden4/kernel/Initializer/random_uniform/min\n",
      "hidden4/kernel/Initializer/random_uniform/max\n",
      "hidden4/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden4/kernel/Initializer/random_uniform/sub\n",
      "hidden4/kernel/Initializer/random_uniform/mul\n",
      "hidden4/kernel/Initializer/random_uniform\n",
      "hidden4/kernel\n",
      "hidden4/kernel/Assign\n",
      "hidden4/kernel/read\n",
      "hidden4/bias/Initializer/Const\n",
      "hidden4/bias\n",
      "hidden4/bias/Assign\n",
      "hidden4/bias/read\n",
      "dnn/hidden4/MatMul\n",
      "dnn/hidden4/BiasAdd\n",
      "dnn/hidden4/Relu\n",
      "hidden5/kernel/Initializer/random_uniform/shape\n",
      "hidden5/kernel/Initializer/random_uniform/min\n",
      "hidden5/kernel/Initializer/random_uniform/max\n",
      "hidden5/kernel/Initializer/random_uniform/RandomUniform\n",
      "hidden5/kernel/Initializer/random_uniform/sub\n",
      "hidden5/kernel/Initializer/random_uniform/mul\n",
      "hidden5/kernel/Initializer/random_uniform\n",
      "hidden5/kernel\n",
      "hidden5/kernel/Assign\n",
      "hidden5/kernel/read\n",
      "hidden5/bias/Initializer/Const\n",
      "hidden5/bias\n",
      "hidden5/bias/Assign\n",
      "hidden5/bias/read\n",
      "dnn/hidden5/MatMul\n",
      "dnn/hidden5/BiasAdd\n",
      "dnn/hidden5/Relu\n",
      "outputs/kernel/Initializer/random_uniform/shape\n",
      "outputs/kernel/Initializer/random_uniform/min\n",
      "outputs/kernel/Initializer/random_uniform/max\n",
      "outputs/kernel/Initializer/random_uniform/RandomUniform\n",
      "outputs/kernel/Initializer/random_uniform/sub\n",
      "outputs/kernel/Initializer/random_uniform/mul\n",
      "outputs/kernel/Initializer/random_uniform\n",
      "outputs/kernel\n",
      "outputs/kernel/Assign\n",
      "outputs/kernel/read\n",
      "outputs/bias/Initializer/Const\n",
      "outputs/bias\n",
      "outputs/bias/Assign\n",
      "outputs/bias/read\n",
      "dnn/outputs/MatMul\n",
      "dnn/outputs/BiasAdd\n",
      "loss/SparseSoftmaxCrossEntropyWithLogits/Shape\n",
      "loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits\n",
      "loss/Const\n",
      "loss/loss\n",
      "gradients/Shape\n",
      "gradients/Const\n",
      "gradients/Fill\n",
      "gradients/loss/loss_grad/Reshape/shape\n",
      "gradients/loss/loss_grad/Reshape\n",
      "gradients/loss/loss_grad/Shape\n",
      "gradients/loss/loss_grad/Tile\n",
      "gradients/loss/loss_grad/Shape_1\n",
      "gradients/loss/loss_grad/Shape_2\n",
      "gradients/loss/loss_grad/Const\n",
      "gradients/loss/loss_grad/Prod\n",
      "gradients/loss/loss_grad/Const_1\n",
      "gradients/loss/loss_grad/Prod_1\n",
      "gradients/loss/loss_grad/Maximum/y\n",
      "gradients/loss/loss_grad/Maximum\n",
      "gradients/loss/loss_grad/floordiv\n",
      "gradients/loss/loss_grad/Cast\n",
      "gradients/loss/loss_grad/truediv\n",
      "gradients/zeros_like\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims\n",
      "gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul\n",
      "gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/outputs/MatMul_grad/MatMul\n",
      "gradients/dnn/outputs/MatMul_grad/MatMul_1\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden5/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden5/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden5/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden4/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden4/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden4/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden3/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden3/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden3/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden2/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden2/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden2/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden1/Relu_grad/ReluGrad\n",
      "gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1\n",
      "gradients/dnn/hidden1/MatMul_grad/MatMul\n",
      "gradients/dnn/hidden1/MatMul_grad/MatMul_1\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/group_deps\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency\n",
      "gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1\n",
      "clip_by_value/Minimum/y\n",
      "clip_by_value/Minimum\n",
      "clip_by_value/y\n",
      "clip_by_value\n",
      "clip_by_value_1/Minimum/y\n",
      "clip_by_value_1/Minimum\n",
      "clip_by_value_1/y\n",
      "clip_by_value_1\n",
      "clip_by_value_2/Minimum/y\n",
      "clip_by_value_2/Minimum\n",
      "clip_by_value_2/y\n",
      "clip_by_value_2\n",
      "clip_by_value_3/Minimum/y\n",
      "clip_by_value_3/Minimum\n",
      "clip_by_value_3/y\n",
      "clip_by_value_3\n",
      "clip_by_value_4/Minimum/y\n",
      "clip_by_value_4/Minimum\n",
      "clip_by_value_4/y\n",
      "clip_by_value_4\n",
      "clip_by_value_5/Minimum/y\n",
      "clip_by_value_5/Minimum\n",
      "clip_by_value_5/y\n",
      "clip_by_value_5\n",
      "clip_by_value_6/Minimum/y\n",
      "clip_by_value_6/Minimum\n",
      "clip_by_value_6/y\n",
      "clip_by_value_6\n",
      "clip_by_value_7/Minimum/y\n",
      "clip_by_value_7/Minimum\n",
      "clip_by_value_7/y\n",
      "clip_by_value_7\n",
      "clip_by_value_8/Minimum/y\n",
      "clip_by_value_8/Minimum\n",
      "clip_by_value_8/y\n",
      "clip_by_value_8\n",
      "clip_by_value_9/Minimum/y\n",
      "clip_by_value_9/Minimum\n",
      "clip_by_value_9/y\n",
      "clip_by_value_9\n",
      "clip_by_value_10/Minimum/y\n",
      "clip_by_value_10/Minimum\n",
      "clip_by_value_10/y\n",
      "clip_by_value_10\n",
      "clip_by_value_11/Minimum/y\n",
      "clip_by_value_11/Minimum\n",
      "clip_by_value_11/y\n",
      "clip_by_value_11\n",
      "GradientDescent/learning_rate\n",
      "GradientDescent/update_hidden1/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden1/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden2/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden2/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden3/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden3/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden4/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden4/bias/ApplyGradientDescent\n",
      "GradientDescent/update_hidden5/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_hidden5/bias/ApplyGradientDescent\n",
      "GradientDescent/update_outputs/kernel/ApplyGradientDescent\n",
      "GradientDescent/update_outputs/bias/ApplyGradientDescent\n",
      "GradientDescent\n",
      "eval/InTopK\n",
      "eval/Cast\n",
      "eval/Const\n",
      "eval/accuracy\n",
      "init\n",
      "save/Const\n",
      "save/SaveV2/tensor_names\n",
      "save/SaveV2/shape_and_slices\n",
      "save/SaveV2\n",
      "save/control_dependency\n",
      "save/RestoreV2/tensor_names\n",
      "save/RestoreV2/shape_and_slices\n",
      "save/RestoreV2\n",
      "save/Assign\n",
      "save/RestoreV2_1/tensor_names\n",
      "save/RestoreV2_1/shape_and_slices\n",
      "save/RestoreV2_1\n",
      "save/Assign_1\n",
      "save/RestoreV2_2/tensor_names\n",
      "save/RestoreV2_2/shape_and_slices\n",
      "save/RestoreV2_2\n",
      "save/Assign_2\n",
      "save/RestoreV2_3/tensor_names\n",
      "save/RestoreV2_3/shape_and_slices\n",
      "save/RestoreV2_3\n",
      "save/Assign_3\n",
      "save/RestoreV2_4/tensor_names\n",
      "save/RestoreV2_4/shape_and_slices\n",
      "save/RestoreV2_4\n",
      "save/Assign_4\n",
      "save/RestoreV2_5/tensor_names\n",
      "save/RestoreV2_5/shape_and_slices\n",
      "save/RestoreV2_5\n",
      "save/Assign_5\n",
      "save/RestoreV2_6/tensor_names\n",
      "save/RestoreV2_6/shape_and_slices\n",
      "save/RestoreV2_6\n",
      "save/Assign_6\n",
      "save/RestoreV2_7/tensor_names\n",
      "save/RestoreV2_7/shape_and_slices\n",
      "save/RestoreV2_7\n",
      "save/Assign_7\n",
      "save/RestoreV2_8/tensor_names\n",
      "save/RestoreV2_8/shape_and_slices\n",
      "save/RestoreV2_8\n",
      "save/Assign_8\n",
      "save/RestoreV2_9/tensor_names\n",
      "save/RestoreV2_9/shape_and_slices\n",
      "save/RestoreV2_9\n",
      "save/Assign_9\n",
      "save/RestoreV2_10/tensor_names\n",
      "save/RestoreV2_10/shape_and_slices\n",
      "save/RestoreV2_10\n",
      "save/Assign_10\n",
      "save/RestoreV2_11/tensor_names\n",
      "save/RestoreV2_11/shape_and_slices\n",
      "save/RestoreV2_11\n",
      "save/Assign_11\n",
      "save/restore_all\n"
     ]
    }
   ],
   "source": [
    "for op in tf.get_default_graph().get_operations():\n",
    "    print(op.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Oops, that's a lot of operations! It's much easier to use TensorBoard to visualize the graph. The following hack will allow you to visualize the graph within Jupyter (if it does not work with your browser, you will need to use a `FileWriter` to save the graph and then visualize it in TensorBoard):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.9200556678219983&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.13093073666095734\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.13093073666095734\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;hidden3/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;hidden3/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden3/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden3/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;hidden3/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden3/MatMul&quot;\\n  input: &quot;hidden3/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden3/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden3/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;hidden4/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;hidden4/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden4/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden4/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  input: &quot;hidden4/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden4/MatMul&quot;\\n  input: &quot;hidden4/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden4/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden4/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\0002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.24494896829128265\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;hidden5/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 50\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;hidden5/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden5/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden5/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  input: &quot;hidden5/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden5/MatMul&quot;\\n  input: &quot;hidden5/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden5/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden5/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;2\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.3162277638912201\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.3162277638912201\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 50\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden5/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden5/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden5/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden5/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden5/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden4/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden4/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden4/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden4/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden4/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden3/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden3/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden3/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden3/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden3/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value/Minimum&quot;\\n  input: &quot;clip_by_value/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_1/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_1&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_1/Minimum&quot;\\n  input: &quot;clip_by_value_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_2/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_2&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_2/Minimum&quot;\\n  input: &quot;clip_by_value_2/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_3/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_3&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_3/Minimum&quot;\\n  input: &quot;clip_by_value_3/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden3/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_4/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_4&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_4/Minimum&quot;\\n  input: &quot;clip_by_value_4/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden3/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_5/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_5&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_5/Minimum&quot;\\n  input: &quot;clip_by_value_5/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden4/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_6/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_6&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_6/Minimum&quot;\\n  input: &quot;clip_by_value_6/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden4/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_7/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_7&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_7/Minimum&quot;\\n  input: &quot;clip_by_value_7/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden5/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_8/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_8&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_8/Minimum&quot;\\n  input: &quot;clip_by_value_8/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/hidden5/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_9/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_9&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_9/Minimum&quot;\\n  input: &quot;clip_by_value_9/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_10/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_10&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_10/Minimum&quot;\\n  input: &quot;clip_by_value_10/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;clip_by_value_11/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;clip_by_value_11&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;clip_by_value_11/Minimum&quot;\\n  input: &quot;clip_by_value_11/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden3/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden3/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden4/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden4/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden5/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden5/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;clip_by_value_11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden3/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden3/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden4/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden4/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden5/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden5/bias/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/accuracy&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^hidden3/kernel/Assign&quot;\\n  input: &quot;^hidden3/bias/Assign&quot;\\n  input: &quot;^hidden4/kernel/Assign&quot;\\n  input: &quot;^hidden4/bias/Assign&quot;\\n  input: &quot;^hidden5/kernel/Assign&quot;\\n  input: &quot;^hidden5/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;hidden3/bias&quot;\\n        string_val: &quot;hidden3/kernel&quot;\\n        string_val: &quot;hidden4/bias&quot;\\n        string_val: &quot;hidden4/kernel&quot;\\n        string_val: &quot;hidden5/bias&quot;\\n        string_val: &quot;hidden5/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden3/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden3/kernel&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden3/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden4/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_6/tensor_names&quot;\\n  input: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_6&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/bias&quot;\\n  input: &quot;save/RestoreV2_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_7/tensor_names&quot;\\n  input: &quot;save/RestoreV2_7/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_7&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden4/kernel&quot;\\n  input: &quot;save/RestoreV2_7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden4/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden5/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_8/tensor_names&quot;\\n  input: &quot;save/RestoreV2_8/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_8&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/bias&quot;\\n  input: &quot;save/RestoreV2_8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_9/tensor_names&quot;\\n  input: &quot;save/RestoreV2_9/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_9&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden5/kernel&quot;\\n  input: &quot;save/RestoreV2_9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden5/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_10/tensor_names&quot;\\n  input: &quot;save/RestoreV2_10/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_10&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2_10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_11/tensor_names&quot;\\n  input: &quot;save/RestoreV2_11/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_11&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n  input: &quot;^save/Assign_6&quot;\\n  input: &quot;^save/Assign_7&quot;\\n  input: &quot;^save/Assign_8&quot;\\n  input: &quot;^save/Assign_9&quot;\\n  input: &quot;^save/Assign_10&quot;\\n  input: &quot;^save/Assign_11&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.9200556678219983&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Once you know which operations you need, you can get a handle on them using the graph's `get_operation_by_name()` or `get_tensor_by_name()` methods:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = tf.get_default_graph().get_tensor_by_name(\"X:0\")\n",
    "y = tf.get_default_graph().get_tensor_by_name(\"y:0\")\n",
    "\n",
    "accuracy = tf.get_default_graph().get_tensor_by_name(\"eval/accuracy:0\")\n",
    "\n",
    "training_op = tf.get_default_graph().get_operation_by_name(\"GradientDescent\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "If you are the author of the original model, you could make things easier for people who will reuse your model by giving operations very clear names and documenting them. Another approach is to create a collection containing all the important operations that people will want to get a handle on:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "for op in (X, y, accuracy, training_op):\n",
    "    tf.add_to_collection(\"my_important_ops\", op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This way people who reuse your model will be able to simply write:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X, y, accuracy, training_op = tf.get_collection(\"my_important_ops\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now you can start a session, restore the model's state and continue training on your data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "    # continue training the model..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Actually, let's test this for real!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.9592\n",
      "1 Test accuracy: 0.9591\n",
      "2 Test accuracy: 0.9542\n",
      "3 Test accuracy: 0.9602\n",
      "4 Test accuracy: 0.959\n",
      "5 Test accuracy: 0.9637\n",
      "6 Test accuracy: 0.9585\n",
      "7 Test accuracy: 0.9654\n",
      "8 Test accuracy: 0.9648\n",
      "9 Test accuracy: 0.9636\n",
      "10 Test accuracy: 0.9663\n",
      "11 Test accuracy: 0.9656\n",
      "12 Test accuracy: 0.9658\n",
      "13 Test accuracy: 0.9669\n",
      "14 Test accuracy: 0.9669\n",
      "15 Test accuracy: 0.9661\n",
      "16 Test accuracy: 0.969\n",
      "17 Test accuracy: 0.9656\n",
      "18 Test accuracy: 0.9671\n",
      "19 Test accuracy: 0.9692\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Alternatively, if you have access to the Python code that built the original graph, you can use it instead of `import_meta_graph()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_hidden3 = 50\n",
    "n_hidden4 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\")\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\")\n",
    "    hidden5 = tf.layers.dense(hidden4, n_hidden5, activation=tf.nn.relu, name=\"hidden5\")\n",
    "    logits = tf.layers.dense(hidden5, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "threshold = 1.0\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "grads_and_vars = optimizer.compute_gradients(loss)\n",
    "capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "              for grad, var in grads_and_vars]\n",
    "training_op = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "And continue training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.9566\n",
      "1 Test accuracy: 0.9579\n",
      "2 Test accuracy: 0.9615\n",
      "3 Test accuracy: 0.9594\n",
      "4 Test accuracy: 0.9618\n",
      "5 Test accuracy: 0.959\n",
      "6 Test accuracy: 0.9634\n",
      "7 Test accuracy: 0.964\n",
      "8 Test accuracy: 0.9633\n",
      "9 Test accuracy: 0.9653\n",
      "10 Test accuracy: 0.966\n",
      "11 Test accuracy: 0.9635\n",
      "12 Test accuracy: 0.965\n",
      "13 Test accuracy: 0.968\n",
      "14 Test accuracy: 0.9643\n",
      "15 Test accuracy: 0.9688\n",
      "16 Test accuracy: 0.9656\n",
      "17 Test accuracy: 0.9675\n",
      "18 Test accuracy: 0.9665\n",
      "19 Test accuracy: 0.9687\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "In general you will want to reuse only the lower layers. If you are using `import_meta_graph()` it will load the whole graph, but you can simply ignore the parts you do not need. In this example, we add a new 4th hidden layer on top of the pretrained 3rd layer (ignoring the old 4th hidden layer). We also build a new output layer, the loss for this new output, and a new optimizer to minimize it. We also need another saver to save the whole graph (containing both the entire old graph plus the new operations), and an initialization operation to initialize all the new variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_hidden4 = 20  # new layer\n",
    "n_outputs = 10  # new layer\n",
    "\n",
    "saver = tf.train.import_meta_graph(\"./my_model_final.ckpt.meta\")\n",
    "\n",
    "X = tf.get_default_graph().get_tensor_by_name(\"X:0\")\n",
    "y = tf.get_default_graph().get_tensor_by_name(\"y:0\")\n",
    "\n",
    "hidden3 = tf.get_default_graph().get_tensor_by_name(\"dnn/hidden4/Relu:0\")\n",
    "\n",
    "new_hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"new_hidden4\")\n",
    "new_logits = tf.layers.dense(new_hidden4, n_outputs, name=\"new_outputs\")\n",
    "\n",
    "with tf.name_scope(\"new_loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=new_logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"new_eval\"):\n",
    "    correct = tf.nn.in_top_k(new_logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"new_train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "new_saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "And we can train this new model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.9226\n",
      "1 Test accuracy: 0.9355\n",
      "2 Test accuracy: 0.9422\n",
      "3 Test accuracy: 0.945\n",
      "4 Test accuracy: 0.9517\n",
      "5 Test accuracy: 0.9523\n",
      "6 Test accuracy: 0.9543\n",
      "7 Test accuracy: 0.9562\n",
      "8 Test accuracy: 0.9573\n",
      "9 Test accuracy: 0.9596\n",
      "10 Test accuracy: 0.9544\n",
      "11 Test accuracy: 0.961\n",
      "12 Test accuracy: 0.9621\n",
      "13 Test accuracy: 0.9624\n",
      "14 Test accuracy: 0.9623\n",
      "15 Test accuracy: 0.9639\n",
      "16 Test accuracy: 0.9644\n",
      "17 Test accuracy: 0.9641\n",
      "18 Test accuracy: 0.9647\n",
      "19 Test accuracy: 0.9654\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = new_saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "If you have access to the Python code that built the original graph, you can just reuse the parts you need and drop the rest:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # reused\n",
    "n_hidden2 = 50  # reused\n",
    "n_hidden3 = 50  # reused\n",
    "n_hidden4 = 20  # new!\n",
    "n_outputs = 10  # new!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")       # reused\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\") # reused\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\") # reused\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\") # new!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\")                         # new!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "However, you must create one `Saver` to restore the pretrained model (giving it the list of variables to restore, or else it will complain that the graphs don't match), and another `Saver` to save the new model, once it is trained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.8991\n",
      "1 Test accuracy: 0.9247\n",
      "2 Test accuracy: 0.9361\n",
      "3 Test accuracy: 0.943\n",
      "4 Test accuracy: 0.9465\n",
      "5 Test accuracy: 0.9498\n",
      "6 Test accuracy: 0.9532\n",
      "7 Test accuracy: 0.9548\n",
      "8 Test accuracy: 0.9559\n",
      "9 Test accuracy: 0.9574\n",
      "10 Test accuracy: 0.9583\n",
      "11 Test accuracy: 0.9594\n",
      "12 Test accuracy: 0.9603\n",
      "13 Test accuracy: 0.9601\n",
      "14 Test accuracy: 0.9608\n",
      "15 Test accuracy: 0.9621\n",
      "16 Test accuracy: 0.9632\n",
      "17 Test accuracy: 0.9641\n",
      "18 Test accuracy: 0.9647\n",
      "19 Test accuracy: 0.9656\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # regular expression\n",
    "reuse_vars_dict = dict([(var.op.name, var) for var in reuse_vars])\n",
    "restore_saver = tf.train.Saver(reuse_vars_dict) # to restore layers 1-3\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):                                      # not shown in the book\n",
    "        for iteration in range(mnist.train.num_examples // batch_size): # not shown\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)      # not shown\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})  # not shown\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,  # not shown\n",
    "                                                y: mnist.test.labels}) # not shown\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)                   # not shown\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Reusing Models from Other Frameworks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "In this example, for each variable we want to reuse, we find its initializer's assignment operation, and we get its second input, which corresponds to the initialization value. When we run the initializer, we replace the initialization values with the ones we want, using a `feed_dict`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_hidden1 = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  61.   83.  105.]]\n"
     ]
    }
   ],
   "source": [
    "original_w = [[1., 2., 3.], [4., 5., 6.]] # Load the weights from the other framework\n",
    "original_b = [7., 8., 9.]                 # Load the biases from the other framework\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "# [...] Build the rest of the model\n",
    "\n",
    "# Get a handle on the assignment nodes for the hidden1 variables\n",
    "graph = tf.get_default_graph()\n",
    "assign_kernel = graph.get_operation_by_name(\"hidden1/kernel/Assign\")\n",
    "assign_bias = graph.get_operation_by_name(\"hidden1/bias/Assign\")\n",
    "init_kernel = assign_kernel.inputs[1]\n",
    "init_bias = assign_bias.inputs[1]\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init, feed_dict={init_kernel: original_w, init_bias: original_b})\n",
    "    # [...] Train the model on your new task\n",
    "    print(hidden1.eval(feed_dict={X: [[10.0, 11.0]]}))  # not shown in the book"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the weights variable created by the `tf.layers.dense()` function is called `\"kernel\"` (instead of `\"weights\"` when using the `tf.contrib.layers.fully_connected()`, as in the book), and the biases variable is called `bias` instead of `biases`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Another approach (initially used in the book) would be to create dedicated assignment nodes and dedicated placeholders. This is more verbose and less efficient, but you may find this more explicit:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  61.   83.  105.]]\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 2\n",
    "n_hidden1 = 3\n",
    "\n",
    "original_w = [[1., 2., 3.], [4., 5., 6.]] # Load the weights from the other framework\n",
    "original_b = [7., 8., 9.]                 # Load the biases from the other framework\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "# [...] Build the rest of the model\n",
    "\n",
    "# Get a handle on the variables of layer hidden1\n",
    "with tf.variable_scope(\"\", default_name=\"\", reuse=True):  # root scope\n",
    "    hidden1_weights = tf.get_variable(\"hidden1/kernel\")\n",
    "    hidden1_biases = tf.get_variable(\"hidden1/bias\")\n",
    "\n",
    "# Create dedicated placeholders and assignment nodes\n",
    "original_weights = tf.placeholder(tf.float32, shape=(n_inputs, n_hidden1))\n",
    "original_biases = tf.placeholder(tf.float32, shape=n_hidden1)\n",
    "assign_hidden1_weights = tf.assign(hidden1_weights, original_weights)\n",
    "assign_hidden1_biases = tf.assign(hidden1_biases, original_biases)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    sess.run(assign_hidden1_weights, feed_dict={original_weights: original_w})\n",
    "    sess.run(assign_hidden1_biases, feed_dict={original_biases: original_b})\n",
    "    # [...] Train the model on your new task\n",
    "    print(hidden1.eval(feed_dict={X: [[10.0, 11.0]]}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note that we could also get a handle on the variables using `get_collection()` and specifying the `scope`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'hidden1/kernel:0' shape=(2, 3) dtype=float32_ref>,\n",
       " <tf.Variable 'hidden1/bias:0' shape=(3,) dtype=float32_ref>]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=\"hidden1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Or we could use the graph's `get_tensor_by_name()` method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'hidden1/kernel:0' shape=(2, 3) dtype=float32_ref>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'hidden1/bias:0' shape=(3,) dtype=float32_ref>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_default_graph().get_tensor_by_name(\"hidden1/bias:0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Freezing the Lower Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # reused\n",
    "n_hidden2 = 50  # reused\n",
    "n_hidden3 = 50  # reused\n",
    "n_hidden4 = 20  # new!\n",
    "n_outputs = 10  # new!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")       # reused\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\") # reused\n",
    "    hidden3 = tf.layers.dense(hidden2, n_hidden3, activation=tf.nn.relu, name=\"hidden3\") # reused\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu, name=\"hidden4\") # new!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\")                         # new!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\"):                                         # not shown in the book\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)     # not shown\n",
    "    train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                                   scope=\"hidden[34]|outputs\")\n",
    "    training_op = optimizer.minimize(loss, var_list=train_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "new_saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.8624\n",
      "1 Test accuracy: 0.8987\n",
      "2 Test accuracy: 0.9142\n",
      "3 Test accuracy: 0.9245\n",
      "4 Test accuracy: 0.9302\n",
      "5 Test accuracy: 0.9341\n",
      "6 Test accuracy: 0.9371\n",
      "7 Test accuracy: 0.9408\n",
      "8 Test accuracy: 0.9434\n",
      "9 Test accuracy: 0.9442\n",
      "10 Test accuracy: 0.945\n",
      "11 Test accuracy: 0.9463\n",
      "12 Test accuracy: 0.9474\n",
      "13 Test accuracy: 0.9494\n",
      "14 Test accuracy: 0.9492\n",
      "15 Test accuracy: 0.9493\n",
      "16 Test accuracy: 0.9513\n",
      "17 Test accuracy: 0.9506\n",
      "18 Test accuracy: 0.9529\n",
      "19 Test accuracy: 0.952\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # regular expression\n",
    "reuse_vars_dict = dict([(var.op.name, var) for var in reuse_vars])\n",
    "restore_saver = tf.train.Saver(reuse_vars_dict) # to restore layers 1-3\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # reused\n",
    "n_hidden2 = 50  # reused\n",
    "n_hidden3 = 50  # reused\n",
    "n_hidden4 = 20  # new!\n",
    "n_outputs = 10  # new!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\") # reused frozen\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\") # reused frozen\n",
    "    hidden2_stop = tf.stop_gradient(hidden2)\n",
    "    hidden3 = tf.layers.dense(hidden2_stop, n_hidden3, activation=tf.nn.relu,\n",
    "                              name=\"hidden3\") # reused, not frozen\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu,\n",
    "                              name=\"hidden4\") # new!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\") # new!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The training code is exactly the same as earlier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.8552\n",
      "1 Test accuracy: 0.9137\n",
      "2 Test accuracy: 0.9249\n",
      "3 Test accuracy: 0.9312\n",
      "4 Test accuracy: 0.9362\n",
      "5 Test accuracy: 0.9401\n",
      "6 Test accuracy: 0.9416\n",
      "7 Test accuracy: 0.9448\n",
      "8 Test accuracy: 0.945\n",
      "9 Test accuracy: 0.9474\n",
      "10 Test accuracy: 0.9472\n",
      "11 Test accuracy: 0.9481\n",
      "12 Test accuracy: 0.9478\n",
      "13 Test accuracy: 0.95\n",
      "14 Test accuracy: 0.9507\n",
      "15 Test accuracy: 0.9515\n",
      "16 Test accuracy: 0.9513\n",
      "17 Test accuracy: 0.9516\n",
      "18 Test accuracy: 0.9523\n",
      "19 Test accuracy: 0.9525\n"
     ]
    }
   ],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # regular expression\n",
    "reuse_vars_dict = dict([(var.op.name, var) for var in reuse_vars])\n",
    "restore_saver = tf.train.Saver(reuse_vars_dict) # to restore layers 1-3\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Caching the Frozen Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300 # reused\n",
    "n_hidden2 = 50  # reused\n",
    "n_hidden3 = 50  # reused\n",
    "n_hidden4 = 20  # new!\n",
    "n_outputs = 10  # new!\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\") # reused frozen\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\") # reused frozen & cached\n",
    "    hidden2_stop = tf.stop_gradient(hidden2)\n",
    "    hidden3 = tf.layers.dense(hidden2_stop, n_hidden3, activation=tf.nn.relu,\n",
    "                              name=\"hidden3\") # reused, not frozen\n",
    "    hidden4 = tf.layers.dense(hidden3, n_hidden4, activation=tf.nn.relu,\n",
    "                              name=\"hidden4\") # new!\n",
    "    logits = tf.layers.dense(hidden4, n_outputs, name=\"outputs\") # new!\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reuse_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,\n",
    "                               scope=\"hidden[123]\") # regular expression\n",
    "reuse_vars_dict = dict([(var.op.name, var) for var in reuse_vars])\n",
    "restore_saver = tf.train.Saver(reuse_vars_dict) # to restore layers 1-3\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n",
      "0 Test accuracy: 0.8589\n",
      "1 Test accuracy: 0.9162\n",
      "2 Test accuracy: 0.9292\n",
      "3 Test accuracy: 0.9358\n",
      "4 Test accuracy: 0.9408\n",
      "5 Test accuracy: 0.9424\n",
      "6 Test accuracy: 0.9442\n",
      "7 Test accuracy: 0.9458\n",
      "8 Test accuracy: 0.9484\n",
      "9 Test accuracy: 0.9493\n",
      "10 Test accuracy: 0.9495\n",
      "11 Test accuracy: 0.9498\n",
      "12 Test accuracy: 0.9514\n",
      "13 Test accuracy: 0.9521\n",
      "14 Test accuracy: 0.9526\n",
      "15 Test accuracy: 0.9526\n",
      "16 Test accuracy: 0.9535\n",
      "17 Test accuracy: 0.9522\n",
      "18 Test accuracy: 0.9527\n",
      "19 Test accuracy: 0.9531\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_batches = mnist.train.num_examples // batch_size\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    restore_saver.restore(sess, \"./my_model_final.ckpt\")\n",
    "    \n",
    "    h2_cache = sess.run(hidden2, feed_dict={X: mnist.train.images})\n",
    "    h2_cache_test = sess.run(hidden2, feed_dict={X: mnist.test.images}) # not shown in the book\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        shuffled_idx = np.random.permutation(mnist.train.num_examples)\n",
    "        hidden2_batches = np.array_split(h2_cache[shuffled_idx], n_batches)\n",
    "        y_batches = np.array_split(mnist.train.labels[shuffled_idx], n_batches)\n",
    "        for hidden2_batch, y_batch in zip(hidden2_batches, y_batches):\n",
    "            sess.run(training_op, feed_dict={hidden2:hidden2_batch, y:y_batch})\n",
    "\n",
    "        accuracy_val = accuracy.eval(feed_dict={hidden2: h2_cache_test, # not shown\n",
    "                                                y: mnist.test.labels})  # not shown\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)                    # not shown\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_new_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Faster Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Momentum optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Nesterov Accelerated Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=0.9, use_nesterov=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## AdaGrad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,\n",
    "                                      momentum=0.9, decay=0.9, epsilon=1e-10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Adam Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Learning Rate Scheduling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\"):       # not shown in the book\n",
    "    initial_learning_rate = 0.1\n",
    "    decay_steps = 10000\n",
    "    decay_rate = 1/10\n",
    "    global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "    learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n",
    "                                               decay_steps, decay_rate)\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)\n",
    "    training_op = optimizer.minimize(loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.9661\n",
      "1 Test accuracy: 0.972\n",
      "2 Test accuracy: 0.9747\n",
      "3 Test accuracy: 0.9775\n",
      "4 Test accuracy: 0.9801\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Avoiding Overfitting Through Regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## $\\ell_1$ and $\\ell_2$ regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's implement $\\ell_1$ regularization manually. First, we create the model, as usual (with just one hidden layer this time, for simplicity):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    logits = tf.layers.dense(hidden1, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next, we get a handle on the layer weights, and we compute the total loss, which is equal to the sum of the usual cross entropy loss and the $\\ell_1$ loss (i.e., the absolute values of the weights):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "W1 = tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")\n",
    "W2 = tf.get_default_graph().get_tensor_by_name(\"outputs/kernel:0\")\n",
    "\n",
    "scale = 0.001 # l1 regularization hyperparameter\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"avg_xentropy\")\n",
    "    reg_losses = tf.reduce_sum(tf.abs(W1)) + tf.reduce_sum(tf.abs(W2))\n",
    "    loss = tf.add(base_loss, scale * reg_losses, name=\"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The rest is just as usual:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.8227\n",
      "1 Test accuracy: 0.8634\n",
      "2 Test accuracy: 0.8825\n",
      "3 Test accuracy: 0.8904\n",
      "4 Test accuracy: 0.8943\n",
      "5 Test accuracy: 0.8983\n",
      "6 Test accuracy: 0.9001\n",
      "7 Test accuracy: 0.9026\n",
      "8 Test accuracy: 0.904\n",
      "9 Test accuracy: 0.9052\n",
      "10 Test accuracy: 0.9067\n",
      "11 Test accuracy: 0.9077\n",
      "12 Test accuracy: 0.9075\n",
      "13 Test accuracy: 0.9081\n",
      "14 Test accuracy: 0.9087\n",
      "15 Test accuracy: 0.9078\n",
      "16 Test accuracy: 0.9072\n",
      "17 Test accuracy: 0.9077\n",
      "18 Test accuracy: 0.907\n",
      "19 Test accuracy: 0.9062\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Alternatively, we can pass a regularization function to the `tf.layers.dense()` function, which will use it to create operations that will compute the regularization loss, and it adds these operations to the collection of regularization losses. The beginning is the same as above:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next, we will use Python's `partial()` function to avoid repeating the same arguments over and over again. Note that we set the `kernel_regularizer` argument:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "scale = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "my_dense_layer = partial(\n",
    "    tf.layers.dense, activation=tf.nn.relu,\n",
    "    kernel_regularizer=tf.contrib.layers.l1_regularizer(scale))\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    hidden2 = my_dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = my_dense_layer(hidden2, n_outputs, activation=None,\n",
    "                            name=\"outputs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next we must add the regularization losses to the base loss:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):                                     # not shown in the book\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(  # not shown\n",
    "        labels=y, logits=logits)                                # not shown\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"avg_xentropy\")   # not shown\n",
    "    reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "    loss = tf.add_n([base_loss] + reg_losses, name=\"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "And the rest is the same as usual:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.7951\n",
      "1 Test accuracy: 0.8673\n",
      "2 Test accuracy: 0.8895\n",
      "3 Test accuracy: 0.8974\n",
      "4 Test accuracy: 0.9039\n",
      "5 Test accuracy: 0.9078\n",
      "6 Test accuracy: 0.9096\n",
      "7 Test accuracy: 0.9119\n",
      "8 Test accuracy: 0.9117\n",
      "9 Test accuracy: 0.9149\n",
      "10 Test accuracy: 0.9159\n",
      "11 Test accuracy: 0.9167\n",
      "12 Test accuracy: 0.9182\n",
      "13 Test accuracy: 0.9177\n",
      "14 Test accuracy: 0.9187\n",
      "15 Test accuracy: 0.918\n",
      "16 Test accuracy: 0.9175\n",
      "17 Test accuracy: 0.9162\n",
      "18 Test accuracy: 0.9167\n",
      "19 Test accuracy: 0.9158\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                                y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tf.contrib.layers.dropout()` rather than `tf.layers.dropout()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dropout()`, because anything in the contrib module may change or be deleted without notice. The `tf.layers.dropout()` function is almost identical to the `tf.contrib.layers.dropout()` function, except for a few minor differences. Most importantly:\n",
    "* you must specify the dropout rate (`rate`) rather than the keep probability (`keep_prob`), where `rate` is simply equal to `1 - keep_prob`,\n",
    "* the `is_training` parameter is renamed to `training`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "dropout_rate = 0.5  # == 1 - keep_prob\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=training)\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X_drop, n_hidden1, activation=tf.nn.relu,\n",
    "                              name=\"hidden1\")\n",
    "    hidden1_drop = tf.layers.dropout(hidden1, dropout_rate, training=training)\n",
    "    hidden2 = tf.layers.dense(hidden1_drop, n_hidden2, activation=tf.nn.relu,\n",
    "                              name=\"hidden2\")\n",
    "    hidden2_drop = tf.layers.dropout(hidden2, dropout_rate, training=training)\n",
    "    logits = tf.layers.dense(hidden2_drop, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.924\n",
      "1 Test accuracy: 0.9413\n",
      "2 Test accuracy: 0.9501\n",
      "3 Test accuracy: 0.9556\n",
      "4 Test accuracy: 0.9575\n",
      "5 Test accuracy: 0.9576\n",
      "6 Test accuracy: 0.9612\n",
      "7 Test accuracy: 0.9634\n",
      "8 Test accuracy: 0.9655\n",
      "9 Test accuracy: 0.962\n",
      "10 Test accuracy: 0.9638\n",
      "11 Test accuracy: 0.9653\n",
      "12 Test accuracy: 0.9696\n",
      "13 Test accuracy: 0.9657\n",
      "14 Test accuracy: 0.9682\n",
      "15 Test accuracy: 0.9662\n",
      "16 Test accuracy: 0.9676\n",
      "17 Test accuracy: 0.968\n",
      "18 Test accuracy: 0.9687\n",
      "19 Test accuracy: 0.9681\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={training: True, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Max norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's go back to a plain and simple neural net for MNIST with just 2 hidden layers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next, let's get a handle on the first hidden layer's weight and create an operation that will compute the clipped weights using the `clip_by_norm()` function. Then we create an assignment operation to assign the clipped weights to the weights variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "threshold = 1.0\n",
    "weights = tf.get_default_graph().get_tensor_by_name(\"hidden1/kernel:0\")\n",
    "clipped_weights = tf.clip_by_norm(weights, clip_norm=threshold, axes=1)\n",
    "clip_weights = tf.assign(weights, clipped_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We can do this as well for the second hidden layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "weights2 = tf.get_default_graph().get_tensor_by_name(\"hidden2/kernel:0\")\n",
    "clipped_weights2 = tf.clip_by_norm(weights2, clip_norm=threshold, axes=1)\n",
    "clip_weights2 = tf.assign(weights2, clipped_weights2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's add an initializer and a saver:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "And now we can train the model. It's pretty much as usual, except that right after running the `training_op`, we run the `clip_weights` and `clip_weights2` operations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.9483\n",
      "1 Test accuracy: 0.9649\n",
      "2 Test accuracy: 0.9695\n",
      "3 Test accuracy: 0.972\n",
      "4 Test accuracy: 0.9759\n",
      "5 Test accuracy: 0.9767\n",
      "6 Test accuracy: 0.9783\n",
      "7 Test accuracy: 0.9779\n",
      "8 Test accuracy: 0.9793\n",
      "9 Test accuracy: 0.9792\n",
      "10 Test accuracy: 0.9802\n",
      "11 Test accuracy: 0.9803\n",
      "12 Test accuracy: 0.9807\n",
      "13 Test accuracy: 0.9819\n",
      "14 Test accuracy: 0.9816\n",
      "15 Test accuracy: 0.9803\n",
      "16 Test accuracy: 0.981\n",
      "17 Test accuracy: 0.9807\n",
      "18 Test accuracy: 0.9811\n",
      "19 Test accuracy: 0.9807\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:                                              # not shown in the book\n",
    "    init.run()                                                          # not shown\n",
    "    for epoch in range(n_epochs):                                       # not shown\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):  # not shown\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)       # not shown\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "            clip_weights.eval()\n",
    "            clip_weights2.eval()                                        # not shown\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images,       # not shown\n",
    "                                            y: mnist.test.labels})      # not shown\n",
    "        print(epoch, \"Test accuracy:\", acc_test)                        # not shown\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")               # not shown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The implementation above is straightforward and it works fine, but it is a bit messy. A better approach is to define a `max_norm_regularizer()` function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\",\n",
    "                         collection=\"max_norm\"):\n",
    "    def max_norm(weights):\n",
    "        clipped = tf.clip_by_norm(weights, clip_norm=threshold, axes=axes)\n",
    "        clip_weights = tf.assign(weights, clipped, name=name)\n",
    "        tf.add_to_collection(collection, clip_weights)\n",
    "        return None # there is no regularization loss term\n",
    "    return max_norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Then you can call this function to get a max norm regularizer (with the threshold you want). When you create a hidden layer, you can pass this regularizer to the `kernel_regularizer` argument:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 50\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "momentum = 0.9\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "max_norm_reg = max_norm_regularizer(threshold=1.0)\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu,\n",
    "                              kernel_regularizer=max_norm_reg, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=tf.nn.relu,\n",
    "                              kernel_regularizer=max_norm_reg, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Training is as usual, except you must run the weights clipping operations after each training operation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Test accuracy: 0.9525\n",
      "1 Test accuracy: 0.9661\n",
      "2 Test accuracy: 0.9713\n",
      "3 Test accuracy: 0.9761\n",
      "4 Test accuracy: 0.9761\n",
      "5 Test accuracy: 0.9769\n",
      "6 Test accuracy: 0.9785\n",
      "7 Test accuracy: 0.9804\n",
      "8 Test accuracy: 0.9776\n",
      "9 Test accuracy: 0.9796\n",
      "10 Test accuracy: 0.9792\n",
      "11 Test accuracy: 0.9776\n",
      "12 Test accuracy: 0.98\n",
      "13 Test accuracy: 0.9791\n",
      "14 Test accuracy: 0.98\n",
      "15 Test accuracy: 0.9802\n",
      "16 Test accuracy: 0.9802\n",
      "17 Test accuracy: 0.9799\n",
      "18 Test accuracy: 0.9808\n",
      "19 Test accuracy: 0.98\n"
     ]
    }
   ],
   "source": [
    "clip_all_weights = tf.get_collection(\"max_norm\")\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "            sess.run(clip_all_weights)\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images,     # not shown in the book\n",
    "                                            y: mnist.test.labels})    # not shown\n",
    "        print(epoch, \"Test accuracy:\", acc_test)                      # not shown\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")             # not shown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Coming soon**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}