11_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 11 – Deep Learning**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 11._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sigmoid_saturation_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGFCAYAAAALnnwfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xd4FFX3wPHv2SSUBBQRlCJFUQQpUkRBmqKggEQQBUGl\n2Sk/UYq+6itRUQRfK0VFUVAkIKiIoAiKNLuJBVQEkQ5Spddk7++Puxs2m002IcnObvZ8nmefZO+0\nM3t3Jid37twRYwxKKaWUUio4l9MBKKWUUkpFCk2clFJKKaVySRMnpZRSSqlc0sRJKaWUUiqXNHFS\nSimllMolTZyUUkoppXJJEyellFJKqVzSxEkppZRSKpc0cVJKKaWUyiVNnJQKQEQWi4jb6TjyQ0Tc\nIrIoD/MneZZpVZhx5ZeITPbEWdXpWABEpJonnjedjsVLRGI99blaRI6KSLqIJDodV16ISGvP5/qY\n07Eo5UsTJxUVRKSkiDwsIikickBEjojIJhFZKiJPi8i5fosYIKITJ+w+5OWZSnmdv1CISG/PH8xe\n2cwS8jhFZL2I/J3DLGHx2fkYCjwGbAGeBR4HVjkaUQC5SO7D7XNVilinA1CqsIlIKeAroB6wBngH\n2A2UAy4FHgT+Atb5LHYbEB/aSAtcbeCw00Gcopz+WD4EjMImBaGSUzxbsJ/1vhDFkhsdgQPA1caY\ndKeDOUXfYT/XXU4HopQvTZxUNLgfmzRNNMbc4z9RRKoBxX3LjDGbQxRboTHGrHY6hlMkOU00xmwH\ntocolqCMMWlAuH3WlYDdEZw0YYw5Svh9rkrppToVFZpiWwwmBJpojNngn2Rk18fJc8lvjIhs9Fzu\nWyEid2TXH8N7KUJEKonINBHZKSL7RWSu9/KgiNQWkdkistszbaaInBUoVhHpJCJfisheETksIj+L\nyP0iEhNg3oCXQUTkHBFJ9mzvgGdfW+b0AWYTSxfPPq0RkUOemJaKyA05LFNfRN71XCY9KiJbReRT\nEbnOM/0twNtXyNuXyS0i6T7ryNTHSURaet6/ns02y4vICRFZ5lPWSETGeerP+1n+KiIPikisz3zV\nPN+DqkB1n3gy6jqnPk4iUlVEJonIZhE55tnvN0SkSoB5F3v6Inn7J63zfEZ/isi9OddGxjpGeOKt\n7hfv357pfbK7DJqL7/BZIjLF8x0+LCLfiEjrbOIo5YnlF5/vRqqIPCEiMd5tYY/LK/w+1145xeOZ\nVkdE3hOR7Z7P6G8ReUFEygaYd71neoKIvCQiWzzL/CIiXXPzuSrlS1ucVDTY7fl5AfBrLpfJ0rdC\nRFzAPOAKz3reBcoC/wOW+M/v4wxgObANmAzUBDoBF4pIZ2AZ8CMwCWgMdPUsc7Xf9h/wbGu3Z9uH\ngETgOaCFZ7kciUgF4FugIjAf+Al7OWQh8GWw5f08DRzzxL8NKO+JZ5aIDDLGjPfbdldP3AAfA38C\nZwGXAf2AucCHwOnA9cBs4GfP/L6fbaa6McYsE5H1wI0iMsAYc9wvzp7YfxLf9im7E7gOWIqt03hs\nvY4CLgFu8sy3F0jCtloa4AVOtogtzvaTsft7AfYS8ZnAHOB3oK5nX68TkRbGmL/89gsgGWgCfAqk\nA92A8SJy3BgzKadtYuvQBIj3X59tnEqfoTLY7/Be7Od4FnAzMF9EGhtjfvfZ7/LYz7Umtv4mYD//\nWsBw7Hd4PfZzTfL8PtlnWz+TAxFpAXyG/fs1E9gANAPuAzqKSFNjzB6fRQwQByzw7McsbH3fDMwQ\nkWuNMZ/n/qNQUc8Yoy99FekX9g+kG9sH5VmgLVA2yDJfAul+Zbd71vMxID7ltbB9idKBx/yWcXvK\nn/UrH++ZtgcY6DdtrmeZBj5l5wHHga1AJZ/yOOwfqXTglgDbXuRXNtkz70N+5Xf4xNoql59r9QBl\n8cAvnv0q4VN+FnAQ2A/UD7Cc7z719sTRK5vtvuWZXtWn7AlP2Y0B5v8ROAKU8Smr4luHPuVveNbT\nzK98HfB3NvFU83x2b/qVL/Ks63a/8ns88y8M8J1zA18DCT7lNT11/3sevvMB483pswVae7af3Xf4\nZb/yfp5pE/zKZ3nmfyLANsoDrpy+oznFg00C//Ks/2q/+Ud75n89wGeRDrwPxPqUt/HM/0luP1d9\n6csYo5fqVNFnjJkLPOB5+wD2v9VdnktMY0Xk/Fyu6lbsf6+PGGN8WzxWkbk1w99B4L9+Zcmen7uM\nMeP8pk33/LzYp+wWIAZ4zhiz1WfbJ7Cd2wXok1PwIhKHbb3YATzvN3kSeexPYoxZH6DsMDY5Ox3b\nauLVBygJ/M8Yk6XVz3efTtE72M/gVt9CEbkQaATMM8bs9dneJt869DHBs56rA0zLNc+luCuwyY5/\nK9Fr2Dvc2ohIZb9pBpvUHvKJdTW25epCEUnIT1z5cAjbKd/XFCANn3oWkbOBLsBa7J18mRhjdhpj\n8nO3anPsPxGfmKytRE9gE/aevpdbfdxvbH80byyLsK1VTQLMq1S2NHFSUcEY8yK2w2w37OWLZdhW\nhwHAr94+NkHUBw4H+sOP/cOWXafmNcZ2dPW1zfMz0Lq2edZVyaesgefnEv+ZjTHfAEd95snOhUAJ\n4EfjdznLk0R8HWT5TDx9h54Xkd89/Vjcnn4rz3lm8Y3f+8dpYV62kVvGmDXA98C1fv1cemGTkXf8\nYo8TkQdE5DsR2efpW+TGtk4Zv9hPRU71ZbCthL7z+UoNUOa9WaFMPuM6Vas9SXEGYzuebydzTJdg\nv7tfmsLpmN7Q8zPQ53oIW38lsN91X3uNMRsDrG8zzn2mKkJpHycVNTwn1vc9L0SkNLafzgBgkohU\n9v2PNIDTgEAnX8j5Lq/9AcrScjEtzm/bOW1nO8H/2J/u+bkjh3Xkioicgf0jdQ42aVyI7f+Sjk0G\nrifznYrebRfmEALvYIeX6A684inrie3f84nfvO9jL+H+iW3h2wGcwP4RHYzfXZanIFh9bfObL4Mx\n5mCA+b3fiSw3AYRIoO8p2Lh8Yyrsej4Nm9jm9XPNbqiINLQBQeWRJk4qahljDgCDPK1NVbFDFvyU\nwyL7sX00Ajm7gMMLtG3vdjYFmH4W2f9x8/L+8Qh4xx5524c7sEnTo8aYUb4TRORBbOLky3uZrDLZ\nJ5/5NR17CfJW4BXPHV/VgFc8lzS98V2CTZo+Ba7zvWQnIpdhE6f88q2vQCr4zRcqbmyLUKBz/+kB\nyvLKt54Lw35s/OH2uaooopm2Urb/Rm78AiSISP0A05pTuCMc/4T9g3GF/wQRaYrtP5RT0ge2deUo\ncImIFPNbhwCX5yGe8zw/5wSYFuiRLd9j42+Xi3Wne+bNU+uKMWY39k7BpiJyHif7pL3rN2sNz89P\nAvRzyu5xM+l5jMd7Z1h262vlN1+oeO+uC5TYNCqA9f+ITc6ulABDZATgJm+fq/c7foX/BBGJx14q\nPIL9ritVKDRxUkWeiNzlaWUINK0z9nb8f4GVQVb1LvYP+khPouFdRy1sX5rCNA17WeEBEanos+04\n7N1Ehsy3dGfhaXV5D9viNMRv8p3Yu7dyawP2s2jhWygiPYH2Aeafgu0kP0RELvafKCK+lxm9t5Jn\nGesoF7x9me4AbgTWGWP8+25t8Pz0j70OtgN0oAR4D1DOP+HMjjFmE/YuuToi0s9vO3djv3NfGGNC\nOfo5QAp2/24WkYzLkZ6hE/6PfCb/xpgd2MugNbBDDWTi6RfnmyjtwbZc5tZX2I7n7UXkKr9p/8UO\n/TAtyCV3pfJFL9WpaNAeeFVE/sKeeLcCCdiOpi2xrQn9fS/nZOMt7KNYOgI/icin2BN1d+wYMZ0o\npOfbGWP+9lwC+x+2M/t72JayTtiEZ7YxZlouVvUQcBU2+WvJyXGc2mPvNsxNixDYBOVBYJyItMEm\nI/U9634fvzGljDE7PQMbJgPfi8gcbKtAOew4TusA78CZ32BbDQZ7Onrv9KzjqVzE9TH2Ms0Q7Pnt\nxQDzfO95dfMkbN9iL+l1wg4FcVOAZRZhx9iaL3YgzePAUmPMsgDzet2LvQlhooh04uQ4Tp2wfXT6\n52J/CpQxZpuIJAM9gBQRmY9NpLtgL13eWACb6Q/UAR4WkQ7Yz06wHbavxl5m815KWwTcJCIfYr+L\n6cBHxpiA/8QYY4yI9MG2LH4iIr7jOF2BfaTSfwpgH5TKliZOKhoMxw7e1xabKHlbbLZgk6FxxphA\nl7ky/fdtjHGLSHvsbdY9sAPurcUONrgXO/ijf9+KnAYczNM0Y8wLIrIGO6TCLUAx7BACDwBjc7mO\nf0SkGTAGuAb7efyI/WyuIpeJkzFmi4i08qznKuy5JNWznmqcTIJ8l5nt6UP0H+wYPZ2wzyH7GXjd\nZ75/PYNlJmFbjkp69sM3cQr4uRljjnn+mN5O4Mt03nrsCDwDXIu9vOP9XD/DJg/+638S23H8Ouxn\n5sJ+D7yJU6DPerWnpXOEZzsdsEngJOwYR4H6quXU4pPX1qDs5r/dE0d3bJLzJ/Zz/geb8PovF2zQ\nTP/93u25fDwUm4QOwF4iXocdYNT30vh9nuXbYD9bF7YPnzdxCvS5fuVZ/2PY79vp2H+GXgRGmsyD\nXwaMMQ/TlMpCAg9lopTKCxEZiU0IOhhjPnM6HqWUUoVDEyel8kBEKhhj/vEruwh7eSkNqBxgzCal\nlFJFhF6qUypvXhGR6tg+Mv9iO8F2wh5L/TRpUkqpok1bnJTKAxHpgX3WWG1s34qD2CTquQCPgFBK\nKVXEaOKklFJKKZVLRfpSnYicib1zaD32rg6llFJKKX8lgOrAZ57BdLNVpBMnbNKU5XZkpZRSSqkA\nbsEOOJytop44rQeYOnUqtWvXdjiUgtG2bVsWLiyUB8yrAqJ1FN7++OMPbr311iJ1XiiKIuE4OnDs\nAKWKlcLnQQJRJRLqKLe85wU8eUNOinridBSgdu3aNGpUEI9hcl7dunWLzL4UVVpHkaEonReKIj2O\nwl8RraOg3Xr0WXUR5vTTC+IB5qowaR0plX96HIW/aK0jTZyUUkoppXJJEyellFJKqVzSxCnCVKhQ\nwekQVBBaR0rlX7gcR0fTjrJ843KnwwhL4VJHoaaJU4SJi4tzOgQVhNaRUvkXDsdRujudnu/3pOO0\njuw5ssfpcMJOONSRE4r6XXVFzvjx450OQQWhdaRU/jl9HBlj6D+vP3P+nMPsm2dTtmRZR+MJR07X\nkVM0cVJKKaX8PL7kcSamTuSt69/iuprXOR2OCiN6qU4ppZTy8coPr/D4ksd55qpn6NOgj9PhqDCj\niZNSSinlMev3WQz4ZACDLxvM8ObDnQ5HhSFNnCJMYmKi0yGoILSOlMo/J46jtXvWcssHt3Bz3Zt5\n7prnovZRKrkVrec67eMUYQYOHOh0CCoIrSOl8s+J46hG2RpM7TKV62tdj0u0XSGYaD3XaeIUYdq1\na+d0CCoIrSOl8s+p4+imOjc5st1IFK3nOk2plVJKKaVySRMnpZRSSqlc0sQpwsyePdvpEFQQWkdK\n5Z8eR+EvWutIE6cIk5yc7HQIKgitI6XyrzCPI7dxczz9eKGtP1pE67lOE6cIM2PGDKdDUEFoHSmV\nf4V1HBljeOCzB7h++vW4jbtQthEtovVcp4mTUkqpqDHmqzG89N1LXH+hDjmgTo1+a5RSSkWFt356\ni4e+eIjHWj3GPZfc43Q4KkJp4qSUUqrIm7t6Lnd+fCd3NbqLpCuSnA5HRTBNnCJM3759nQ5BBaF1\npFT+FeRx9PWmr+k2sxuJFyYyoeMEfZRKAYnWc50mThEmWkdqjSRaR0rlX0EdR7/v/J3rpl1Hk8pN\nmNZ1GjGumAJZr4rec50mThGmR48eToeggtA6Uir/Cuo4inXF0qJqCz66+SNKxJYokHUqK1rPdfqs\nOqWUUkVWzTNrMqfHHKfDUEWItjgppZRSSuWSJk4RZvny5U6HoILQOlIq//Q4Cn/RWkeaOEWYMWPG\nOB2CCkLrSKn80+Mo/EVrHWniFGGmT5/udAgqCK0jpfJPj6PwF611pIlThImPj3c6BBWE1pFS+ZfX\n4+iVH17hy3VfFlI0KpBoPddp4qSUUiqiTV85nf6f9GfB2gVOh6KigCZOSimlItbCtQvp9WEvbqt/\nG09d9ZTT4agooIlThBk2bJjTIaggtI6Uyr/cHEc/bv2RG967gavPu5pJiZNwif5JC6VoPdfptyzC\nVK1a1ekQVBBaR0rlX7DjaM3uNXR4twMXlb+ImTfNJC4mLkSRKa9oPddp4hRhBg0a5HQIKgitI6Xy\nL6fj6J+D/3DN1GsoW7Is83rOI6FYQggjU17Req7TR64opZSKKFN+nsKx9GN82ftLysWXczocFWW0\nxUkppVREGd58OKl3pVKtTDWnQ1FRKGSJk4gkiMjjIvKpiOwWEbeI9MrD8qeLyEQR2SEiB0VkkYg0\nLMyYw9GqVaucDkEFoXWkVP7ldByJCGeXOjuE0ahAovVcF8oWp3LAf4FawM+Aye2CIiLAJ8DNwMvA\nMKA8sFhEahR8qOFr+PDhToeggtA6Uir/9DgKf9FaR6Hs47QVqGCM2SEijYEf8rDsTUAzoKsx5kMA\nEZkJrAYeB24t6GDD1bhx45wOQQWhdaRU/ulxFP6itY5C1uJkjDlhjNlxiot3Bf7xJk2e9e0C3gOu\nF5GouQ81Wm//jCRaR0rlnx5H4S9a6yhSOoc3BFIDlH8PxAM1QxuOUkqpwpayNYU0d5rTYSiVSaQk\nThWBbQHKvWWVQhiLUkqpQrZ0w1Kav9mcsd+NdToUpTKJlMSpJHAsQPlRQDzTo8Lo0aOdDkEFoXWk\nVP6s2L6Ca+65huZVm9O/SX+nw1HZiNZzXaQMgHkEKB6gvAT27rwjOS38xx9/ZDutRIkSXHTRRTlu\n/Pfff+fo0aPZTq9YsSIVK1bMdvqRI0dyjAGgdu3alCyZff63bds2tm3bxtq1a0lNzXrVMtL2IztF\nYT/Wrl3L77//HvH7AUWjPkD3w1e478f6veu5Zuo1lDKlGHHhCH779beA84X7fnhFen14BdoP379H\nkbAfxsAvv/zBvn3HOXpUOH7cxbFjLo4dE1avXp9jbH4rMiF/AY0BN9Arl/OvBuYGKO8HpAN1slmu\nETaxyvZ10UUXGa/PPvvMdOrUyfg744wzclzHiBEjMuZ97LHHzDPPPJNp+YULF+a4PGBWrlxpjDHm\n0KFDplOnTmbZsmWZ1nHDDTfofuh+6H4UwH6MGzfOACYlJSWi96Oo1Ifvfuw8tNPUHFvTnPfSeeb0\ncqdH7H4UlfoI9X6sWLHSHD5szPr1h8xVV3UyU6YsM998Y8zChcbMnm3MJZcMNlDTwAMG/mtgtIHx\nBi41MNwkJCw3rVoZ06SJMdWqfWbi4zuZypWNKVvWmJIljRExBvobuNNAJ59XKwNlvXE0MkFyEjE2\nwQgpn+EI+hhj3s7F/O8BLYwxlfzKJwI9gLLGmBMBlmsEpEydOpXatWsHXHe0/8fgS/fjJN0Pqyju\nR2pqKo0bNyYlJYVGjRplzBNp+5GdSN2Pg8cPctXbV7F+73q+6vcVx/85HpH74S9S68NfbvajQoWK\n7N8Pu3bBnj2wdy/s22d/7tp1grVrd3HgQAwHD2Z++Za53ZJjHIUnFdumQ2NjTKCb0TKEXeIkIhWA\n04G/jDHpnrJuQDJwkzHmA09ZOWxL1KfGmFuy2U4jIMX/BKmUil7ZJU7KWTfPupl5a+axpM8SGlXU\negkHR4/CP//Ajh02Gdq1C3bvPvm7//vduyEtDG6CFIESJaBkycwv/zLf9/v3pzJ1au4Sp5D2cRKR\nAUAZoLKnKFFEqnh+f9kYcwB4BugFVAc2eqbNAgYDb4lIHWAX0B+IAZJCEnyY2LVrF+XK6UMtw5nW\nkVJ590jLR7i78d0ZSZMeR4Vn3z7YtAm2bbOvf/45+fJ9v3dvsDXtwj4U5NTFxUGZMnD66fZVujSU\nKgUJCfan/+85TUtIsElQsWI2ecqL1FSYOjV384a6c/hQwDtilgG6eF4A7wAHPOVu34WMMW4RaQ88\nCwzC3kX3PbaP1JoQxB02+vXrx5w5c5wOQ+VA60ipvKt3dr1M7/U4OjXHjsHmzbBxo02OAv08cKCg\nttYPmEPJklCuHJx5pv1ZrhyULWsTIm9S5P/T+3uJEnlPcgrKDz/8wPz583n00UfztFxIEydjzLm5\nmKcv0DdA+T7gLs8raiUlJTkdggpC60ip/NPjKHt798LatfDXX/bl+3sOXZlyrVQpqFABKla0P886\nC8qXz5wcnXkmbNuWxBVXQHx8/rcZSm63mxdeeIEHH3yQ9PR0Bg4cmKflI2U4AuWhfTLCn9aRUvkX\n7cfR0aOwejX8/jv88UfmBGn37lNbZ/HiULUqVKliX5Ur28TIN0mqUMEmTrnRsGHk1dGuXbvo1asX\nn376aUaZy5W3IS01cVJKKaUccvSoTY78X2vXgtsdfHlfZ58N555rkyNvguT7s1w55y6LhYOlS5fS\nrVs3du3alalcEyellFJhK92dTowrxukwHLF7N/zyC/z8s3399JNtTUpPz/06qlSB88+HGjXsT+/v\nNWrYjtUqq/T0dJ566imSkpIQEdx+GakmTkXcpEmTuP32250OQ+VA60ipwFbtWkXn6Z2ZfuN0GlRo\nkOO8kX4c7dsHP/wA330H339vk6RNm3K3bHw81KoFF11kX7VrQ82atjUph+GaQi4S6mjbtm3cfPPN\nLFu2zHdw7Ew0cSriUlNTw/6LGu20jpTKasv+LVwz9RpKFytN1dOrBp0/ko6jtDRYudImSd9+a3+u\nWgXBhkmMjbWJUYMGUK/eyUSpalXI499yR4R7Hc2fP5+ePXty4MCBgAmTl+Tx+qUmThFm/PjxToeg\ngtA6Uiqzf4/8y7XvXosxhvm3zqdsybJBlwnn4+j4cduatGQJLF4MX38Nhw7lvEzp0jZBatjQ/mzQ\nwCZJxQM9hTVChGsdnThxgkceeYRnn30Wl8uV5dKcP21xUkopFTaOnDhC4vREth7YyvK+yznntHOc\nDinPjh61rUhLltjXN9/AkRweLR8baxOjyy47+Tr//MhoRYp069evp1u3bvz4448AQZMm0MRJKaVU\nmEhzp9Hj/R6kbE1hUe9F1C4f+Jmh4cYYWLMGPv0U5s+3rUo5PKaNSpWgRQto2tQmSQ0bhldfpGjx\nwQcf0KdPH44cOZLjpTl/mjgppZRynDGG/vP6M3f1XOb0mEPTc5o6HVKODhyARYtsojR/Pqxfn/28\nVatC69YnXzVqRPdt/k47evQoQ4YMYcKECYhInpIm0D5ORV5iYqI+hiDMaR0pBekmnWPpx3jz+jfp\ncEGHPC8fiuNo82aYPRs+/BCWLYMTJwLPV6kStG0LV1xhX9WrF2pYESMcznX79u2jefPm/PHHHwB5\nTppAE6ciL69Dw6vQ0zpSCmJdsUy+fnKe/yh5FdZxtGaNTZQ++MD2WwokLg5atYJrr7WvOnW0RSmQ\ncDjX7du3j+3bt59SwgR5v0wHmjhFnHbt2jkdggpC60gp61STJijY42jlSpg1yyZLK1YEnufcc6FD\nB5soXXFF7h87Es3C4VxXtWpV1q9fzyuvvMJTTz3F/v37c9Uh3OtUvqOaOCmllCpytm6F5GR45x07\nWncg9evDDTfYV9262qoUqRISEhg6dCj33HMPY8eOZdSoURw6dKhQ7qgDTZyUUkoVEQcP2stw77wD\nX3wR+FlvzZpBly72df75oY9RFZ5SpUoxdOhQXnnlFQ4ePJirZU6lxUlHlYgws2fPdjoEFYTWkVL5\nl9vjyO2Gzz+H226zD7nt1QsWLsycNF12Gbz8su0M/vXXMGyYJk0FIRzPdZMnT2bTpk257vN0Ki1O\nmjhFmOTkZKdDUEFoHalosv3gdp5e9jRuk/t+JbkR7DjasQNGj7bPcGvbFqZOhcOHT04/91z473/h\nzz/tY1AGDYLKlQs0xKgXbue6Y8eOZTzI15+3LCYm/w+Y1kt1EWbGjBlOh6CC0DpS0eLAsQN0mNaB\nrQe20q9hPyqUqlBg6w50HBkDX34Jr75qhxHwHz6gTBno1s22PjVvrn2WClu4nevefPNNtm7dGnCa\nMYYxY8awfft2xo0bR1paGunp6drHSSmlVGgcSztGlxld+GvPXyzru6xAkyZ/hw/Du+/CSy/Bb79l\nnX711XDnnXD99ZH97Dd16o4ePcrjjz8ecJqIULVqVe6//35iY2MZOnQoo0aN4pVXXqHkKQzxromT\nUkqpPHEbN71n92b5xuV8dutn1D+7fqFsZ/NmGD8eJk6EPXsyTytfHvr1gzvu0P5KCiZOnMiOHTsC\nTjPG8OSTTxIba1OeChUq8NJLLzF8+HB27tyZ521p4qSUUirXjDEMnj+Y9357j1ndZtG6eusC38bK\nlbb/UnIypKdnnnb55fB//2fviitWrMA3rSLQ4cOHefLJJwN2CHe5XJx77rn07Nkzy7TKlStT+RQ6\nvmnn8AjTt29fp0NQQWgdqaLsmeXPMPb7sUzoOIEbat9QoOv+6ivo1Anq1YOpU/tmJE1xcXDrrfDD\nD3ae7t01aQoH4XKue/XVV9m9e3fAaW63m5EjRxZIp3AvbXGKMOEwUqvKmdaRKqq2HdjGU8ueIql1\nEvdcck+BrNMY+OwzeOopWL7cd0o7ypaFAQPg3nuhYsUC2ZwqQOFwrjt48CAjR47MtrXpggsu4Kab\nbirQbWriFGF69OjhdAgqCK0jVVRVLF2RFfeuoHqZ6vlelzF2/KXHHrPDBfg65xwYOrQHd9wBCQn5\n3pQqJOFwrhs/fjx79+4NOK0wWptAEyellFJ5cO4Z5+Zree+QAo89Zi+7+apdGx58EHr00EtxKrj9\n+/czatSobFubatWqxQ03FOzlZNDESSmlVIh89x089BAsXpy5vE4dGDECunaFUxhWR0WpsWPHcuDA\ngYDT3G43Tz/99CmN0xSMfkUjzPLMnQBUGNI6Uiqzv/6yA1M2bZo5aapVC6ZPh19/hZtuypw06XEU\n/pyso7179zJ69OiAD/J1uVzUr1+fxMTEQtm2Jk4RZsyYMU6HoILQOlLK2rEDBg60l+BmzjxZfv75\ndkDLlSvWSI2pAAAgAElEQVTtHXKBGgX0OAp/TtbRiy++yKFDhwJO87Y2ncoDfHNDL9VFmOnTpzsd\nggpC60hFusMnDrPz0E6qlal2SssfO2ZH+R45EnyvpJx1lr0kd+eddoiBnOhxFP6cqqM9e/bwv//9\nL2BrU0xMDA0aNKBDhw6Ftn1NnCJMfHy80yGoILSOVCQ7kX6CbjO78efuP/m9/+/ExQTJcPzMmweD\nB9vLc14JCTB0KAwZAqVL5249ehyFP6fq6LnnnuPIkSMBp6Wnp/PUU08VWmsTaOKklFLKwxjDnR/f\nyWdrP2Nez3l5Spr+/BPuvx8+/fRkmYhtXXr8cahQeI+yU1Fk165dPP/889m2NjVp0qTQx5fSxEkp\npRQA//niP0z5ZQrv3vAu7Wrk7o/P4cPw5JPwv/9BWtrJ8pYt4eWXoUGDQgpWRaUxY8Zw4sSJgNNC\n0doE2jk84gwbNszpEFQQWkcqEr347YuM/mo0L1zzAj3rZX2uVyDz50PduvDMMyeTpnPOsXfKLVmS\nv6RJj6PwF+o62r59O2PHjiXd/wGG2NamFi1acOWVVxZ6HJo4RZiqVas6HYIKQutIRZrkFcnc/9n9\nPNj8QQY3HRx0/m3b4OaboX17WLfOlhUrBo8+CqtW2Tvl8vtPvx5H4S/UdfTMM8843toEIUycRKSY\niIwWkc0iclhEvhWRq3O57NUiskhEdorIvyLynYjcWtgxh6NBgwY5HYIKQutIRZJvNn1D79m96X1x\nb0ZdNSrHeY2BiRPt8AIzZpwsb90afvnFXrIrqEek6HEU/kJZR1u3bmXChAnZtjZdeeWVtGrVKiSx\nhLLF6W1gMDAV+D8gDfhERC7PaSERSQQ+A+KAEcDDwGHgbRG5r1AjVkqpIq5hxYaMbDOS1zu9nuN/\n6+vXQ9u2cPfdsG+fLTvzTHjrLfsIlVq1QhOvik6jRo0KmDSBbW0aOXJkyGIJSedwEbkU6AYMMca8\n4Cl7B1gJjAFa5LD4AGArcKUxJs2z7ERgFdAHeKnwIldKqaKtRGwJhjcfnu10Y+C112DYMDh48GR5\n7962Q3i5ciEIUkW1TZs28eqrr2bb2tSmTRsuvzzHNpgCFaoWpxuxLUyvewuMMceASUAzEamcw7Kn\nAf96kybPsunALiDwQA5F2KpVq5wOQQWhdaSKivXr4eqr4d57TyZN55xjO4VPnly4SZMeR+EvVHX0\n1FNPBXyQL5zs2xRKoUqcGgCrjTEH/cq/95mencVAHRF5QkRqiMh5IvJfoDG2tSqqDB+e/X+GKjxo\nHalIZ4xNjOrVg0WLTpbfcYd9TMo11xR+DHochb9Q1NHmzZuZNGlSwNam2NhYOnbsSJMmTQo9jkzb\nDdF2KgLbApRvAwSolMOyTwDnAo8Aj3rKDgFdjTEfF2SQkWDcuHFOh6CC0DpSkWzPHtuPadask2VV\nqsAbb0AhjyuYiR5H4S8UdWSM4eyzz2bLli1ZpqWlpfHkk08Wegz+QtXiVBI4FqD8qM/07BwHVgMz\ngZuBW4AfgXc9faeiit6iG/60jlQ4yu5Sh69Fi6B+/cxJU58+sGJFaJMm0OMoEoSijqpUqcLff//N\nq6++SoUKFTJuYIiJieH666+nYcOGhR6Dv1AlTkeA4gHKS/hMz8544DpjzM3GmPeMMclAW2xrVa46\nhnfo0IHExMRMr2bNmjF79uxM8y1YsIDExMQsyw8YMIBJkyZlKktNTSUxMZFdu3ZlKh8xYgSjR4/O\nVLZx40YSExOzXA8eO3ZslgHEDh8+TGJiIsuXL89UnpycTN++fbPE1r17d90P3Q/djzzsxzfffJNl\nvkjcj7zUR0pKCuc2PZfhH2W+tOLdj+PHYfhw259py5aNQCKnnbaKmTPtXXOnnx4e+1FU6kP3I2/7\nUaxYMe6++26eeeYZLr30UsqXL4/b7eaJJ544pf246667MuUDrVu3pm3btlnmzY7k5r+Q/BKRBUAl\nY0xdv/I2wOdAJ2PMvADLxWEvy402xvzXb9qL2Dvu4o0xAUfEEpFGQEpKSgqNGjUqmJ1RSkW01NRU\nGjduTDSdFx5f/DhJS5KYlDiJfg37ZZq2bp0dsPKHH06WtWkDU6bYjuBKhZsjR46wbt06LrroogJb\np/e8ADQ2xqTmNG+oWpx+BmqKSCm/8qaA8UwP5ExsP6yYANPisPFH1ejn/lm8Cj9aRyqcvPrjqyQt\nSeLpNk9nSZpmz4ZGjU4mTXFxdoiBhQudT5r0OAp/TtVRyZIlCzRpyqtQJR2zsAnQXd4CESmGHYfp\nW2PMFk9ZFRG50Ge5HcBeoIuIxPosWwroBPzhGdYgahw+fNjpEFQQWkcqXLz/+/v0n9efQZcO4qEW\nD2WUHz8O998PXbrA3r22rEYN+PZbGDIEXGHw76geR+EvWusoJJfqAERkBtAZeBH4C5s0XQK0McZ8\n5ZlnMdDKGOPyWe5h4Elsq9Tb2ATsduBC4BZjzPQctqmX6pRSmUTLpbrF6xdzzdRr6FyrM8ldk3GJ\nPa2uX28vzX3//cl5b7oJXn/d9mVSKhrl5VJdqIYjALgNmwDdCpwB/Ap09CZNHgZw+y5kjHlaRP4G\n7gMew3Yy/xU7HEHm3mBKKaX45Z9fuH769bSs2pK3O7+dkTQtXGgfzrtnj52vWDF44QU7wGUIno2q\nVJEQssTJGHMceNDzym6eK7Mpnw5k27KklFLqpE37N1H3rLp80P0DiscWxxh49ln4z3/A7fnX9Lzz\nYOZM28dJKZV7YXAlW+WF/22fKvxoHSmnXVfzOpb3Xc5pxU/j4EF7ae7BB08mTR07QkpKeCdNehyF\nv2itI02cIky/fv2Cz6QcpXWkwoGIsGYNNG1qW5a8RoyAOXOgTBnnYssNPY7CX7TWkSZOESYpKcnp\nEFQQWkcqHCxcCJdeCr/9Zt+fdhp89BEkJYXHXXPB6HEUXiZPnozL5eLtt9/OKIvWOoqAw0f5Ksp3\nARUVWkfKaRMmQPv2J4caqF3b3kUXYCDlsOXUcXTkyBGefvppGjduTOnSpSlZsiRVqlShVatWPPzw\nw6xbt+6U1+1yuWjTpk0BRltwNmzYgMvlyrYVSUQyHnfiFa3nulDeVaeUUqoQpaXB4MEwfvzJsk6d\n4N13oXRp5+KKFAcPHqR58+asWLGCCy64gNtuu40zzzyTXbt28f333zN69GjOP/98zj33XKdDDbkb\nbriBZs2aUbFiRadDcZwmTkopFaHe+ukt2tVoR+XTKvPvv9CtG3z++cnpw4fD009DTKBnL6gsXnjh\nBVasWMFdd93Fq6++mmX6hg0bOHasaI65HGxMx9KlS1Nas29AL9VFHP+HLKrwo3WkQmHKz1PoN6cf\nySuTWbsWmjU7mTTFxdmH844eHblJkxPH0bfffouI0L9//4DTq1WrRs2aNTPeL168mNtvv51atWpl\nJBZNmjTh9ddfz7TckiVLcLlciAiLFy/G5XJlvLx9hpKSknC5XCxdujTLdqdMmZKlf5HvpbVVq1bR\npUsXypUrR0xMDBs3bgTgww8/pGfPnlxwwQUkJCRQpkwZWrVqxQcffJBl/eeddx4iktGXyeVyERMT\nkxFPoBgmTZqUcflxx44d9O7dm/LlyxMfH0+zZs1YsmRJwM9xxYoVdOjQgdNOO40yZcrQsWNHfvvt\nN/r06YPL5cqIP1xpi1OESU1N5fbbb3c6DJUDrSNV2Oatnsftc27nzkZ30sI1hGbNYOdOO61cOfjw\nQ2jRwtkY88uJ4+jMM88EYM2aNdSvXz/o/KNHj2bt2rU0bdqUc845h7179zJ//nzuvvtuVq9ezbPP\nPgtA9erVSUpKIikpierVq9OnT5+MdTRo0AAI3IfIV3bT1qxZQ9OmTalfvz59+/Zl9+7dFCtWDICH\nH36Y4sWL07JlSypWrMjOnTuZM2cON954I2PHjmXAgAEANGzYkMGDB/Piiy/SoEEDOnfunLH+6tWr\nZxtDaqodYHvv3r20aNGCMmXK0KtXL3bs2MH06dO59tprSUlJyfRcuV9++YWWLVty5MgRunbtyvnn\nn8+PP/5IixYtuPjii3P8DMKGMabIvoBGgElJSTFKKWWMMSkpKSaSzwvfbPrGlBxZ0nSe3tm8/0Ga\nKVnSGLCv2rWN+ftvpyOMXB9//LEREXPaaaeZoUOHmgULFpjdu3dnO//69euzlKWnp5t27dqZuLg4\ns2nTpkzTRMRceeWVAdeVlJRkXC6XWbJkSZZpkydPNi6Xy0yZMiXTtkXEuFwuk5SUFHCd69aty1J2\n6NAhU79+fXPGGWeYI0eOZFlf3759A64rUAzefXK5XGbQoEGZyidNmmRExNx7772Zylu0aGFcLpeZ\nPn16pvIRI0ZkrGvDhg0BYyhM3vMC0MgEyS30Up1SSkWIP3b+QcdpHbmk0iW0/GcGN3aN4cgRO+2K\nK+CrryAK+y0XmOuuu47nn38egOeff55rrrmGcuXKccEFFzBo0CD++uuvTPNXq1YtyzpcLhf33HMP\n6enpfPnll4Uec4UKFXjkkUcCTvNtLfKKj4+nT58+7Nu3jx9++KFAYkhISOCZZ57JVNa7d29iY2Mz\nbWPjxo189dVX1K9fn+7du2eaf/jw4ZxxxhkFEk9h08RJKaUiwOb9m7lm6jVUSjiHi39dwJDBxfD2\n5+3ZE+bPhwj5uxPWBg8ezNatW3nvvfe4//77admyJZs2bWL8+PHUr1+fuXPnZsx78OBBRowYQYMG\nDShdunRG36CuXbsCsHXr1kKP9+KLLyY2NnCvm507d/LAAw9w0UUXkZCQkBHfkCFDCjS+mjVrEh8f\nn6ksJiaGs88+m73eMTGwl+kAWgS4jhwfH59x2TLcaR8npZSKAI8uehTccVyw5BvGzSyRUf6f/8DI\nkZExqGWkSEhIoGvXrhkJ0IEDB3j44YcZP348t99+O1u2bMEYQ+vWrfn5559p2LAhvXr14swzzyQ2\nNpb169czZcqUkNyBd/bZZwcs//fff7nkkkvYvHkzzZs3p23btpQpU4aYmBh+/vlnPvroowKL77TT\nTgtYHhsbS3p6esb7/fv3A3DWWWcFnD+7fQk3mjhFmMTERObMmeN0GCoHWkeqMDx7xXhuGi98+Ln9\nz97lsgNd3n23w4EVknA6jkqXLs3YsWOZO3cuGzduZMWKFaxdu5affvqJO++8k9deey3T/DNmzGDy\n5Ml52obLk/mmpaVlmbZv375sl8uuM/Ubb7zB5s2bGTlyJP/5z38yTRs9ejQfffRRnuILJDGPI6p6\nE6wdO3YEnL59+/Z8xxQK+j9KhBk4cKDTIaggtI5UQdu7F7pcl8AST9JUvLi9c66oJk0QnsdRQkJC\nxu9r165FRAImD0uXLg2Y0LhcrkwtML68/Xu2bNmSZZr37rW8+Pvvv4HAyU2gIQ9iPONWZBdfIHmt\no4svvhiAr7/+Osu0I0eOZFzKC3eaOEWYdu3aOR2CCkLrSBWkbdugdWvb8RvsM+c++yyyHp9yKpw4\njiZOnMiPP/4YcNrs2bP5448/OOOMM6hbty7VqlXDGMPy5cszzbdkyRLeeOONgOsoW7YsmzdvDjit\nSZMmGGN4++23Mw1G+c033zBt2rQ870t28U2bNo1PP/00y/xnnHEGIsKmTZtyvY281lHVqlVp3rw5\nP//8MzN9nzwNjBkzhj179uRpfU7RS3VKKRWm/v4b2ra1PwHKl7dJU8OGzsZVVH366afcc889nH/+\n+TRv3pxKlSpx6NAhfvrpJ5YtW0ZMTAwTJkwgLi6OTp06Ub16dcaMGcOKFSuoW7cuf/75J/PmzaNz\n587MmjUry/rbtGnDzJkz6dKlCw0bNiQmJobExETq1avHZZddRvPmzVm0aBHNmjWjVatWbNiwgY8/\n/pjExMQsg1YGc9tttzF69GgGDhzIokWLqFatGr/++itffPEFXbt25f333880f0JCAk2aNGHp0qX0\n6tWLCy64AJfLRa9evahSpQoQfHTx3Bg7diytWrWiR48ezJo1i/PPP5/U1FS+++47WrduzdKlSzMu\nW4YrTZyUUioMrVgB7drBP//Y91WrwsKF4DNwtSpgY8aMoUWLFixcuJBly5axbds2ACpXrkzfvn0Z\nOHAgDT1Za0JCAl9++SXDhg1j6dKlLFmyhDp16jBt2jTKly/P+++/n+Vy3UsvvYSIsGjRIubOnYvb\n7aZKlSrUq1cPgDlz5vDAAw8wd+5cVq5cycUXX8zHH3/M5s2b+fDDD7PEm9OgmZUrV2bp0qUMHz6c\nL774grS0NBo1asTChQvZsGFDwERs6tSp3H///cybN499+/ZhjKFly5YZiVOgbeV14M4GDRqwfPly\nHnroIebPn4+I0LJly4wyyL6zebiQgsggw5WINAJSUlJSisxTnGfPnp1pVFcVfrSOwltqaiqNGzcm\nXM8LG/dtZPNvVenY0fZtArjoItvSdM45zsYWSnochb+CrCO3202NGjU4evRoRsIaSt7zAtDYGJNj\np7Lwbg9TWSQnJzsdggpC60idqi/+/oIa993DlVelZyRNl14KS5dGV9IEehxFglOpo/T0dHbv3p2l\nfNSoUWzYsIEuXboURGiFSi/VRZgZM2Y4HYIKQutInYrUbal0euIV3NNmk3bC3uF09dX27rlSpRwO\nzgF6HIW/U6mjgwcPUrlyZdq2bUvNmjU5ceIE3333HT/88AOVK1dmxIgRhRBpwdLESSmlHLZ2z1ra\n/Ocljk5NxqTHAdClCyQn26EHlCoq4uPjueOOO1i0aBHLli3j6NGjVKxYkXvvvZdHH300IgbB1MRJ\nKaUctP3gdi4f+hz73nkD3DZp6tYNpk6FuDiHg1OqgMXFxTFu3Dinw8gX7eOklFIO2X9sP03vf4Ed\nU17OSJpuuQXefVeTJqXClSZOEaZv375Oh6CC0DpSueE2bpr933jWT3oK3Lbxv08fmDIFsnlma1TR\n4yj8RWsdaeIUYXRU6vCndaRyY8pkF3+8/hAY2xH8jjtg0iTwPPki6ulxFP6itY40cYowPXr0cDoE\nFYTWkQpm4kTo1w+MsYMD9u8Pr71mH9yrLD2Owl+01pEepkopFUKvvZb54bz33QfjxmnSpFSk0ENV\nKaVC5M034Z57Tr4fNgxeeAFyeGKFUirMaOIUYfyfdK3Cj9aRCuSdd2w/Jq9hw2D0aE2asqPHUfiL\n1jrSxCnCjBkzxukQVBBaR8rf9On2jjnvo0EHD9akKRg9jsJftNaRJk4RZvr06U6HoILQOlK+np+0\ngZ63pON22/cDBsDzz2vSFIweR+EvWutIRwuJMPHx8U6HoILQOlJer0/bwZC7KoHbjjFw113w8sua\nNOWGHkfhL1rrKGQtTiJSTERGi8hmETksIt+KyNV5WL67iHwtIgdF5F8R+UpErijEkJVS6pQlf7CP\nu3qVyRgRvE8feOUVvXtOqUgXykP4bWAwMBX4PyAN+ERELg+2oIgkAdOAjcD9wCPAL0DlwgpWKaVO\n1Zx5R7mlewlILwbYx6i88YYmTUoVBSE5jEXkUqAb8JAx5iFjzBvAVcAGIMfeZSLSFPgvcL8x5mZj\nzOvGmAnGmP7GmHcLPfgwM2zYMKdDUEFoHUW3Lxal0aWLYNKKA9C9O0yerCOC55UeR+EvWusoVP//\n3IhtYXrdW2CMOQZMApqJSE4tR4OBbcaYlwFEJKEwAw13VatWdToEFYTWUfT67ns313Y8gfuETZpu\nuMEOQ6DPnss7PY7CX7TWUagSpwbAamPMQb/y732mZ6cN8IOI3CciO4EDIrJVRAYURqDhbtCgQU6H\noILQOopOK1dC+/aQdrQkAB06QHIyxMU5HFiE0uMo/EVrHYXq/6CKwLYA5dsAASoFWkhEygDlgBbY\nBCoJ2AT0BcaKyHFjzOuBllVKqVD56y9o2xb+3WP/F23dGmbNgmLFHA5MKVXgQpU4lQSOBSg/6jM9\nkFKen2WB7saYWQAi8j6wAngUn8t/SikVaps3w9VXwz//2PdNmsCcOVAyu7OaUiqihepS3RGgeIDy\nEj7Ts1sO4ATwvrfQGGOAGcA5InJOsI136NCBxMTETK9mzZoxe/bsTPMtWLCAxMTELMsPGDCASZMm\nZSpLTU0lMTGRXbt2ZSofMWIEo0ePzlS2ceNGEhMTWbVqVabysWPHZulcd/jwYRITE7MMZZ+cnEzf\nvn2zrKN79+4RuR/+itJ+jBs3rkjsR1GpD//9+Oabb7LMd6r7sWOHbWnasAFgI6VLJ/Lyy6s47bTC\n34+iUh/Z7UfPnj2LxH4UlfoItB/vv/9+RO7HXXfdlSkfaN26NW3bts0yb7aMMYX+AhYAKwOUtwHc\nQMdslhPgMLAlwLS7gXSgXg7bbQSYlJQUU1R06tTJ6RBUEFpH4S0lJcUUxHnh33+NadDAGPsgFWNq\n1DBm69YCClLpcRQBilIdec8LQCMTJKcJVYvTz0BNESnlV97UE+jPgRYyxninlRcR/8uK3jvxdhZk\noOHOvzVDhR+to6Lv0CHo2BF+9py5zjkHPv8cKlZ0Nq6iRI+j8BetdRSqxGkWtj/VXd4CESkG9AG+\nNcZs8ZRVEZEL/ZadAcQAvX2WLQHcAvxmjPmncEMPL9F6+2ck0Toq2o4dgw6djvL11/Z9+fI2aape\n3dGwihw9jsJftNZRSDqHG2O+F5GZwCgRORv4C5s0VcPeIef1DtCKzAnda8AdwHhPUrUR6AVUAa4r\n/OiVUspKT4ebex5n6Ze2e+bppxsWLBAu9P93TylVZIVyWLbbgCeBW4EzgF+xfZu+8pnHYPs8nSww\n5qiIXIkdYbwvkIC9fNfBGPN5KAJXSiljoP/ANGZ/YMcYKBnv5pNPXDTIaRQ6pVSRE7InJxljjhtj\nHjTGVDbGxBtjmvonPsaYK40xWZI5Y8wuY0w/Y0x5z7KXR2vS5H+nggo/WkdF0+NPuJn4qj09xcS6\neX+Wi8uDPmlTnSo9jsJftNaRPnIywhw+fNjpEFQQWkdFz4QJhseTTp4uJ7/lon17BwOKAnochb9o\nrSOxN64VTSLSCEhJSUmhUaNGToejlAoDqampNG7cmNyeF2bOhG7d3WBs4vT883D//YUdpVIqlLzn\nBaCxMSY1p3m1xUkppbLxxRdwy60nk6aHHtKkSalop4mTUkoFkJICnTvDieP2NNm3r+Hppx0OSinl\nOE2cIoz/0PYq/GgdRb41a6B9ezh40L6//nqYOFEQcTauaKLHUfiL1jrSxCnC9OvXz+kQVBBaR5Ft\n61Zo1w52ep5J0LIlJCdDbCgHb1F6HEWAaK0jTZwiTFJSktMhqCC0jiLX3r1w7bWwfr19X78+zJkD\nJUs6GlZU0uMo/EVrHWniFGH07sDwp3UUmY4cgU6dYMUK+756dZg/H8qUcTSsqKXHUfiL1jrSxEkp\nFfXS0qDrTWksX27fly8PCxboQ3uVUllp4qSUimrGwB13pvPpPNuJqXRp29J0wQUOB6aUCkuaOEWY\nSZMmOR2CCkLrKLI89JBhyuQYAOKKuZk9G6L0CkRY0eMo/EVrHWniFGFSU3Mc0FSFAa2jyPHcc4Yx\nY+wYAyKGae+6aNPG4aAUoMdRJIjWOtLEKcKMHz/e6RBUEFpHkWHevLIMHXpyYKYJE4Qbb3QwIJWJ\nHkfhL1rrSBMnpVQUas+IpKoZ7x5/HO65x8FwlFIRQxMnpVRU+fXXBJBZGLc9/fXvb/jvfx0OSikV\nMXQsXKVU1PjtN7jvvhpg7KnvppvcvPyySx+lopTKNW1xijCJiYlOh6CC0DoKTxs3wjXXwP79sUAi\nl122n3fecRET43RkKhA9jsJftNaRJk4RZuDAgU6HoILQOgo/u3bZ589t2eItacuzz/5N8eJORqVy\nosdR+IvWOtLEKcK0a9fO6RBUEFpH4eXgQejQAf78076vVu0o8AQJCW5H41I50+Mo/EVrHWnipJQq\nso4fh65d4Ycf7PtKlWDcuL+AXY7GpZSKXNo5XClVJLnd0Lu3feYc2If1fvYZHD9+3NnAlFIRTVuc\nIszs2bOdDkEFoXXkPGPg//7PMH26fV+iBHz8MdStm/NyW7dupX79+rRr144hQ4bw5ptv8t1333Hg\nwIHCD1plosdR+IvWOtIWpwiTnJxM586dnQ5D5UDryHlPPgnjx9sxBlwxbmbNctGiRfDl4uPjWbdu\nHStWrGDx4sWcOHEiY1qlSpWoX78+9erVo06dOtSpU4fatWuTkJBQWLsR1fQ4Cn/RWkdijHE6hkIj\nIo2AlJSUFBrpUzuVigqvvAL9+598P2UK9Op18n1qaiqNGzcmu/PCyJEjGTFiBG531s7jIkJsbGxG\nQiUinHPOORkJVd26dalTpw61atWiRIkSBb5vSqnC4T0vAI2NMTk+hE9bnJRSRcZ778GAAQawrU3P\nPZc5acqN++67j+eee469e/dmmWaMydQKZYxh06ZNbN68mQULFmRMc7lcVKtWjXfeeYfmzZuf8v4o\npcKP9nFSShUJn38Ot9zqxhibND34oOGBB/K+ntKlS/PII48geRhO3D+hcrvdrF+/nmLFiuU9AKVU\nWNPESSkV8X74ARKvTyfthD2l9e3nZtSoU3+OSv/+/SlXrtwpLy8ijBgxgiZNmpzyOpRS4UkTpwjT\nt29fp0NQQWgdhdaqVdC+PRw5bJ+dcl1iGhNfy9/z5+Lj4xkxYkSeWp28YmJiuPzyy3n00UdPPQCl\nx1EEiNY60sQpwkTrSK2RROsodDZvto9S2b3bvm/ZKp2ZM2KJLYDem3fccQcVK1bMU/IkIpQqVYrp\n06cTow/Byxc9jsJftNaRJk4RpkePHk6HoILQOgqN3btt0rRpk33foAF8PCeGgrqZrXjx4jzxxBPk\n5c5jYwzVq1c/pZYqlZkeR+EvWutIEyelVMQ5dAg6doQ//rDva9SA+fPh9NMLdju9evXKcyK0cuVK\n6tSpw9y5cws2GKVUWNDESSkVUbzPn/vuO/u+QgX7WJWzzy74bcXFxTFy5Mg8tTqlp6ezf/9+OnXq\nxD/I5IoAACAASURBVH333cexY8cKPjCllGNCljiJSDERGS0im0XksIh8KyJXn8J6FoqIW0ReLow4\nw93y5cudDkEFoXVUeNxu6NPHPnMObAvTZ5/BeecV3jZvvvlmLrzwQlyu3J8uvYnWuHHjaNKkCatX\nry6s8IosPY7CX7TWUShbnN4GBgNTgf8D0oBPROTy3K5ARG4AmgJFd7jzIMaMGeN0CCoIraPCYQwM\nHgzJyfa99/lz9esX7nZjYmJ4+umnA44kHozb7eb333/n4osvZsqUKXlquYp2ehyFv2ito5AkTiJy\nKdANeMgY85Ax5g3gKmADkKtPXkSKA/8DnsE7LHAUmu59aqkKW1pHheOJJ2DsWPu7K8bNe+9By5ah\n2XaXLl2oV69ellanmJgY4uLicryDLj09naNHj9KnTx9uueUWfWBwLulxFP6itY5C1eJ0I7aF6XVv\ngTHmGDAJaCYilXOxjgexCdP/CiXCCBEfH+90CCoIraOC98ILkJR08v2kN4ROnUK3fRFh1KhRmVqd\nYmJiqFy5MmvXruWGG27I1Xree+896tWrx48//lhYoRYZehyFv2ito1AlTg2A1caYg37l3/tMz5aI\nVMUmTsM9CZdSKkpMmkSmR6f87zk3ffqEvtG5Q4cONGnSJFPr0syZM6lSpQozZszgjTfeoHjx4kFb\nnzZv3kyzZs147rnnTunyn1LKWaFKnCoC2wKUb8O2IlUKsvxzQKoxZmZBB6aUCl8zZsCdd57sF/To\nY2kMecCZm4G9rU7p6ekAjBo1iksvvTRj2u23385PP/0UtCN5eno6aWlpDB06lGuvvZYdO3aEJH6l\nVMEI1RmoJBCopeioz/SARORKoAtwXyHEFXGGDRvmdAgqCK2jgjFvHtx6q8l4aO/A+47xRFIBDAme\nD23atKF9+/Z06tSJIUOGZJleu3ZtUlJSuOeeewCCjv+0aNEi6tSpw+eff14o8UYyPY7CX7TWUagS\npyNA8QDlJXymZyEiLuAl4G1jTGohxRZRqlat6nQIKgito/xbvBhuvNGQlmYTj569D/PyC8Xz9fy5\ngiAizJ07l48++ijbVqUSJUowfvx4PvzwQ0qXLh300t2ePXto164dDz30ECdOnCis0COOHkfhL1rr\nKFSJ0zbs5Tp/3rKt2SzXB6gJTBSRap5Xdc+00p732bZWeXXo0IHExMRMr2bNmjF79uxM8y1YsIDE\nxMQsyw8YMIBJkyZlKktNTSUxMZFdu3ZlKh8xYgSjR4/OVLZx40YSExNZtWpVpvKxY8dmydgPHz5M\nYmJilvExkpOT6du3L4MGDcpU3r1794jcD39FaT+qVKlSJPbDqfr4/nvo1AmOHh0HDKNjlwO8PSke\nkYLZj2+++SbLfHnZD5fLRVJSUtD96Ny5MytXrgz6x8XtdmOMYcyYMdSqVYsbb7wxV/tR1L9Xq1at\nKhL7UVTqI9B+NG/ePCL346677sqUD7Ru3Zq2bdtmmTc7EopxRURkDHYMp7K+HcRF5GHgSaCqMWZL\ngOVGAI+RdfgB4ykzQBdjzJxsttsISElJSaFRo0YFsi9KqcKzYgW0bg3//mvfd+wIH34IcXEFt43U\n1FQaN25MqM4LaWlpPPnkkzz55JOISI4dwmNiYihRogRvvvkm3bp1K/TYlFKW97wANA52hStULU6z\ngFjgLm+BiBTDtih9602aRKSKiFzos1wytn9TZ7+XAPM8v38XgviVUoVszRr70F5v0nTFFTBzZsEm\nTU6IjY3l8ccf58svv6R8+fJBL90dPnyY7t27c8cdd3D48OEQRqqUyo2QJE7GmO+BmcAoz2NX7gS+\nBKoBw31mfQf4w2e51caYOf4vz+R1xpiPjTHbQ7EP4cK/mVOFH62jvFu7Fq68Ev75x76/9FKYMwdK\nBr0QHzlat27Nb7/9Rvv27XOcz3sV4K233qJBgwb8+uuvoQgv7OhxFP6itY5CeV/vbcCLwK3YDt8x\nQEdjzFc+8xggNwObGKL0sSvDhw8PPpNylNZR3mzYAG3awBbPxfp69eDTT6F0aWfjKgxnnnkmc+bM\nYezYscTGxubY+uR2u/n777+55JJLmDBhQtQ9rkWPo/AXrXUUssTJGHPcGPOgMaayMSbeGNPUGPO5\n3zxXGmOC3m9sjIkxxkTl8ATjxo1zOgQVhNZR7m3ebFuaNm607y+6CD7/HMqWdTauwiQiDBw4kB9/\n/JHq1asHHfPpxIkTDBgwgC5durBnz54QRuosPY7CX7TWkTMjyalTFq23f0YSraPc2brVtjStW2ff\nV69xjC++gLPOcjauULn44ov55Zdf6N27d67mnzt3LnXr1mXZsmWFHFl40OMo/EVrHWnipJQKue3b\n4aqrbIdwgPizt7H4yxgqVHA2rlBLSEjgzTffZNq0acTHxxMbm32De3p6Otu3b6d169Y88cQTGSOY\nK6VCSxMnFVKTJ0/G5XLx9ttvOx2KcsjOnTZp8vYrLV5uK6lflaFaFWdHBXdSjx49WLFiBfXr189x\ntHHvmE9JSUlcccUVbN68OYRRKqVAE6dCc+TIEZ5++un/b+++46Oo1sePf85uICEhgVAEpIQuoFIC\nSO8SBDFKEylCABsIClfAH+gXY3tp0CsoyLXcoKBSpamIcAEBlSaELkivCdIVQk1yfn9M2iabnuzs\nZp736zWvsFOf9XEmT2bOnEPjxo3x9/enWLFiVK5cmbZt2zJx4kSOJT2fyKGIiAhsNhsdO3bM54jz\nx4kTJ7DZbAwdOtTpcqVUlsNQeLq0Hb+JFJcuQefOsG+f8blIYAybf/HlnhqF6PW5XKpevTqbNm1i\n7NixAJm2fdJas2nTJu677z6++85pN3YeT84j92fVHEnhVACuXbtG8+bNefXVV7l27RpPPvkkY8eO\npXv37sTGxhIREcHPP/+cq317er8uPXv2ZP/+/fTo0cPsUAqMp+eooFy+bPTTtGuX8dkecJa1a6Fh\nnZLmBuZGihYtyuTJk1m5ciUlS5bMss+nf/75h0cffZRRo0Zx8+bNDNf1RHIeuT/L5khrXWgnIBjQ\n27dv1670xhtvaKWUfvbZZ50uP378uP7zzz9zvX+llO7QoUOuty9Ix44d00opPWTIELNDEW7kwgWt\nGzXSGoxJ+cfo5RsPmxLL9u3btRnXhZw6e/as7tSpU1LXK5lONptN33vvvXr//v1mhy2ER0q6LgDB\nOovaQu44FYDNmzejlGLEiBFOlwcFBVG7du3kz+vWrWPYsGHUqVMHf39//P39adq0KZ9//rnDduvX\nr8dms6GUYt26ddhstuQpqc1QeHg4NpuNDRs2pDvurFmz0rUvSv1o7cCBA/To0YMyZcpgt9s5mfiO\n+JIlS+jfvz+1atXCz8+PkiVL0rZtWxYvXpxu/9WrV0cpldyWyWazYbfbk+NxFgOQ/Pjx3LlzDB48\nmLJly+Lr60uLFi1Yv3690/+Oe/bsoVu3bgQEBFCyZEkefvhh9u3bR1hYGDabLTl+Ya5z54wuB3bs\nMD77lrzKN0v/oluLGuYG5ubKlSvHqlWrmDx5Mna7Pcs+nw4cOECjRo344osvLNfnkxCuZN3WmAWo\ndOnSABw6dIj69etnuX5ERARHjhyhefPmVKpUiStXrvDTTz/x7LPPcvDgQd577z0AqlatSnh4OOHh\n4VStWpWwsLDkfTRs2BDIug1RRssOHTpE8+bNqV+/PkOGDOHixYsULVoUgIkTJ+Lt7U2bNm2oUKEC\n58+f57vvvqN3795MmzaN559/HoBGjRoxevRopk6dSsOGDXnssceS91+1atUsY7hy5QqtW7emZMmS\nDBo0iHPnzjFv3jweeughtm/fTr169ZLX3bVrF23atOHGjRv06tWLmjVrsm3bNlq3bk2DBg0KfTsq\nT3H2rNEQ/I8/jM8VKsDatf7UqdPA3MA8hM1mY9y4cbRr144+ffpw5syZDN+mi4+PJz4+nqFDh7Jy\n5Uo+/fRTSpQo4eKIhbCArG5JefKESY/qvv/+e62U0gEBAXrs2LF61apV+uLFixmuf/z48XTz4uPj\ndUhIiC5SpIg+depU8vzz589n+qguPDxc22w2vX79+nTLvvzyS22z2fSsWbMcjq2U0jabTYeHhzvd\n57Fjx9LNi42N1fXr19eBgYH6xo0b6faX0aM6ZzForZNjGDVqlMP8yMhIrZTSw4cPd5jfunVrbbPZ\n9Lx58xzmv/baa8n7OnHihNMYCtr58+dNOa67OXNG63vuSXk8V6mS1gcPmh2V5zyqS+vKlSu6b9++\n2Xp0Z7fbdeXKlfWWLVvMDjvX5Dxyf4UpR/KozmTdu3fngw8+AOCDDz6gS5culClThlq1ajFq1CgO\nHz7ssH5QUFC6fdhsNp577jni4+MdGpJn9LZaXpUvX55XXnnF6bLUd4uS+Pr6EhYWxt9//83vv/+e\nLzH4+fnx7rvvOswbPHgwXl5eDsc4efIkv/32G/Xr16dv374O648fP57AwMB8iSe3CipHnuTUKWjX\nDv780/hcpQqsXw+1apkblycrUaIEc+fOZebMmfj4+GTZcDw6OpqWLVsyefJkEhKyM5KVe5HzyP1Z\nNUdSOBWQ0aNHEx0dzYIFCxgzZgxt2rTh1KlTfPzxx9SvX58ffvghed1r167x2muv0bBhQ/z9/ZPb\nBvXq1QuA6Ojo5HXDw8MLJN4GDRpk2Pne+fPn+de//kW9evXw8/NLju+ll15KF19e1K5dG19fX4d5\ndrudcuXKceXKleR5uxJfy2rdunW6ffj6+iY/tjRLQeXIUxw/bhRNSX8fVKsGGzZA9eqmhlUoKKUY\nMmQIO3bsoE6dOlkO1xIfH8/LL79MSEgIf/3lWeOhW/088gRWzZG0cSpAfn5+9OrVK7kAunr1KhMn\nTuTjjz9m2LBhnDlzBq017dq1Y+fOnTRq1IhBgwZRunRpvLy8OH78OLNmzeLWrVvJ+wwODi6QWMuV\nK+d0/uXLl2nSpAmnT5+mVatWdO7cOfk16Z07d7Js2TKH+PIiICDA6XwvLy+Hdh3//PMPAHdlMDZH\nRt/FVQoqR57gyBFjGJWkdvm1asHatVCpkrlxFTZ16tRh27ZtjBs3junTp6OUyrRB+Lp166hXrx5z\n584lJCTEhZHmnpXPI09h1RxJ4eRC/v7+TJs2jR9++IGTJ0+yZ88ejhw5wo4dO3j66af59NNPHdaf\nP38+X375ZY6OkfQXaFxcXLplf//9d4bbZdSY+r///S+nT5/mrbfeYsKECQ7LIiIiWLZsWY7iyw9J\nBda5c+ecLve0v6wLi/37jc4tz5wxPgdWPsvqtWWoVEkuMwXBx8eHadOm0blzZwYNGkRsbKzT8x6M\nu09XrlyhS5cujB07lrfffjv55Q8hRM7IozoT+Pn5Jf/7yJEjKKUIDQ1Nt96GDRucFjQ2my3DN2uS\n2vecSfrtlUpUVFSOYz169ChAhvGlldTuoiDH0WrQwHgja+PGjemW3bhxI/lRnnCdrVuhTZuUosle\n7gBVXhxMqbsKV6eM7ig0NJS9e/fSrFmzLIdrAfj3v/9NixYtks9tIUTOSOFUAD777DO2bdvmdNnS\npUvZv38/gYGB3HfffQQFBaG15tdff3VYb/369fz3v/9Nt31kZCSlSpXKcIyqpk2borVm9uzZDrfu\nN23axJw5c3L8XTKKb86cOaxYsSLd+oGBgSilOHXqVI6PlV1VqlShVatW7Ny5k4ULFzosmzx5Mpcu\nXSqwY2dHZGSkqcd3tTVrjMdzFy8an4tW2kuVFwbzv+e+pnjR4uYGZxGVKlVi/fr1hIeHo5TKcriW\nXbt20ahRI2JjY10YZc5Y7TzyRFbNkdxDLwArVqzgueeeo2bNmrRq1Yq7776b2NhYduzYwS+//ILd\nbmfGjBkUKVKERx55hKpVqzJ58mT27NnDfffdx59//sny5ct57LHH+Pbbbx32HRUVRceOHVm4cCE9\nevSgUaNG2O12QkNDuf/++2nWrBmtWrVi7dq1tGjRgrZt23LixAm+//57QkND03VamZUnn3ySiIgI\nRo4cydq1awkKCmL37t2sWbOGXr16sWjRIof1/fz8aNq0KRs2bGDQoEHUqlULm83GoEGDqFy5MkC+\ndM43bdo02rZtS79+/fj222+pWbMmUVFRbNmyhXbt2rFhw4ZMf3kUpKioKIYNG2bKsV1t8WLo1w9u\n3zY+F6+9Db9B/Vk7YhVl/cqaG5zF2O12Jk2aRIcOHXj88cc5f/58hnd+lVIEBwenexnDnVjpPPJU\nls1RVv0VePKESf04HTx4UL///vu6S5cuukaNGtrX11f7+vrqWrVq6aFDh+qoqCiH9Y8fP6779Omj\ny5Urp4sXL66bNWumFy5cqNetW6dtNpt+4403HNY/e/asfuKJJ/Rdd92lvby80vWLdOnSJR0WFqbL\nlCmj/fz8dMuWLfXq1asz7MfJZrPpoUOHZvh9du/erR966CFdunRpXaJECd2hQwf9888/Z9gn06FD\nh3T37t11qVKltN1ud+hXKqNtbDab7tixo9PjV61aVVevXj3d/F27dumuXbvqgIAAXaJECd29e3e9\nb98+/cgjj2ibzab//vvvDL+TyLvPP9faZkvpp6lM8K/a/42yetfZXWaHlilP7ccpJy5evKhDQ0Mz\nHJ4lMDBQR0dHmx2mEG4jJ/04KZ0Pf/27K6VUMLB9+/btlm39bzUJCQnUqFGDmzdvEhMTY3Y4hdbk\nyfDyyymfq7ZfR3T7h1k1+EfaVW1nXmDZEBUVRePGjSns1wWtNf/5z38YPXo0CQkJDneffvzxR7p2\n7WpidEK4l6TrAtBYa51pg2Bp4yQ8Unx8PBeTGtWk8s4773DixAl69OhhQlSFn9Ywfrxj0TRk+GX+\n7tKbuY9/5fZFk5UkjZe5bds2qlevnjzO5ZgxY6RoEiIPpI2T8EjXrl2jYsWKdO7cmdq1a3Pnzh22\nbNnC77//TsWKFXnttdfMDrHQiYuDZ5+FmTNT5r39NkyYEMiUW0co4SPjormj+vXrs2PHDl566SWO\nHTuWrnd+IUTOyB0nD+OsWwAr8vX15amnnuLIkSNERkby2Wefce7cOYYPH87WrVtN7QSzMObo6lV4\n5JGUokkp+OQTmDjR+LcUTe7Nz8+PTz75hJUrV3pM/02F8TwqbKyaI7nj5GFGjhxpdghuoUiRIkyf\nPt3sMJwqbDmKiYGHH4YdO4zPRYvCV1/B44+bG5co3ArbeVQYWTVHUjh5GE8ZLsHKClOO/vgDunZN\nGUKlZElYutQYi06IglSYzqPCyqo5kkd1Qgin1q2Dli1TiqagINi4UYomIYS1SeEkhEhn1iwICYGk\n4Q2Dg+HblWegzH5zAxNCCJNJ4eRhli5danYIIguenKP4eKOrgbAwuHPHmNetGyxecZFBqzvz5JIn\n86Xnd1E4rV+/HpvNxhtvvJHnfXnyeWQVVs2RFE4eZu7cuWaHILLgqTm6dg169jQ6t0wyfDjMWRjL\nE99358L1C8ztNTfTgWRF4XTixAlsNhvdunXLcl2lVL78P+Kp55GVWDVH0jjcw8yfP9/sEEQWPDFH\nJ05AaCjs3m18ttvhww/hmefu0GN+X/b8tYd1YeuoVbqWuYEKt9asWTP2799PmTJl8rwvTzyPrMaq\nOZLCKYeOHj1KiRIlKF26tNmhCJEvNm6EHj3g3Dnjc4kSsHAhPPigZsiyp1l5ZCXL+y+nyd1NzA1U\nuD0fHx9q165tdhhCFCh5VJcDV69epUGDBlSoUIGwsDCiojIdzkYIt6Y1zJgB7dunFE01a8LmzdC5\nM0xYM4FZu2Yx67FZhNSw5mvHImcyauNUtWpVqlevTmxsLC+++CIVK1bEx8eHBg0asGjRIqf7unPn\nDh988AGNGzemePHiBAQE0LZtW77//vt06x46dIjx48fTuHFjypQpQ7FixbjnnnuYMGECsbGx6dZv\n3749drudW7du8eqrr1KzZk2KFi2aL22zROEnhVMOzJ49m9jYWO7cucM333xD48aNeeCBB5gzZw63\nb982Ozwhsu3mTRg2DJ5/PqUReMeOsGUL1KkDi/cvJuK3CKZ0mUL/+/ubG6zweEop7ty5Q0hICKtX\nr6Z37948+eSTHD16lL59+7J69WqH9W/fvk1ISAhjx44F4KmnnuLJJ5/k5MmTPProo8yYMcNh/cWL\nF/PFF19Qo0YNwsLCGD58OKVLlyYiIoKQkBCHAY6T4gHo1asXs2fPpmPHjowePZpq1aoV4H8FUWho\nrQvtBAQDevv27TqvEhISdM2aNTXgMNlsNg3oSZMm5fkY2REWFuaS44jcc/ccnTypdZMmWhv3nIzp\npZe0vnMnZZ3bcbf1gr0LzAuyAG3fvl3n13XBKo4fP66VUrpr166Zrrdu3TqtlNKvv/66w/yqVatq\nm82me/bsqe+k+h9tzZo1Tvc7ceJErZTS4eHhDvOvXbummzZtqn18fHRMTEzy/OjoaIf9JnnzzTe1\nzWbTc+bMcZjfvn17rZTSwcHB+sqVK5l/eZEhd7/W5UTSdQEI1lnUFi6746SUKqqUilBKnVZKXVdK\nbVZKPZiN7XoqpeYqpY4opWKVUgeUUu8rpVw6ONbatWs5fPhwuvkJCQkAtGzZ0iVxWLWnVk/izjn6\n+Wdo3Bi2bTM+FysGc+fC+++DV6oWj0XsRehzbx9zghSF1pQpU/BK9T9ax44dCQoK4vfff0+ep7Xm\nk08+oVy5cukG6/bz82PSpEncunWLxYsXJ8+vUKGCw36TjBgxAq11ujtaYNx1euONNyhRQsZZzC13\nvtYVJFc2Dp8N9ACmAIeBMOBHpVR7rfXGTLb7FDgDfAWcBO4HRgJdlVLBWutbBRp1og8//BC73Z7u\nli9AtWrV6Ny5syvCoF+/fi45jsg9d8xRfDy88w689hok1vpUrw5LlkD9+ubGJqyhZMmSVKlSJd38\nSpUqsXnz5uTPf/75J5cvX+a+++7j9ddfT7f+ucQGeQcOHHCYP3PmTGbNmsXevXv5+++/k/+oVUoR\nHR3tNKamTZvm+vsI97zWuYJLCiel1APA48BLWuspifO+AvYCk4HWmWzeS2u9Ic3+ooBZwABgZoEE\nncrx48f54YcfnHb8p5TixRdfxGaT5mLCPZ07BwMHwv/+lzKvSxeYMwdKlTIvLmEtGd3Z8fLySi5y\nAC5dugTAvn372Ldvn9NtlFIOjb5HjRrFxx9/TJUqVXj00UepUKEC3t7eAISHh3PrlvO/r++6665c\nfRdhba6649QbiAM+T5qhtb6llIoE3lZKVdRan3G2YdqiKdESjMKpbkEEm9Ynn3yCzWZzerfJ29ub\nsLAwV4QhRI6tWwf9+0NMjPHZZoPXX4cJE4y+moRwNwEBAYDRcHvBggVZrn/+/HlmzJhBw4YN2bRp\nU3LBBPDXX38RHh5eUKEKi3LVbZKGwEGt9bU087emWp4TFRJ/XshTVNlw48YNPvnkE6dFk5eXF2Fh\nYS59Rv7rr7+67Fgid9whR/Hx8Oab0KlTStFUvjysWQOvvppSNN2MuylDqAi3UrduXQICAvj111+d\nXnfTOnr0KFprOnXq5FA0AWzY4OzvbpFf3OFaZwZXFU4VgBgn82MABdydw/29jHEH69s8xpWlefPm\n8XfSSKdpxMXFMXLkyIIOwcHk1ONhCLdkdo5OnIAOHWDSpJT2TA8+CDt3Gn02JbkZd5OHvn6IST9P\nMiVOIZyx2+0MHz6cmJgYXnrpJeLi4tKts2/fPs6fPw9AUFAQABs3bnT4I+D06dNMnDhRhggqQGZf\n68ziqkd1xQBnD5lvplqeLUqp/sBQ4F2t9ZF8iC1DWmumTJmCzWZzeAYPxsndunVr7r333oIMIZ15\n8+a59Hgi58zM0TffwIgR8M8/xmebDcLDYeJEx0dz8QnxDFg8gC1ntvB2x7dNiVV4nj179jBkyBCn\ny+rUqUPz5s3z5Tivv/4627ZtY9q0aSxfvpy2bdty1113ER0dze7du9m9ezebNm2ibNmylC9fnl69\nerF48WKaNGlCp06dOHv2LMuXL6dTp04cOVKgvyYszaq/j1xVON0AvJ3M90m1PEtKqTbAf4EVwKv5\nE1rGNm3axJ49e5wui4+P58UXXyzoENLx9fV1+TFFzpiRoytXjM4s58xJmRcUBF9/Da3TvHqhteb5\nH59n2YFlLOm7hFZVWrk2WOGRkt5Omz17ttPl7dq1o3nz5hkO8pvZnZ+0y4oWLcqqVauIjIxk9uzZ\nLF68mFu3blGuXDnq1avHiBEjuP/++5PXnzVrFtWqVWPRokVMnz6dKlWqMHbsWMaNG8eiRYtyHI/I\nHqv+PnLVo7oYUtolpZY0z/m7oqkopRoAy4DdQB+tdUIWmyTr1q0boaGhDlOLFi1YunSpw3qrVq0i\nNDQ0+fO0adOc9g0CUKZMGSIjI7lwwbGZ1WuvvUZERITDvJMnTxIaGpru9dlp06Yxbtw4h3nXr18n\nNDQ03bPjuXPnOv1Lr2/fvll+jyTPP/88kZGRDvOioqIIDQ2V7+HB3+PLLw/QoEHqomkadeuOY9eu\nlKIp9fd4Y/0bfLr9Uz7t/inXoq65zfdwVT42bdqUbj1P/B6uzEdQUBBTp07lX//6F/Hx8cnT1atX\nefjhh1m/fj1r166lXbt2xMfHU7NmTYfvcezYMY4cOeL0e0yYMIFu3bql+x4jR45EKcWGDRu4fPky\n169fZ9GiRdjtdnr06EGxYikPKiIiIihdujRHjhzh+vXrHDhwgAEDBtCzZ0/27dvHmjVrktedNm0a\nTZo0cXgE6Gn5SPoenv7/lVnf45lnnnGoB9q1a5ezLoWy6iEzPyaMLgduA8XTzJ8IxAMVs9i+Bkbx\n9QdQKgfHzXXP4dHR0dput6frKZzE3sLfeeedHO9TiPx09arWw4c79gBeooTWc+dmvM1/fv+PJhz9\n9oa3XReom5Gew4UQabljz+HfYjwWfCZphlKqKEYnmJt1YlcESqnKSql7Um+olCoHrMJoDP6QQe3U\n9AAAHApJREFU1vqSKwL+7LPPMlxmt9sZNmyYK8JIJ21lLtyPK3K0di3cfz/85z8p89q1g9274Ykn\nnG+z6I9FjFg+glEPjGJC6wkFHqMQeSHXOvdn1Ry5pI2T1nqrUmoh8E5iIZTUc3gQkPq+21dAWxwf\nIa4EqmLctWqT5rn0X1rr9H3p59Ht27eZPn16hl0Q9OvXj7Jly+b3YbPFWc+7wr0UZI6uXoWXX3Ys\nmHx9ISLCaBSeWT+s/t7+DGk4hKkPTZX2HcLtybXO/Vk1R0q7qA+XxDtMbwIDgUCMtkqvpi58lFI/\nA2201l6p5mXWkcd6rXXHTI4ZDGzfvn07wcHB2Y51/vz5PJHRn+3Atm3baNy4cbb3J0R+WLYMRo6E\n06dT5rVrB5GRUKOGeXF5mqioKBo3bkxOrwtCiMIr6boANNZaR2W2rsvGqtNa38bof+nlTNbp4GSe\ny/s3njp1qtMuCGw2G8HBwVI0CZc6dQpeeAFSt33M7l0mIYQQ+cuVg/x6hB07djgMOJlaQkICo0eP\ndnFEwqri4mD6dPi//4Nrqfrc79IFZswwBukVQgjhWvK3ahqZdUFQunRp+vTp4+KIHKV9lVO4n/zI\n0S+/QNOmMGZMStFUvjzMmwcrVkjRJAo/uda5P6vmSAqnVC5evMg333zjtIt/u93OiBEjKFq0qAmR\npRg/frypxxdZy0uOTp2Cfv2gbVtjiBQApWD4cNi/H/r2NT4LUdjJtc79WTVHUjilMnPmTKdFU5Ln\nnnvOhdE4N336dLNDEFnITY5u3IC33oI6dYy7SkkaNYLffjMezZUsmfV+EnQCL//vZY5ePprjGIRw\nJ3Ktc39WzZEUToni4+P56KOP0jUIB6MLgh49enD33Tkdizj/WfX1T0+SkxwlJBjjy9Wta7Rlun7d\nmF+mDHz2Gfz+O7Rokb19aa15aeVLvLfxPXad3ZWLyIVwH3Ktc39WzZE0Dk+0fPlyTqd+zzuVuLg4\nXnjhBRdHJAq71ath/HjYsSNlnt1udDnw2msQGJiz/b238T2mbpnKx90+pkfdHvkbrBBCCEAKp2Qf\nfvghdrs9XaeXSinq1atH67QjpQqRSzt3Gp1YrlrlOD8kBD74AO69N+f7nLVzFi+vfpn/a/t/jGg6\nIn8CFUIIkY48qsN4M2Dt2rVOewrXWjN69Gi36Wk57cCJwv1klKM//jAadwcHOxZNjRoZn1euzF3R\ntPzgcoZ9N4yng5/m9fav5zJqIdyLXOvcn1VzJIUTRgO3jLogCAgIoH///i6OKGPXkxrBCLeVNkcH\nDkD//nDffbBggTEcL0BQEHz9NWzbBjkZmDu1zac302dhHx655xFmPDzDbQp8IfJKrnXuz6o5ctmQ\nK2bIzpAr//zzD+XLl+fGjRvpltntdsaMGcN7771XwJGKwujAAXj7bZgzx2gEnuSuu+D//T+j129v\n77wdo/eC3pyLPcfKgSspVqRY3nZmETLkihAiLbcccsVdzZ49m5s3bzpdlpCQwPDhw10ckfB0v/8O\n774LS5ak3F0C40258eONgsnPL3+O9XXPr7kVd0uKJiGEcBFLF05aa6ZOnep0md1u56GHHqK6dNEs\nskFr4y25d9+FtWsdl5UqBePGGW/LFS+ev8f18fLBx8snf3cqhBAiQ5Zu47R69WqOHDmCs8eV8fHx\nbtkFwYULF8wOQaRy8yZ8+aXR4DskJKloMnJUoQK89x4cP248msvvokmIwkyude7PqjmydOH00Ucf\nYbfb081XSlG9enUefPBBE6LK3NChQ80OQQAxMTBpElSpAkOGpAyPAuDnN5TPP4djx2DsWPD3Ny9O\nITyVXOvcn1VzZInC6a233uL777936G7g+PHjLF++3GkXBACjR4/GZnO//zzh4eFmh2BZWhuD7w4Y\nYLwR9+abcP58yvImTYy35n7+OZynnsp7w28hrEyude7Pqjlyv8qgACxZsoTQ0FCqVavG+++/z6VL\nl5gxY0aGhZGPjw+DBw92cZTZI28Bud7ly/Dhh0Z3Am3bGm/J3bljLLPbjb6ZNm6ErVuhTx9o2jT/\nc7Tl9JZ836cQ7kyude7PqjmyVOPwU6dO8fLLL/PKK69gs9mc3m3y8vIiLCyMgIAAEyIU7iIhATZs\ngJkzYeFCoy1TaqVKwTPPGG/IVa5csLFM3zqdUStG8euQX2lVpVXBHkwIIUSmLFU4gdHFwO3btzPs\nKDAuLo5Ro0a5OCrhLg4dgtmz4auv4MSJ9MvbtDEKpt69wccFL7Mt2LeAF1a8wNgWY6VoEkIIN2CJ\nR3XOOHuTzm63065dO+rWrWtCRNkTGRlpdgiFTkwMTJ8OLVtC7drw1luORVNgILz4IuzbZ9yFGjgw\n86Ipv3K0+uhqBi4eyID6A4jobM2hDYR1ybXO/Vk1R5YtnJzRWvPbb7/Rv39/Nm/e7LS4MltUVKYd\nmopsSiqW2rWDihVh1CjYtClluc0GXbvCvHlw5gxMnQr16mVv3/mRo6iYKHrM70Gn6p2YGToTm5JT\nVViLXOvcn1VzZIkhV7y9vbl161a2t/Py8iIuLo6GDRsyevRoBg4c6LTbAuFZYmJg0SKjzdIvvzj2\n6p3kvvtg8GDjzbkKFVwfI8DhS4dpNbMVVUtWZc2gNRQvKh1A5ScZckUIkZYMuZKGn59fjgqnuLg4\nAHbu3ElYWBglS5bk0UcfLajwRAHR2uhfaflyY9qyxXmxdM89xttwffrA/feDmePk3o6/TbdvuhHo\nE8jy/sulaBJCCDdjicLJ39+fS5cu5WrbIUOG0LVr13yOSBSUa9dgzRr44Qf48UeIjna+XlKx9Pjj\nxl0mM4ul1Irai/LvkH9Tv1x9yviWMTscIYQQaViicCpRokSO1ldKobXm3XffZfz48Rm+gSfMFx8P\nu3YZQ53873+wbh3cvu183XvvhR493OPOUmYeuecRs0MQQgiRASmc0rDZbHh5efHNN9/Qu3fvAowq\nd0JDQ/nuu+/MDsM0WsMffxiF0tq1RqF05YrzdX18oGNHePhhYwoKck2MVs+REPlBziP3Z9UcWaJw\nCggIwG63Zzi8ShK73U6JEiX48ccfadasmYuiy5mRI0eaHYJL3bkDu3cbPXP/9hv8/DOcO5fx+lWq\npBRKHTqAr6/rYk1itRwJURDkPHJ/Vs2RJQqn4sWLZ9hTeBK73U7NmjX56aefqFq1quuCy6GQkBCz\nQyhQFy7A5s1GoZQ0jMmNGxmvX7q0USB16GDcXbrnHvMfwRX2HAnhCnIeuT+r5sgShZN/FsPTK6Vo\n3749ixYtynF7KJF7ly/Djh0QFQXbtxvToUOZb1O8uNH3UseOxlS/vtHnkieKS4jDy2aJU1AIIQoN\nS1y1AwICSEhIyHD5008/zfTp0ylSpIgLo7IOrY232/buNQql7duNYuno0ay3DQqCFi2MXr1btjQK\npcKQpmUHljFp3STWDFojb88JIYQHsUTh5O/vn+4xXdKbcu+//z5jxozxmDfnli5dymOPPWZ2GBm6\neNEokNJOGTXgTs3bGxo1SimSWrSAu+8u+JjzW1Y5+uXELzyx6Am61+5OoE+gCyMTwnO4+7VOWDdH\nliicihd37ETQbrdTpEgR5s6d63FJd4eYY2Ph8GHjsdqhQ3DwYMq/M2u4nVqxYtCwIQQHQ+PGxs96\n9QrH3aTMcrT33F5C54XSolILvu7xNXab9EgvhDPucK0TmbNqjixROKVu4+Tl5UVgYCArVqxI6l7d\no8yfP7/Aj6E1nD9vDHR74gQcP+5YIJ05k7P9VapkdDJ5773Go7bGjY1G3F6F9P++jHJ04soJunzd\nhaolq7Kk7xK8vbxdHJkQnsMV1zqRN1bNUSH91eUodeFUu3ZtfvrpJypXrmxiROa6dQvOnoWTJ1OK\no9TTyZOZv8mWkQoVjILo/vuNIimpWCpZMv+/g6e5cP0CXb7ugrfdmxUDVlDCR15CEEIIT+Sywkkp\nVRR4ExgAlAJ2A69qrVdnY9u7galAZ8AG/AyM0Vofy86xS5cujc1mIyQkhAULFmT5lp2nio01HpXF\nxBhTdLTznxcv5v4YZcpArVpQu7bxM+nfNWsab7yJ9GJvx9J9Tncu3bjExmEbKV+8vNkhCSGEyCVX\n3nGaDfQApgCHgTDgR6VUe631xow2Ukr5AesAf+AtIA74F7BOKdVQa305qwOXLVuWAwcOUK1aNbw8\n5PlQfLzRoPr8eWO6cMHxp7N5ublLlJavr/EmW9qpZk2jSAqUtsw5div+Fv7e/qwYsIKapWqaHY4Q\nQog8cEkVoZR6AHgceElrPSVx3lfAXmAy0DqTzZ8HagBNtdZRidv+lLjtS8Cr2YmhVq1auY4/p7Q2\n7v5cu2ZMV6/CP/8YhdCVK0b/RUn/zmjeP/9ktPchwBe5isvb23hLrUIF42fFiukLpNKlze9A0tMN\nGTKEL75IyVGpYqX435P/MzEiITxP2vNIuB+r5shVt196Y9wp+jxphtb6llIqEnhbKVVRa51Rk+Ne\nwO9JRVPitn8qpdZgFGPZKpxSS0gw7s7kdLp6NaUQSl0Upf13bKxRPBUMx55abTbj8VmZMlC2rDEl\nFUapi6QKFYy7RVIUFTyr9qYrRH6S88j9WTVHriqcGgIHtdbX0szfmmp5usJJGZ0r1QcinexzK9BZ\nKeWntY7N7OCPPOJYLN2+nfMv4ApFixrFTcmSKVNSMZRSGPVzKJJKlvTcnrMLq379+pkdghAeT84j\n92fVHLmqcKoAxDiZHwMoIKNuDksB3plsS+K2mQ7UER2dvSBzy8sL/P2NqXhxY0r976TPgYHpC6PU\nU7FiBRunEEIIIfLGVYVTMeCWk/k3Uy3PaDtyuW0yf/84/Pxu4e2t8fFJwNs7adJ4eyekmqeTlxnz\ndKp1E/DzS8DXNx5f3wR8fRMoViweP78EihTJ/XO5pLtgMc5KQyFEvtu/f7/DTyGEyMn1wFWF0w2M\nO0dp+aRantF25HLbZFevNuPq1azWEiKPKgLXgL/NDkRkx8CBA80OQQjhgVxVOMXg/HFchcSfGT1M\nu4Rxt6mCk2VJ87K8V/P1119Tt27drFbzCGPGjGHKlClmhyHSOH7tOMN+G0aQXxAB3wYwdepUs0MS\nGdi/fz8DBw4sVNeFwkiude6vMOUo6bqQHa4qnHYC7ZVSxdM0EG8O6MTl6WittVJqD9DEyeJmwFEn\nDc7TqVu3LsHBwbkI2/2sWLECX19fs8MQqURfjaZnZE/uLnE3a4esxecpH8mRByhM14XCSK517s+q\nOXLV+1jfYhRpzyTNSOxJPAzYnNQVgVKqslLqHifbNlVKBafa9h6gI7CggON2O1b8n9SdXbl5hYe+\nfoh4Hc9PA36iVLFSkiMh8oGcR+7PqjlyyR0nrfVWpdRC4B2lVDlSeg4PwujRMclXQFscC7oZwNMY\nvYy/j9Ef1BiMR3QfFHz0Qjh3484NQueGcvqf0/w69Fcql7Du+IdCCGEVrhx/5EmMseoGAoEYY9U9\nrLX+LdU6GkhIvZHW+ppSqh3GUC2vkDJW3b+01nkYdU2I3ItPiKf/4v5si97GmkFrqFe2ntkhCSGE\ncAGXdZ2otb6ttX5Za11Ra+2rtW6edoBfrXUHrXW6Yk5rHa217qu1DtRal9BaP6a1Puqq2N3JuHHj\nzA5BAOuOr2P5weUs6LOAFpVbOCyTHAmRd3IeuT+r5sgzRrwVyapUqWJ2CALoVL0TB0cdpGrJqumW\nSY6EyDs5j9yfVXMkg3V4mFGjRpkdgkjkrGgCyZEQ+UHOI/dn1RxJ4SSEEEIIkU1SOAkhhBBCZJMU\nTh7mwIEDZocgsiA5EiLv5Dxyf1bNkRROHmb8+PFmh2ApJ/8+ydVbORvoUHIkRN7JeeT+rJojKZw8\nzPTp080OwTL+uvYXHWd15Knvn8rRdpIjIfJOziP3Z9UcSeHkYaz6+qerXb11lW5zuhF7J5Z3O72b\no20lR0LknZxH7s+qOZJ+nIRI43b8bXou6MnhS4dZH7aeaoHVzA5JCCGEm5DCSYhUEnQCg5cOZsOJ\nDawcuJKG5RuaHZIQQgg3Io/qPExERITZIRRaWmvG/DSG+XvnM6fnHNpXbZ+r/UiOhMg7OY/cn1Vz\nJHecPMz169fNDqHQmvH7DD7a+hEzus2gV71eud6P5EiIvJPzyP1ZNUdKa212DAVGKRUMbN++fTvB\nwcFmhyPc3NlrZ1l2YBnPNnnW7FBEAYqKiqJx48bIdUEIkSTpugA01lpHZbauPKoTIlH54uWlaBJC\nCJEpKZyEEEIIIbJJCicPc+HCBbNDEFmQHAmRd3IeuT+r5kgKJw8zdOhQs0MQWZAcCZF3ch65P6vm\nSAonDxMeHm52CB6voF+IkBwJkXdyHrk/q+ZICicPI28B5c31O9fpNLsTi/cvLrBjSI6EyDs5j9yf\nVXMkhZOwjLiEOPp+25etZ7ZSKaCS2eEIIYTwQNIBprAErTXPfP8MPx3+iR/6/cADFR8wOyQhhBAe\nSO44eZjIyEizQ/BIr6x9hS92fsGXj35Jl5pdCvRYkiMh8k7OI/dn1RxJ4eRhoqIy7dBUOPHh5g95\n59d3+CDkAwbUH1Dgx5McCZF3ch65P6vmSAonD/Pxxx+bHYJHmbd3HqNXjmZ8y/GMaTHGJceUHAmR\nd3IeuT+r5kgKJ1GoHbt8jMENBvPug++aHYoQQohCQBqHi0JtQpsJaK1RSpkdihBCiEJA7jiJQk+K\nJiGEEPlFCicPExoaanYIIguSIyHyTs4j92fVHEnh5GFGjhxpdggiC5IjIfJOziP3Z9UcSeHkYUJC\nQswOQWRBciRE3sl55P6smiMpnITHi0+I598b/831O9fNDkUIIUQhJ4WT8Ghaa15Y8QLjV49ny+kt\nZocjhBCikHNZ4aSUKqGU+kwpdU4pdU0ptVYp1Sgb2ymlVJhSaplS6mTitnuUUq8opbxdEbs7Wbp0\nqdkhuJW3NrzFjG0z+Kz7Z3So1sHscADJkRD5Qc4j92fVHLmkcFLG++A/Ak8AHwHjgLLAOqVUjSw2\n9wVmAmWA/wAvAluA1xP3aSkRERFmh+A2Ptv+GZPWTeKtDm8xLHiY2eEkkxwJkXdyHrk/q+bIVR1g\n9gFaAL201ksAlFILgYMYBdDATLa9DbTUWm9ONS9SKXUCCFdKddRary2guN1O2bJlzQ7BLSzZv4Th\ny4czsulIJraZaHY4DiRHQuSdnEfuz6o5ctWjul7A2aSiCUBrfQFYADyqlCqS0YZa6ztpiqYkSwAF\n1M3vYIV723BiA/0W9aN3vd5MfWiqdHAphBDCZVxVODUCnA2jvBXjUVztXOyzQuLPC7kNSnie+IR4\nRiwfQasqrZj92GzsNrvZIQkhhLAQVz2qqwCsdzI/JvHn3cC+HO5zPPA3sCIPcQkPY7fZ+XHAj5T0\nKYm3l+XeDRBCCGGyHBdOiQ29i2ZnXa31rcR/FgNuOVnlJsbjtmI5jGEi0BEYrrX+J5NVfQD279+f\nk927ta1btxIV5ezmnbVccOMbjZIj95Z0PShM14XCSM4j91eYcpTqeuCT5cpa6xxNQDsgIRtTPFA7\ncZurwOdO9tU1cb3OOTh+38RtPs3Guv0BLZNMMskkk0wyyZSNqX9WtUVuHtUdAMKyuW5Mqp8VnCxP\nmhednZ0ppToDs4DvgeHZ2GQlMAA4jnF3SwghhBAiLR+gKkbdkCmVeGemQCmlFgCttdZ3p5n/GdAP\nKKW1vpPFPh4A1gA7MO5QOXv0J4QQQghRYFz1Vt23QDmlVM+kGUqpMkBv4LvURZNSqrpSqnrqjZVS\ndYHlwFHgESmahBBCCGEGV91xsgG/AvcC72N0ITACqAI00VofSrXucSBBa1098XNx4A+Mx3oTSf9Y\n70gG/TwJIYQQQuQrlxROYIxVB7wHPIbxFt1WYKzWekea9Y5hFE41Ej8HYdxpysgsrfXQgolaCCGE\nECKFywonIYQQQghP56o2TkIIIYQQHk8Kp0JEKfVfpVSCUuo7s2MRBqVUR6VUpFLqT6VUrFLqiFLq\nc6VUebNjsxqlVFGlVIRS6rRS6rpSarNS6kGz4xIGpVQTpdR0pdRepdQ1pdQJpdR8pVQts2MTziml\nXk38nbPb7FhcSR7VFRJKqcbAJuAOsEZrHWpySAJQSv0OBAILgUNAdWAUEAs01FqfMzE8S1FKzQN6\nAFOAwxj90T0AtNdabzQxNAEopRYCLTHOld1AeYxzpTjQTGv9h4nhiTSUUhUx+nXUwHGtdX2TQ3IZ\nKZwKCaXUbxhvHz4I7JHCyT0opVprrX9NM68NxtiNb2mtJ5kTmbUk9gO3GXhJaz0lcZ43sBf4S2vd\n2sz4BCilmgPbtNZxqebVxMjRAq31INOCE+kk/iFSGmPottJWKpzkUV0hoJQahNHVwytmxyIcpS2a\nEuf9AlwC6ro+IsvqDcQBnyfNSOwPLhJokfjXszCR1npz6qIpcd5hjMJJzhU3opRqC/QExpgdixmk\ncPJwif1cvQO8LY99PINSyg/j8YP7jlRc+DQEDmqtr6WZvzXVcuGeyiHnittI7JfxI4zxZ/eaHY8Z\ncjNWnXAvrwE3gKlmByKybQxQBJhndiAWUoGUsTNTiwEUcLeTZcJkSqmBQEXgVbNjEcmGY3Re3dHs\nQMwihZObUEopoGh21k0ackYpVRt4Aeib1Vh/Iu9ykyMn+2gLTALma63X52N4InPFAGc5uZlquXAj\nSqk6wHTgN2C2yeEIQClVCngdeENrfcnseMwij+rcR1uMO0dZTdcTCyYw7jL9prVe6vpwLSk3OUqW\n+ItgMcYbQ0+7KGZhuAF4O5nvk2q5cBNKqbswxie9DPTR8haTu3gbuIhR0FqW3HFyHwcwXo/Ojhil\nVEfgIaBH4rA0YDxy8AKKJc67pLW+mu+RWleOcpT6g1KqMrAK4xfBw1rr2PwNTWQhBueP4yok/kw7\nBqYwiVIqAFgJBACttdZnTQ5JkPyG49PAi0BF4wY8CuOPjyKJv3P+0VpfNi9K15DCyU1orf8iB7ej\nE38Ra2BJ2l1htAk4itGW5qP8itHqcpqjJIm3t1dhtGtqn7gf4Vo7gfZKqeJpGog3xzhndpoTlkgt\nsYuI74GaQCet9Z8mhyRSVMQolD4CpjlZfhT4EPiXK4Myg/Tj5KGUUpWAYCeLPgeOA28Be7XWx1wZ\nl3CklPIFfgbuwSia5Be0CVL14zRWa/1B4ryiGK+6n9datzIzPpH8ttYSjDvpoVrrlSaHJFJRSpUG\nnJ0nb2O8JfwCcFRrvc+lgZlACqdCRil1DOkA020opZYCoRj9Ba1Ls/ia1nqZy4OyKKXUfOAxjLaB\nST2HNwE6aq1/MzE0ASilpmL88v0Oo/dwB1rrb1welMiSUupnLNYBphROhYxS6ihG4fSo2bGI5EK2\nSgaLT2itq7syHitLvMP0JjAQYxic3cCrWuvVpgYmgORfwG0zWq61trswHJFNiXkrpbVuYHYsriKF\nkxBCCCFENkl3BEIIIYQQ2SSFkxBCCCFENknhJIQQQgiRTVI4CSGEEEJkkxROQgghhBDZJIWTEEII\nIUQ2SeEkhBBCCJFNUjgJIYQQQmSTFE5CCCGEENkkhZMQQgghRDZJ4SSEEEIIkU1SOAkhhBBCZNP/\nB6nqEf62Hbe0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1064ebc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "plt.plot(z, logit(z), \"b-\", linewidth=2)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"sigmoid_saturation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha*z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure leaky_relu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGFCAYAAAALnnwfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl8VNX9//HXh4BAWETABalg1a+VL2r5Jt+qtNatiop2\nVEQQRQUsVmVRUHD5VolCi2BRfiKK2lhKqWERRbpY0boVl2oTFaVQpeJStGpEQQwCSc7vjzMhC1kn\nN3Pnzryfj8c8uOfOnZnPzSfn5sO9Z8415xwiIiIi0rBWYQcgIiIiEhUqnEREREQaSYWTiIiISCOp\ncBIRERFpJBVOIiIiIo2kwklERESkkVQ4iYiIiDSSCicRERGRRlLhJCIiItJIKpwko5hZnpmVm9lx\nYccizWdmB8fzeX/YsVQws4XxmPYPO5YKZnaUmT1lZp/FY3sl7JiaysxWmdnOsOMQUeEkLc7MescP\n1n8KOxbAxR9JY2aXxPe/6qPEzP5pZneZ2b4Bfc57ZlbSiO0qio2VzdkmWczs32b2dj2bJDWnZnZp\n/GdzQT3xlCcrnoaY2Z7AH4Ec4CEgD0iZQrOCmU2L/1y/X8cmKfVzlczVOuwARDLIU8Cq+HI34EfA\nWOAsM8txzn3ezPdP1xtP1rdf7wF9gC+TE8ou9cV0LTAV+E+SYmnIMfjft0nOuVlhB1OPhgrgYUD7\nJMUiUicVTiLJ85RzbmbVFWb2B+B0fAF1SyhRRZhzrgyo72xUS7D6nnTOfQJ8kqRYGqMnviD5OOxA\nGtDQz/XfyQpEpD66VCcpyczOMrO/mNkmM9tmZm+a2TVm1qrGdp3N7Doze9bMNprZ9vi/vzGzg5rw\neYfHLwl9bmbfN7OT45cN5tSx/UHx5x9v5q7Ox//ByK3jcw40s1+Z2ftm9o2ZfWRmvzazXs383ECZ\n2f5mdquZvWxmn8RjfdfM5phZ9zpes4eZTTSzV81si5l9ZWZvmdntZtap4nIhsD9wSI1LnTfG32O3\nMU5m9ryZldY1xsjM7o2/5vgqcYw3syfM7IN47J+Y2cNmdmSN1/6WystcC6vEs6PKNnWOcYpf5vub\nmW2N7+9LZnZRLdv9qGI/zex78b6wxcy+iMd1QCNykhX/+f0K/ztWNd4L4tvUeRnUahlTVPVympkN\nN7PX4/3zIzO7w8za1vFeJ5jZY1V+Nz4ws6Vmdkz8+b8CN8Y3X1UlzrervEetY5zMrLWZXWtmb5i/\nBP5l/Oc1sJZtd11mNbPTzexFM/va/NivB82sS0M/VxGdcZKUY2a/AK4H/g0sAzYDxwG3A0cBQ6ts\n3gc/ZuMZ4BHga+Aw/Gn9geYvgX3YwOcdC/we2AIc65xbG1//L+BCM5vknPumxstG4/8X39yxIhX/\nyy6tJa6jgSfwlyf+ALwDHAhcAJxuZsc4595r5ucH5UTgKuBp4CX8/vwPMAY4xcz+1zm3tWJjM2sP\n/AV/GemfQD6wE/gv4Arg1/gzJHnANfHn/h+VP6/n64nlN8AP8D+nX1Z9wszaAOcB7zvnnouv3huY\nFX/PP+Av+x0MxIDTzOxY59zr8W2XAZ2AH+N/31bH15dV+ZhaLzmZ2T3A5cAH+N+bVsBg4DdmdqRz\nblIt+3IM8DP8Zd55+AJ7ENA3/pr6BkuX439+OfF4H60Sb8W/9V0aq20/KtZNAAYAj8VjGwhcDewF\njKyx3xPxfffreAwf4s+CHRffl5fx+S8HjgUexP+MADbVF4+ZWfw9zwDWAXPw+RkK/MHMxjnn5tay\nD+fGY16Bv3x+AjAC+Db+d1mkbs45PfRo0QfQG39Q/FMjtj0lvu0fgXY1nrsH/wfqnCrrOgFdanmf\n4/F/vO+rsX5K/D2Oi7dj+AP6GuBbNbadFN/2ohrrs4CN+D/sWY3Yp0vi+zS5xnoD/hT/jAk1nmsN\nbMD/ET+yxnPfxxcSj9VYvwEoaUQ8B8fjWdmcbWps3x1oX8v6EfH3mVRj/ez4+gcAq/Fc56rvhf9D\n+3YDcd5f4/UlwOu1bH92fPtpVda1BfatZdu+wFbgjzXWXxrP2QV1xPTb+PP7V1l3Yvxz3wA6VFnf\nBV8QlwHHVFn/o/j2ZcDZNd7/d/H1gxqZmzrjbeBn+1dgR411U+NxFQMHVVnfLr4fO4G9q6z/H3w/\nfA/oWctn7FfjvcuA7zchnlHxeJ6gSl8EesVj/AY4oMbPojy+/ns1+uLz8c/PaczPVY/MfehSnaSa\nsfj/Ef7U7X6W5/r4v8MqVjjnvnLO7TYw2PmzCWuAk+v6IDO7FH8G4Q3gh273MRS/xv8h+EmN9WcC\nPYD5zo+xaaxTzGxK/PH/gLeAU4EX8GcTqvoxvuC83Tm3uuoTzrkX8f/TH2hmHZvw+S3GOVfsnNtW\ny1ML8IXprjyYWWv8H7BNwETnXLWzCM65LXW8V2Nj2YI/g3iEmR1e4+mL8L9fv6uy/XbnxyXVfJ81\nwHPACfEzG80xIv65Nzvnvq7yGV8Ct+L/cF9Sy+v+4pxbXmPdg/Htv9fMmBLlgDucc+/uWuH76iL8\nWbScKttegY/1Rufcxt3eyLnmDqC/JB7P5Kp90Tn3Af4MZRv8mceaFjjnXq2yvcOfqYTwfq4SEbpU\nJ6nmaPwf2ktr+VtlwDb8pbjKlWYn4C8THIU/81H193p7HZ8zEX+26XFgcG1/qJ1zxWb2CDDUzA51\nzlWMt/gJ/mCd3/jdAuCk+KOqF4CTnXM7aqw/Ov4Zh5nZlFreaz/8H6lDgaImxtEizGwwcBnQD+hK\n9TGUVcf79AU6AM87575qoXB+i78kdxFwXTy+PfGXZ4pc/HJsldhzgMn4s3n74v/gVnD4/WnOtx77\nxf99rpbnnqmxTVW15baiwA9zPE5j46ooQp5soTj6AV85596o5bln8F+4aMrP1Qj35yoRoMJJUk1X\n/KWwm+vZJrtiwczOw/9P9yv86fr38JdpHH6sRW2DqA0/lsLhL0XVd3bjPuB8fLE02cx6AKcBzzrn\n1jdul3a53jl3ezzuA/HjTy7GD969uMa2XeNx1jVXEPH4OzQxBqicC6e+M84VzzVq3hwzuw6Yjv82\n2RP4S0AVZwyvwV8Oq7Bn/N/dzkAE6M/4SzXDiBdO+HEvbfFF1S7mJ0Ndib9MsxJYj79EVzEW5nCq\nx5+IzkBpbWdHqfy2W+danttSy7qK8XBZzYypORob1574/f6sheLoiM9Xber6uTpS9+cqEaDCSVLN\nFqDcObdPI7fPw5+Fyql66QDAzIbV+gp/4LwUP+j2TjMrc87dXeuGzj1nZuuAi81/k2sUvqh4oJHx\nVQupyvu+B4yIF1AXmtnDzrkVVbbdEo/zTOdcc7+5V9Pm+L/d6tmm4ptwm+vZBtg14Pr/8MVSP+fc\nF1WeawXcUOMlFcVDz0ZFmwDnXKmZLQLGmNmJzrln8GefSvGFdlX/hz/D9MOql28AzOyHAYW0Behl\nZl1qKZ72q7JNspVT99+BPetY3xRfAr3NbO8WKp624s8Q1ibMn6ukMY1xklTzN6CbmR3cyO0PAtbW\nUjT1iD9Xly/wA3D/DtxlZuPq2fZ+/DevzsGfxfoC/42qIFwV/3d6jXE0f8MXWnXNopww59wm4CP8\nZcDaznJQ5XNX1/F8Vfvg/+f/YtWiKe5odj9b8w/8H7yjzKxTI96/jMTOAvwW/zMcbma98fv0pHPu\n0xrbHQR8WkvR1IHaL/NUjKVpSkyvxf89oZbnTqyxTTJ9AexXcwxXfOzcIQG8f8WtXQY0YttEf66d\nzKy2PFX8XF+v5TmRhKlwklRzF/6P3YNm1rXmk2a2r5lVHeP0Pn6On72rbNMWuJfq41R2Ex9EfDK+\nSPl/Zja+jk1/gx8rdSf+68oLahmTlJD42Izl+HFbVS/LPYb/SvbE2s56xOeu+UEzPnoBvqCZWfMJ\n83NEXYM/O/NQI97rP/ifT66ZtavyPl3xA3Srcc6V4s/YdcOf8av5R7uzmWVXWbUJ2Cd+ZqvR4oXQ\nP/GX2y6Lr/5tLZu+D3Q3s0OrxNAKn+/azsptwv+ONjiXUhW/ib8mL16QVXzOXvjL0g6fk2R7Ff97\ncH6VmAz/e9Gurhc1wTz8vv3Capl7ysz2q9Jszs/1NjPbVXDFf4evBnbQuN9hkUbTpTpJpiPM7Nd1\nPLfOOTfDOfeEmU3FX0Zbb2Z/xv9h64b/H/AP8ZdW1sVfNwdfbL1uZg/jf6dPiT/3BlBtAsOanHNb\nzGwAflzObDNr5ZybXWObL8xsKZXfyGrqoPCG5OG/Jn+zmRU458qdczvig63/BDxnZk8Db8Y/vzf+\n51AM/HeN92pTz88Y51zFHDtT8f8jHx2fx+op/DixA4Gz8OPIxjvnNjQUvHOuzMzmAePxefgj/jLP\n6fjxJ7XNov1/+MH8I4EfxPO8Ez+9wKnx5/4R3/Zp4LvA42a2Kr7ds865FxqKDVgY39dr8We5Hqtl\nmzn4Qfsvmdli/B/bE/GXgJ7H/6yrehFfKF5jZvsAn+EvL0+vKwjn3DNmdi9+Hqc18S8dVMzj1AOY\n5Zx7uRH7k6i6vhU4Bz++br6ZnY7/nToOP3buLWp8EaOpn+Oce93MrsHPk7XGzJbj/0PQI/45j+IH\n5YMfzO2AmfEzSJuBTc65e+v5vPn4uaAGAqvNz8TfGRiCH+Q93u0+j1tzvyEpmS7s+RD0SP8H/g99\nWQOPp2u85iT8mZj/4AcZb8RPVHcDNeaDwU9GuRr/bbyN+AHd3fEH4tIa21abx6nK+o7x999tTqUq\n8ZQDqxLY/0vi7zu5nm2WxrcZUWN9D+AOfKFYgr+08lZ8H0+ose2GBn7GNX8We+C/XfgyfizK9vjP\nbyl+ItCm7GNr/MzP/4zH+S5wG37yzg+Bf9bymjb4M1tF+KJmczyPtwGdqmzXCX+5dCO+aCrDf70d\nfKFVRo35umr87pXGt/l1PfGfi79s+1X8d25h/LW/jb9+/xrbn4E/U7k1/t7bqzxX62viz42q8rqv\n8JOFDq9lux/F3/eGWp6rd59r2b6headOjMdRAnyK/49BN/y8SdtrbFvnXEv1fQ7+EuXv8UXmNvyX\nOBYBR9XYbgT+Pzwl8fd6u8pzu8UTX58V/z2qeN2X+P8InNbEGOv8meuhR9WHOVdtChURqYWZXQvM\nAEY5537T0PYiIpKeVDiJNCA+Zuqf+LNS33K7T8wpIiIZQmOcROoQH3x9An7MzQH4eZhUNImIZDAV\nTiJ1Oxn/jadi/DijWeGGIyIiYdOlOhEREZFGSsoZJzPrhr/c8R6Vt2EQERERSQXt8NOxPOGcq/e+\nlMm6VHcqVe5GLiIiIpKCLqSBSVOTVTi9B7Bw4UL69OmTpI9MrlNOOYUnn2ypG4BLS1Heomnt2rUM\nHz48rY8p6Uz9Lrrqy91HH8Hw4bA5fofLkSNh7NgkBtcMFccU4vVKfZJVOH0D0KdPH3JycpL0kcl1\n+OGHp+2+pTPlLdrS+ZiSztTvoquu3JWUwKWXVhZNp58ODzwAWYncZTJcDQ4n0r3qArLnnkHcSFyS\nTXkTST71u+iqLXfOwejR8Hr8dsqHHAK/+10ki6ZGUeEkIiIiCbvjDngoPiqoY0dYvhz22ivcmFqS\nCicRERFJyFNPweTJle3f/Ab69g0vnmRQ4RSQ/fbbL+wQJAHKm0jyqd9FV9XcvfsuDB0K5eW+fdNN\nMGhQSIElUbMLJzP7mZmVm9nqIAKKqjZt2oQdgiRAeRNJPvW76KrI3ddfw9lnw6ZNfv2ZZ0JeXnhx\nJVOzCicz6wlcB2wNJpzomjt3btghSAKUN5HkU7+Lrrlz5+IcjBoFb77p133nO7BwIbTKkGtYzZ2O\nYBbwcvx9ujU/HBEREUllM2fCkiV+uVMnPxg8k74omXB9aGbHAYOACcGFIyIiIqnqz3+GG26obC9c\nCIcdFl48YUiocDKzVsBdwAPOubeCDUlERERSzfr1MGyYn7cJ/JimWCzUkEKR6BmnK4BewE0BxhJp\nsUz87UkDyptI8qnfRc/WrX4w+Jdf+tyddZb/Fl0manLhZGZdgVuAW51zm4IPKZrGRuWGPFKN8iaS\nfOp30eIcXHIJrFkDMJY+fWDBgswZDF5TIrv9c+Bz4O6mvnDgwIHEYrFqj/79+7N8+fJq261cubLW\n/5GMGTOG/Pz8auuKioqIxWIUFxdXWz9lyhRmzJhRbd0HH3xALBZj3bp11dbPmTOHSZMmVVtXUlJC\nLBZj1apV1dYXFBQwcuTI3WLLz89Pi/0YOnRoRu3HgAED0mI/ID3y0dT9mDdvXlrsR7rko7H7MWDA\ngLTYD0iPfDS0H5ddtopHHvHr9txzAD/5SQFXXRW9/ajIR0FBAbFYjNzcXHr16kUsFmPChMYP1zZX\ncbGyMRubHQKsA64C/lCxGigAugCnAVucc1/UeF0OUFhYWKgbO4pIsxUVFZGbm4uOKSIt649/hB//\n2J91MoPf/x7OOCPsqIJXcUwBcp1zRfVt29QzTj3xhdJdwIb4413gaOA78eUMveopIiKSPt5+Gy64\noHIw+NSp6Vk0NVVTC6e3gHPij7OrPNYA78eX8+t8dRqreXpSokF5E0k+9bvUt2WLHwy+ZYtvn3su\n3HijcgdNLJycc58751bUfADFwFfOud8759a0TKipraCgIOwQJAHKm0jyqd+ltvJyuPhiWLvWt/v2\nhfnz/aU65S7Ym/w2frBUGlq8eHHYIUgClDeR5FO/S23TpsFjj/nlLl38zOAdO/q2ctf8W64A4Jw7\nMYj3ERERkfCsWAFTpvjlVq1g0SI45JBwY0o1GToLg4iIiFS1bh0MH17Z/sUv4NRTw4snValwEhER\nyXCbN/vZwL/6yreHDIHJk8ONKVWpcApIbZOaSepT3kSST/0utZSX+zNNb7/t20ceCQ8+6AeD16Tc\nqXAKzIABA8IOQRKgvIkkn/pdasnLgz/Ep7Tu2hUefRQ6dKh9W+VOhVNghg0bFnYIkgDlTST51O9S\nxyOP+IktwQ8GX7wYDjqo7u2VOxVOIiIiGWnNGn/z3gozZ8LJJ4cXT1SocBIREckwX3zhZwbfutW3\nL7gAJk4MN6aoUOEUkJp3wZZoUN5Ekk/9LlxlZb5QWr/et/v1gwceqH0weE3KnQqnwMycOTPsECQB\nyptI8qnfheumm+DPf/bL3bv7mcGzsxv3WuVOhVNgFi1aFHYIkgDlTST51O/Cs3QpTJ/ul7OyYMkS\n6N278a9X7lQ4BSa7seW6pBTlTST51O/CsXo1jBhR2Z41C05s4g3TlDsVTiIiImlv0yY/GLykxLcv\nvhjGjw83pqhS4SQiIpLGSkvh/PNhwwbfzs2FefMaNxhcdqfCKSCTJk0KOwRJgPImknzqd8l1443w\n5JN+ee+9/czg7dsn9l7KnQqnwPTq1SvsECQByptI8qnfJU9BAdx+u19u3RoefhgOOCDx91PuVDgF\nZty4cWGHIAlQ3kSST/0uOV5/HS69tLI9ezYcd1zz3lO5U+EkIiKSdoqL/WDwbdt8e+RIuPLKcGNK\nFyqcRERE0khpKQwdCu+/79tHHQX33KPB4EFR4RSQdevWhR2CJEB5E0k+9buWNXkyPP20X953X3jk\nEWjXLpj3Vu5UOAVm8uTJYYcgCVDeRJJP/a7lLFwId97pl9u0gWXLoGfP4N5fuVPhFJi777477BAk\nAcqbSPKp37WMwkIYPbqyPWcO/OAHwX6GcqfCKTD6imY0KW8iyad+F7xPP4VzzoFvvvHt0aPhpz8N\n/nOUOxVOIiIikbZzJwwZAh9+6Nv9+/uzTdIyVDiJiIhE2DXXwHPP+eUePfy4prZtw40pnalwCsiM\nGTPCDkESoLyJJJ/6XXDmz688u7THHv4bdD16tNznKXcqnAJTUnHLaYkU5U0k+dTvgvHqq3D55ZXt\nuXPhmGNa9jOVOzDnXMt/iFkOUFhYWEhOTk6Lf56IpLeioiJyc3PRMUUy1SefQG4ubNzo21dc4Se5\nlMRUHFOAXOdcUX3b6oyTiIhIhOzYAYMHVxZNxx7r70MnyaHCSUREJEKuvhpWrfLLPXvCww/78U2S\nHCqcAlJcXBx2CJIA5U0k+dTvEverX8G99/rltm3h0Uf9bVWSRblT4RSYUaNGhR2CJEB5E0k+9bvE\nvPwyjBlT2Z43D773veTGoNypcApMXl5e2CFIApQ3keRTv2u6jz+GQYP8+CaAceNgxIjkx6HcqXAK\njL7ZE03Km0jyqd81zfbtcO65vngCOP54mDUrnFiUOxVOIiIiKW38eHjpJb98wAGwZAm0aRNuTJlM\nhZOIiEiKuu8+uP9+v9yunR8Mvs8+4caU6VQ4BSQ/Pz/sECQByptI8qnfNc4LL/ixTBUeeMBPehkm\n5U6FU2CKiuqdaFRSlPImknzqdw3buNGPa9q507cnTIDhw8ONCZQ7UOEUmLlz54YdgiRAeRNJPvW7\n+n3zjf8G3Sef+PZJJ8HMmeHGVEG5U+EkIiKSMpzzczW98opv9+4NixdD69bhxiWVVDiJiIikiHvu\ngQcf9Mvt28Py5dC9e7gxSXUqnERERFLA88/7+9BVyM+Hfv3Ci0dqp8IpILFYLOwQJAHKm0jyqd/t\n7sMPYfBgKC317UmTYNiwcGOqjXKnwikwY8eODTsESYDyJpJ86nfVbdsG55wDn33m26ecAtOnhxtT\nXZQ7FU6BGTBgQNghSAKUN5HkU7+r5BxcfjkUFvr2QQfBokWQlRVuXHVR7lQ4iYiIhOauu2DBAr+c\nne0Hg3ftGm5MUj8VTiIiIiF45hm45prK9vz5cMQRoYUjjaTCKSDLly8POwRJgPImknzqd/D++zBk\nCJSV+fb118N554UbU2ModyqcAlNQUBB2CJIA5U0k+TK935WU+MHgxcW+fdppMG1auDE1VqbnDlQ4\nBWbx4sVhhyAJUN5Eki+T+51zMHo0vPaabx9yCDz0UOoOBq8pk3NXQYWTiIhIktxxhy+UADp29IPB\n99or3JikaVQ4iYiIJMFTT8HkyZXt3/wG+vYNLx5JjAonERGRFrZhAwwdCuXlvv2zn8GgQeHGJIlR\n4RSQkSNHhh2CJEB5E0m+TOt3X38NZ58Nmzb59plnwi23hBtTojItd7VR4RQQzaYaTcqbSPJlUr9z\nDkaNgtWrffs734GFC6FVRP/6ZlLu6hLR1KWeYal4N0ZpkPImknyZ1O9uvx2WLPHLnTr5weB77hlu\nTM2RSbmrS5MKJzP7bzNbYmb/MrOvzewzM3vOzM5sqQBFRESi6M9/9hNbVli4EA47LLx4JBitm7h9\nb6AjMB/4CMgGzgVWmNllzrlfBRueiIhI9KxfD8OG+Ut1AHl5EIuFGpIEpElnnJxzjzvnBjrnpjrn\n8p1zc4ATgTeAiS0SYUSsWrUq7BAkAcqbSPKle7/butUPBv/yS98+6yy46aZwYwpKuueuMZo9xsk5\n54APgS7NDye6Zs6cGXYIkgDlTST50rnfOQcjRsCaNb592GGwYEF0B4PXlM65a6ymXqoDwMyygfbA\nnsBZwOlARt/AZtGiRWGHIAlQ3kSSL5373fTpsGyZX+7cGR57zP+bLtI5d42VUOEEzAJ+Gl8uB5YB\n4wKJKKKys7PDDkESoLyJJF+69rs//tFPbAlg5m+tcuih4cYUtHTNXVMkWjjdCSwF9geGAFlA26CC\nEhERiZK334YLL6wcDD51KpxxRrgxSctI6Kqrc+5t59zTzrmFzrkY/pt2f2jodQMHDiQWi1V79O/f\nn+XLl1fbbuXKlcRq+frBmDFjyM/Pr7auqKiIWCxGcXFxtfVTpkxhxowZ1dZ98MEHxGIx1q1bV239\nnDlzmDRpUrV1JSUlxGKx3QbCFRQU1Dpz6tChQ7Uf2g/tR5L3Y968eWmxH+mSj0zdj/HjJ3H22bB5\ns1931lkl/O1v0duPdMlHQ/tRUFBALBYjNzeXXr16EYvFmDBhwm4x1sk51+wHMBooA/6rjudzAFdY\nWOjS1bXXXht2CJIA5S2aCgsLXbofU9JZOvW7sjLnzjrLOX+uybm+fZ376quwo2o56ZS7qiqOKUCO\na6DmCWqcf/v4vxGeD7V5evXqFXYIkgDlTST50qnfTZvmB4ADdOniZwbv2DHcmFpSOuUuUU2dOXzv\nWta1Bi4BtgH/CCiuyBk3LqPHxkeW8iaSfOnS71asgClT/LIZFBTAIYeEG1NLS5fcNUdTB4ffZ2ad\ngeeBjcB+wIXAd4CJzrmSgOMTERFJOevWwfDhle3p0+G008KLR5KnqYXTIuBS4HKgG/AVUAhMcs79\nMeDYREREUs7mzX428K++8u0hQ2Dy5HBjkuRp6i1XljjnTnXO7e+ca+uc6x5vZ3zRVHNkv0SD8iaS\nfFHud+Xl/kzT22/79pFHwoMP+kt1mSDKuQtKmkwCH77J+u9GJClvIskX5X6Xlwd/iE++07UrPPoo\ndOgQakhJFeXcBUWFU0DuvvvusEOQBChvIskX1X73yCN+Ykvw955bvBgOOijcmJItqrkLkgqngOgr\nmtGkvIkkXxT73Zo1cMklle2ZM+Hkk8OLJyxRzF3QVDiJiIjU44sv4OyzYetW377gApg4MdyYJDwq\nnEREROpQVubvQbd+vW/36wcPPJA5g8FldyqcAlLzPjoSDcqbSPJFqd/ddBM8/rhf7t7dzwyenR1u\nTGGKUu5aigqngJSUaO7PKFLeRJIvKv1u6VI/sSVAVhYsWQK9e4cbU9iikruWZM7fhLdlP8QsBygs\nLCwkJyenxT9PRNJbUVERubm56JgiLWX1aujfHyrqhNmz4aqrwo1JWk7FMQXIdc4V1betzjiJiIhU\nsWmTHwxeUTRdfDGMHx9uTJI6VDiJiIjElZbC+efDhg2+nZsL8+ZpMLhUUuEUkOLi4rBDkAQobyLJ\nl8r97sYb4ckn/fLee/uZwdu3DzemVJLKuUsWFU4BGTVqVNghSAKUN5HkS9V+t2gR3H67X27dGh5+\nGA44INyYUk2q5i6ZVDgFJC8vL+wQJAHKm0jypWK/e/11qFoTzJ4Nxx0XXjypKhVzl2wqnAKib/ZE\nk/Imknyp1u+Ki+Gcc2DbNt8eORKuvDLcmFJVquUuDCqcREQkY5WWwtCh8N57vn3UUXDPPRoMLnVT\n4SQiIhld0YwRAAAgAElEQVRr8mR4+mm/vO++8Mgj0K5duDFJalPhFJD8/PywQ5AEKG8iyZcq/W7h\nQrjzTr/cpg0sWwY9e4YbU6pLldyFSYVTQIqK6p1oVFKU8iaSfKnQ74qKYPToyvZdd8EPfhBePFGR\nCrkLmwqngMydOzfsECQByptI8oXd7z791M8M/s03vj16NPz0p6GGFBlh5y4VqHASEZGMsXMnDBkC\nH37o2/37w5w5GgwujafCSUREMsa118Jzz/nlHj38uKa2bcONSaJFhZOIiGSE+fP9WCaAPfbw36Dr\n0SPUkCSCVDgFJBaLhR2CJEB5E0m+MPrdq6/C5ZdXtufOhWOOSXoYkadjpgqnwIwdOzbsECQByptI\n8iW7333yiZ8ZfPt2377iCvjJT5IaQtrQMVOFU2AGDBgQdgiSAOVNJPmS2e927IDBg2HjRt8+9lh/\nHzpJjI6ZKpxERCSNTZgAq1b55Z49YelSP75JJFEqnEREJC3l5/v7zoH/5tyjj8J++4Ubk0SfCqeA\nLF++POwQJAHKm0jyJaPfvfwyXHllZXvePPje91r8Y9OejpkqnAJTUFAQdgiSAOVNJPlaut99/DEM\nGuTHNwGMGwcjRrToR2YMHTNVOAVm8eLFYYcgCVDeRJKvJfvd9u1w7rm+eAI4/niYNavFPi7j6Jip\nwklERNLI+PHw0kt++YADYMkSaNMm3JgkvahwEhGRtHDffXD//X65XTs/GHyffcKNSdKPCicREYm8\nF17wY5kq3H8/5OaGF4+kLxVOARk5cmTYIUgClDeR5Au6323c6Mc17dzp2xMmwEUXBfoREqdjpgqn\nwGg21WhS3kSSL8h+9803/ht0n3zi2yedBDNnBvb2UoOOmSqcAjNs2LCwQ5AEKG8iyRdUv3MOxoyB\nV17x7d69YfFiaN06kLeXWuiYqcJJREQi6p574MEH/XL79rB8OXTvHm5Mkv5UOImISOQ8/zxcfXVl\nOz8f+vULLx7JHCqcArKq4i6SEinKm0jyNbffffghDB4MpaW+fe21oCtIyaFjpgqnwMzUaMRIUt5E\nkq85/W7bNjjnHPjsM98+5RSYPj2gwKRBOmaqcArMokWLwg5BEqC8iSRfov3OObj8cigs9O2DDoJF\nizQYPJl0zFThFJjs7OywQ5AEKG8iyZdov5szBxYsqHgPPxi8a9cAA5MG6ZipwklERCLgmWdg4sTK\n9vz5cMQRoYUjGUyFk4iIpLT334chQ6CszLevvx7OOy/cmCRzqXAKyKRJk8IOQRKgvIkkX1P6XUmJ\nHwxeXOzbp50G06a1UGDSIB0zVTgFplevXmGHIAlQ3kSSr7H9zjkYPRpee823Dz4YHnoIsrJaMDip\nl46ZKpwCM67qbbklMpQ3keRrbL+7805fKAF06ACPPQZ77dWCgUmDdMxU4SQiIinoqaeg6lWhBQug\nb9/w4hGpoMJJRERSyoYNMHQolJf79s9+BoMGhRuTSAUVTgFZt25d2CFIApQ3keSrr999/TWcfTZs\n2uTbZ54Jt9ySpMCkQTpmqnAKzOTJk8MOQRKgvIkkX139zjkYNQpWr/btQw+FhQuhlf5SpQwdM1U4\nBebuu+8OOwRJgPImknx19bvbb4clS/xyp05+ZvA990xiYNIgHTNVOAVGX9GMJuVNJPlq63dPPOEn\ntqywcCH06ZPEoKRRdMxU4SQiIiFbvx7OP99fqgPIy4NYLNSQROqkwklEREKzdasfDP7ll7591llw\n003hxiRSnyYVTmb2v2Z2t5m9ZWZbzex9M1tsZv/VUgFGxYwZM8IOQRKgvIkkX0W/cw5GjIA1a/z6\nww7z8zVpMHjq0jETWjdx++uA7wNLgdXAfsA4oMjMjnbO/SPg+CKjpKQk7BAkAcqbSPJV9Lvp02HZ\nMr+uc2c/M3jnziEGJg3SMRPMVVxUbszGZscAf3fOlVZZdwjwFrDEOXdxHa/LAQoLCwvJyclpZsgi\nkumKiorIzc1Fx5To+uMf4cc/9medzOD3v4czzgg7KslUFccUINc5V1Tftk064+Sce7mWdevN7C1A\n338QEZEGvf02XHhh5WDwqVNVNEl0BHUleV+gOKD3EhGRNLVlix8Mvnmzb597Ltx4Y7gxiTRFswsn\nMxsO9AQWNT+c6CouVt0YRcqbSPKUl8Mll8Datb7f9e0L8+f7S3USDTpmNrNwMrPDgLuBF4AFgUQU\nUaNGjQo7BEmA8iaSPNOm+dnAYRRduvjljh3DjkqaQsfMZhROZrYP8EfgC+A814hR5gMHDiQWi1V7\n9O/fn+W+J+2ycuVKYrXMfjZmzBjy8/OrrSsqKiIWi+1WBU+ZMmW3r01+8MEHxGKx3W5SOGfOHCZN\nmlRtXUlJCbFYjFWrVlVbX1BQwMiRI3eL7ZtvvkmL/Rg6dGhG7UdeXl5a7AekRz6auh/z5s1Li/1I\nl3zUtx/33LOOKVMq1uZx/vlzuO++6O1HuuQj0f3Iy8uL/H4UFBQQi8XIzc2lV69exGIxJkyYsFuM\ndWnSt+p2vcisM/Ac8C3gWOfcPxvYXt+qE5HA6Ft10bJuHRx1FHz1lW/fdhtcd124MYlU1WLfqgMw\ns7bA74FDgB81VDSJiEjm2rzZzwZeUTQNGQKTJ4cbk0hzNKlwMrNWwBLgGCDmnHulRaISEZHIKy+H\n4cP99AMARxwBDz6oweASbU0d43QH8GPgcaC7mV1Y9RF8eNFR81qtRIPyJtJy8vLgD3/wy127+sHg\nHTqo30WZctf0wum7gMMXTwtqeWSsoqJ6L4lKilLeRFrGo4/6iS3B33tu8WI46CDfVr+LLuWu6TOH\nn9hSgUTd3Llzww5BEqC8iQRvzRq4uMoNuGbOhJNPrmyr30WXchfczOEiIiJ88YWfGXzrVt++4AKY\nODHcmESCpMJJREQCUVbm70G3fr1v9+sHDzygweCSXlQ4iYhIIG66CR5/3C936+bHOWVnhxuTSNBU\nOAWktplRJfUpbyLBWLoUpk/3y1lZsGQJHHhg7duq30WXcqfCKTBjx44NOwRJgPIm0nxvvgkjRlS2\nZ82Ck06qe3v1u+hS7lQ4BWbAgAFhhyAJUN5EmmfTJj8YvKTEty++GMaPr/816nfRpdypcBIRkQSV\nlsL558O77/p2bi7Mm6fB4JLeVDiJiEhCbrwRnnzSL++9tx8M3r59uDGJtDQVTgFZvnx52CFIApQ3\nkcQsWgS33+6XW7eGhx+GAw5o3GvV76JLuVPhFJiCgoKwQ5AEKG8iTff66zBqVGX7zjvhuOMa/3r1\nu+hS7lQ4BWbx4sVhhyAJUN5Emqa4GM45B7Zt8+2RI2HMmKa9h/pddCl3KpxERKSRSkth6FB47z3f\nPuoouOceDQaXzKLCSUREGuW66+Dpp/3yvvvCI49Au3bhxiSSbCqcRESkQQsXwh13+OU2bWDZMujZ\nM9yYRMKgwikgI0eODDsESYDyJtKwoiIYPbqyfddd8IMfJP5+6nfRpdypcAqMZlONJuVNpH6ffupn\nBv/mG98ePRp++tPmvaf6XXQpdyqcAjNs2LCwQ5AEKG8iddu5E4YMgQ8/9O3+/WHOnOYPBle/iy7l\nToWTiIjU4dpr4bnn/HKPHn6Sy7Ztw41JJGwqnEREZDfz5/uxTAB77OG/Qbf//qGGJJISVDgFZNWq\nVWGHIAlQ3kR29+qrcPnlle25c+GYY4J7f/W76FLuVDgFZubMmWGHIAlQ3kSq++QTPzP49u2+fcUV\n8JOfBPsZ6nfRpdypcArMokWLwg5BEqC8iVTasQMGD4aNG3372GNh9uzgP0f9LrqUOxVOgcnOzg47\nBEmA8iZSacIEqLgS07MnLF3qxzcFTf0uupQ7FU4iIgLk5/v7zoH/5twjj8B++4Ubk0gqUuEkIpLh\nXn4Zrryysn3vvf4GviKyOxVOAZk0aVLYIUgClDfJdB9/DIMG+fFNAOPGQUvfVUP9LrqUOxVOgenV\nq1fYIUgClDfJZNu3w7nn+uIJ4PjjYdaslv9c9bvoUu5UOAVm3LhxYYcgCVDeJJONHw8vveSXDzgA\nliyBNm1a/nPV76JLuVPhJCKSke67D+6/3y+3awePPgr77BNuTCJRoMJJRCTDvPCCH8tU4f77ITc3\nvHhEokSFU0DWrVsXdgiSAOVNMs3GjX5c086dvn311XDRRcmNQf0uupQ7FU6BmTx5ctghSAKUN8kk\n33zjv0H3ySe+fdJJcPvtyY9D/S66lDsVToG5++67ww5BEqC8SaZwDsaMgVde8e3evWHxYmjdOvmx\nqN9Fl3Knwikw+opmNClvkinuvRcefNAvt28Py5dD9+7hxKJ+F13KnQonEZG09/zzcNVVle38fOjX\nL7x4RKJMhZOISBr78EMYPBhKS3372mth2LBwYxKJMhVOAZkxY0bYIUgClDdJZ9u2wTnnwGef+fYp\np8D06eHGBOp3UabcqXAKTElJSdghSAKUN0lXzsHll0NhoW9/+9uwaFE4g8FrUr+LLuUOzDnX8h9i\nlgMUFhYWkpOT0+KfJyLpraioiNzcXHRMqdtdd1WOa8rO9rdWOfLIcGMSSVUVxxQg1zlXVN+2OuMk\nIpJmnnkGJk6sbM+fr6JJJCgqnERE0sj778OQIVBW5tvXXw/nnRduTCLpRIVTQIqLi8MOQRKgvEk6\nKSnxg8Erfq1POw2mTQs3ptqo30WXcqfCKTCjRo0KOwRJgPIm6cI5GD0aXnvNtw8+GB56CLKywo2r\nNup30aXcqXAKTF5eXtghSAKUN0kXd97pCyWADh3gscdgr73Cjaku6nfRpdypcAqMvtkTTcqbpIOn\nnoJJkyrbCxZA377hxdMQ9bvoUu5UOImIRNqGDTB0KJSX+/bPfgaDBoUbk0g6U+EkIhJRX38NZ58N\nmzb59plnwi23hBuTSLpT4RSQ/Pz8sEOQBChvElXOwaWXwurVvn3oobBwIbSKwFFd/S66lDsVToEp\nKqp3olFJUcqbRNXtt8PixX65UydYvhz23DPcmBpL/S66lDsVToGZO3du2CFIApQ3iaInnvATW1ZY\nuBD69AkvnqZSv4su5U6Fk4hIpKxfD+ef7y/VAeTlQSwWakgiGUWFk4hIRGzd6geDf/mlb591Ftx0\nU7gxiWQaFU4iIhHgHIwYAWvW+PZhh/n5mqIwGFwknajLBSSmc+WRpLxJVEyfDsuW+eXOnf3M4J07\nhxtTotTvoku5U+EUmLFjx4YdgiRAeZMo+NOf/MSWAGb+1iqHHhpuTM2hfhddyl0ChZOZdTCzW8zs\ncTP73MzKzezilgguSgYMGBB2CJIA5U1S3dtvwwUXVA4GnzoVzjgj3JiaS/0uupS7xM44dQduAg4D\nXgdcoBGJiAgAW7b4weCbN/v2oEFw443hxiSS6Von8JqPgP2cc5+aWS7wasAxiYhkvPJyuOQSWLvW\nt/v2hfnz/aU6EQlPk884Oed2Ouc+bYlgomz58uVhhyAJUN4kVU2b5mcDB+jSxS936hRuTEFRv4su\n5U6DwwNTUFAQdgiSAOVNUtGKFTBlil82g4ICOOSQcGMKkvpddCl3KpwCs7jiplESKcqbpJp162D4\n8Mr29Olw2mnhxdMS1O+iS7lT4SQikjI2b/aDwb/6yreHDIHJk8ONSUSqS2rhNHDgQGKxWLVH//79\nd7tmunLlylon2RozZgz5+fnV1hUVFRGLxSguLq62fsqUKcyYMaPaug8++IBYLMa6deuqrZ8zZw6T\nJk2qtq6kpIRYLMaqVauqrS8oKGDkyJG7xTZ06FDth/ZD+5Hk/Zg3b15a7MeUKVO47bYZDB8O//yn\nX/ed73zA1q0x/vnPaO1HuuRD+5G++1FQUEAsFiM3N5devXoRi8WYMGHCbjHWxZxLfDaBKt+qG+Gc\nW1DPdjlAYWFhITk5OQl/nogI+ANsbm4u6XRMuflmP0cTwF57wd//DgcdFG5MIpmi4pgC5Drniurb\nVpfqAlJb9S2pT3mTVPDoo5VFU6tWsHhxehdN6nfRpdypcAqMZlONJuVNwvaPf8DFVe69MHMmnHJK\nePEkg/pddCl3iU2AiZmNAboAPeOrYmZ2QHz5LufcV0EEFyXDhg0LOwRJgPImYfrySzjrLNi61bcv\nuAAmTgw3pmRQv4su5S7Bwgm4FugVX3bAOfEHwG+BjCucRESaoqzMF0rr1/t2v37wwAOaGVwk1SVU\nODnnvh10ICIimeTmm+Hxx/1yt25+nFN2drgxiUjDNMYpIDW/rinRoLxJGJYuhV/8wi9nZcGSJXDg\ngaGGlFTqd9Gl3KlwCszMmTPDDkESoLxJsr35JowYUdn+5S/hpJNCCycU6nfRpdypcArMokWLwg5B\nEqC8STJt2uRnBi8p8e2LLoKrrgo3pjCo30WXcqfCKTDZGpwQScqbJEtZGQwbBu++69u5uXDffZk5\nGFz9LrqUOxVOIiJJceONsHKlX957bz8YvH37cGMSkaZT4SQi0sIWLfITWwK0bg0PPwwHHFD/a0Qk\nNalwCkjNmwpKNChv0tLeeANGjaps33knHHdcePGkAvW76FLuVDgFplevXg1vJClHeZOWVFzsB4Nv\n2+bbI0fCmDHhxpQK1O+iS7lT4RSYcePGhR2CJEB5k5ZSWgpDh8J77/n2UUfBPfdk5mDwmtTvoku5\nU+EkItIirrsOnn7aL++7LyxbBu3ahRuTiDSfCicRkYAtXAh33OGX27TxRdO3vhVuTCISDBVOAVm3\nbl3YIUgClDcJWlERjB5d2b7rLvjBD8KLJxWp30WXcqfCKTCTJ08OOwRJgPImQfrsMzjnHPjmG98e\nPRp++tNwY0pF6nfRpdypcArM3XffHXYI9crLy6NVq1Y8//zzYYeSUlI9bxIdO3fCkCHwwQe+3b8/\nzJmjweC1Ub+LLuVOhVNg6vqK5vvvv0+rVq0YOHBgkiOqzswwHcF3o6/WSlCuvRaefdYv9+jhJ7ls\n2zbUkFKW+l10KXcqnEREmm3+fD+WCSoHg++/f6ghiUgLUeEkItIMr74Kl19e2b7nHn+ZTkTSkwqn\ngMyYMSOQ93nsscf40Y9+RNeuXWnfvj1HHHEEs2bNory8vNp2W7ZsYcaMGZxwwgn07NmTtm3b0rNn\nTy655BLerbj9eiO89dZbfOtb36Jbt268+OKLgexDlASVN8lMn3ziB4Nv3+7bV1wBP/lJuDFFgfpd\ndCl3KpwCU1JS0uz3uPHGGznnnHN45513OPfccxkzZgzt27dn0qRJDBs2rNq2a9euJS8vj+zsbAYN\nGsSECRP43ve+R0FBAUcffTQffvhhg5+3atUqfvjDH5KVlcWqVav4/ve/3+x9iJog8iaZaccOGDwY\nNm707WOPhdmzw40pKtTvoku5A5xzLf4AcgBXWFjoMs17773nzMydfvrp9W63cuVKZ2Zu4MCBbtu2\nbdWeu+KKK1yrVq3cI488smvdli1b3BdffLHb+zz77LMuKyvLXXbZZdXW5+XluVatWrnnnnvOOefc\nY4895rKzs91///d/uw8//DDR3RMJRWFhoQv7mHLllc6Bf/Ts6dzHH4cWiog0U8UxBchxDdQ0OuOU\nIu6++27MjPvuu492Ne7LcNtttwFQUFCwa12nTp3o0qXLbu9z/PHH07dvX5566qk6Pys/P59zzz2X\n7373u/z1r3/lW5rSWKRJ8vP9WCbw35x75BHYb79wYxKR5GgddgDi/e1vf6NDhw7k5+fv9pxzjvbt\n2+82Y+uzzz7L7NmzeeWVVyguLqa0tHTXc23r+B70HXfcwYoVKzj99NN5+OGHad++fbA7IpLmXn4Z\nrryysn3vvf4GviKSGVQ4BaS4uJju3bsn/PpNmzZRVlbGrbfeWuc2Va8tL126lPPPP59OnTpx6qmn\ncuCBB5KdnY2Z8etf/5oPKmbhq8I5x6pVqzAzBgwYoKKJ5udNMsvHH8OgQX58E8DYsTByZLgxRZH6\nXXQpdyqcAjNq1ChWrFiR8Os7d+5Mq1at+PTTTxu1fV5eHu3bt6eoqIiDDjqo2nNVL+lVZWbk5+cz\nbdo0JkyYQFZWFmPHjk045nTQ3LxJ5ti+Hc491xdPAMcdV3kjX2ka9bvoUu70rbrA5OXlNev1Rx99\nNJ9//jn/+te/GrX9u+++S58+fXYrmj7++ON6pyPYa6+9+Mtf/sL//u//Mn78eObMmdOsuKOuuXmT\nzDF+PLz0kl8+4ABYutRPdilNp34XXcqdCqfA5OTkNOv148ePxznHqFGj2LRp027Pf/LJJ9XGOPXu\n3Zv169fz2Wef7Vq3fft2rrjiCnbu3FnvZ3Xu3JmnnnqKo48+mquuuoq7KqY8zkDNzZtkhvvug/vv\n98vt2sGjj8I++4QbU5Sp30WXcqdLdUnz5ptvMrKOwRCHHXYY1113HTfddBPTpk3jkEMO4bTTTqN3\n7958/vnnrF+/nr/+9a/8/Oc/57DDDgNg3LhxjB8/nn79+jF48GBKS0t58sknAfjud7/L6tWr642n\nc+fOrFy5klNPPZWrr76a8vJyrr766mB3WiQNvPACjBtX2b7/fsjNDS8eEQmXCqckMDM++ugjFixY\nUOvzxx9/PNdddx233HILxx9/PHfddRdPP/00X375Jd26dePb3/42t956KxdeeOGu14wZM4Y99tiD\nOXPm8Ktf/YouXbpw5pln8vOf/5zzzjuvUTf07dSpEytXruS0007jmmuuwTnHhAkTAttvkajbuNFP\ncllxEvfqq+Gii8KNSUTCZc5PUNmyH2KWAxQWFham7Wm+/Px8Lr300rDDkCZS3qKpqKiI3NxcWvKY\n8s03cPzx8Morvn3SSfDEE9Ba/91sNvW76ErX3FUcU4Bc51xRfdtqjFNAiorq/TlLilLepDbOwZgx\nlUVT796weLGKpqCo30WXcqfCKTBz584NOwRJgPImtbn3XnjwQb/cvr0fDJ7hU9cESv0uupQ7FU4i\nItU8/zxcdVVlOz8f/ud/wotHRFKLCicRkbgPP4TzzoOKuxddey0MGxZuTCKSWlQ4iYgA27b526lU\nTN5/yikwfXq4MYlI6lHhFJBYLBZ2CJIA5U3ADwa/4gr4+999+9vfhkWLNBi8pajfRZdyp8Kp0QoL\nC5k2bRplZWW1Pp/p93yLKuVNAObMgd/8xi9nZ8Py5dC1a7gxpTP1u+hS7jQBZoOcc8yePZvJkydT\nWlrKzp07ueWWW3bbbsCAASFEJ82lvMkzz8DEiZXtX/8ajjwyvHgygfpddCl3OuNUr+LiYs444wwm\nTpxIaXy06NSpU3n22WfDDUxEAvH++zBkCFScSL7+et8WEamLCqc6PP/88xx++OGsXLmy2nozY+jQ\nodVurisi0VNSAuecA8XFvn3aaTBtWrgxiUjqU+FUQ1lZGbfeeisnnHACn3322W5jmsrLy/n888+5\n6KKLKC8v37V++fLlyQ5VAqC8ZSbn4LLL4LXXfPvgg+GhhyArK9y4MoX6XXQpdyqcqvnoo4848cQT\nycvLwzlXrTCqqqysjCeeeIJXKu7HABQUFCQrTAmQ8paZ7rwTfvc7v9yhAzz2GOy1V7gxZRL1u+hS\n7lQ47fL444/Tt29fXnzxReq78XGrVq0wM26++WaOOuqoXesXL16cjDAlYMpb5nnqKZg0qbK9YAH0\n7RtePJlI/S66lDt9q44dO3Zwww03cMcdd9CqVas6zzIBZGVl0bVrV5YsWcIJJ5yQvCBFJBAbNsDQ\noVDRzX/2Mz/ppYhIY2V04fTuu+9y3nnn8Vp8oEN9RRPAj370IxYuXMjee++djPBEJEBffw1nnw2b\nNvn2GWdALTOLiIjUK2Mv1S1ZsoQjjzyS1atX13tpLisri6ysLH75y1/y+OOPq2gSiSDn4NJLYfVq\n3z70UFi4EFpl7BFQRBKVcYeNkpISLrvsMoYOHUpJScmu+Zlqk5WVxf7778+LL77INddcQ6t6jrIj\nR45siXClhSlvmeH226FiaEanTn5m8C5dwo0pk6nfRZdyl2GX6v7xj38waNAg3nnnHYB6zzQBnH32\n2eTn57Pnnns2+N6aTTWalLf098QTcMMNle2FC6FPn/DiEfW7KFPuMuSMk3OO/Px8cnJyWL9+fYMD\nwPfYYw/uu+8+li5d2qiiCWDYsGFBhStJpLylt3/9C84/v3IweF4e6B6l4VO/iy7lLgPOOG3ZsoXL\nLrusUV+hbNWqFQcddBDLli3jiCOOSEJ0ItJStm71g8G//NK3zzoLbrop3JhEJPrS+ozT3//+d448\n8kgefvjhRm0/YsQIXnvtNRVNIhHnHIwYAW+95duHHebna9JgcBFprrQ8jDjnmD17Nv379+ff//73\nbrdNqap169ZkZ2fz0EMPkZ+fT4cOHRL6zFWrViUaroRIeUtP06fDsmV+uXNnPxi8c+dwY5JK6nfR\npdylYeH0+eefc+aZZzJhwgRKS0vrLZrMjL59+/LGG280+7rtzJkzm/V6CYfyln7+9Cc/sSWAmb8H\n3Xe+E25MUp36XXQpd2lWOD3//PP07duXJ554ot7tzAyA8ePH87e//Y1DDjmk2Z+9aNGiZr+HJJ/y\nll7eeQcuuMBfqgOYOtVPdCmpRf0uupS7NCmcysrKuPXWWznhhBP47LPP6j3LlJWVRefOnVmxYgWz\nZ8+mbdu2gcSQnZ0dyPtIcilv6eOrr/wA8M2bfXvQILjxxnBjktqp30WXcpfihdOmTZvo169fvYO7\nP/roI0466STy8vJwztU71YCZcfTRR/PWW2/x4x//uCVCFpEQlJfDxRfD2rW+3bcvzJ/vL9WJiASp\nyYWTme1hZjPM7N9mVmJmL5vZyS0R3KxZs3jjjTe45JJLWL9+/W7PP/744xx++OG8+OKL9U5m2apV\nK8yM//u//+O5557jW9/6VkuEKyIh+fnP/QBw8DOCL1/uZwgXEQlaImecFgBXAwuB8UAp8Ccz+36Q\ngX322WfccccdAOzYsYPBgwezffv2Xe1rr72WgQMHsnnz5gZvm9KtWzf+8pe/MHXqVFq3bpmpqyZN\nmqOTiGAAAAzxSURBVNQi7ystS3mLvhUr4Oab/bIZFBRAAMMWpQWp30WXctfEwsnMjgKGANc75653\nzv0K+BHwPhDoUPuZM2eyc+dOAEpLS3nzzTe5/vrr2bBhA/37999VVNV3aQ7gpJNO4q233uLEE08M\nMrzd9OrVq0XfX1qG8hZtGzbA8OGV7enT4bTTwotHGkf9LrqUO7CG7tdWbWOzmfizTV2dc1urrL8e\n+DnQyzm3sZbX5QCFhYWF5OTkNPg5//nPfzjwwAN3nWGqKjs7m+3btzc4ABzgtttuY+LEifXenFdE\noqeoqIjc3Fx69y7k/ff9MWXIEFi0SOOaRKTpKo4pQK5zrqi+bZt63aof8HbVoinulSrP71Y4NdX0\n6dNrvfxmZmzbtq3e8UxZWVn06NGDhx9+mKOPPrq5oYhICtqwwf/7/vv+3yOOgAcfVNEkIi2vqYVT\nD+DjWtZ/DBiwf3MD2rhxI/fee2+tZ5Qac3bsrLPOIj8/ny5dujQ3FBEJ2Y4d/l5zX37ppxn48kt4\n8UWYNq1ym7328oPBE5z0X0SkSZpaOLUHdr9+Bt9Ueb5Oayu+K1yP2267rd7LcLVp1aoVWVlZTJo0\niUGDBvHuu+826fVB2LBhA9/+9reT/rnSPMpby9qxw8+vVPHYurX6v3U9Kp6v5Wp9nD+W7LvvWn75\nS19QFdV7cl1SifpddKVr7hpTn1Ro6hinN4H/OOdOqbG+D7AG+Klz7oFaXpcDFDb6g0RERESSL/Ax\nTh9T++W4HvF/P6rvxQsXLqRPnz51Pj916lR+//vfN+mM04ABA7j55ptp377ek10t7uOPP6ZHjx4N\nbygpJZ3z5pw/Y1NxBqexZ3mqPuJfbE2q7Gw/B1PFo2PH6u1OnWCffdZyww3DGzymSGpK536X7tI1\nd2vXrmV41a/o1qOphdPrwAlm1rHGAPFjABd/vk59+vSp81t17777bpOLpqysLJxz9O/fX9+ck7Tj\nHGzbtvsYn6Y8duxIftydO/tJKBvz2HPP6u3OnaExU60VFcENN9R/TBERaQlNLZweBq4FLgPuAD+T\nODACeLm2qQga65Zbbmnya8rKynjyySeZNWuWJuWSlOMclJQ0r/AJ44xPzWKmMQVP1cInPhuIiEha\nalLh5Jx7xcyWAtPNbF9gPb5o6g2MTDSIt99+m9/+9reN+tZcbW644QZ++MMfcswxxyQagshunIOv\nv25e4VPPpPYtwqzhwqe+5zt1UuEjIlKfRO4/chEwFRgO7AWsBs5wzr2QaBBTpkwhKyur3lun1Ke8\nvJzzzjuPdevW0SGk7yTPmDGD6667LpTPlto558ft1Ff4rFw5g0MPva7OwqeJX/BsNrPECp6qhY+u\nWkuq0/EyupS7BAon59wO4Lr4o9nWrFnD4sWLm3y2qU2bNrtuyQLQsWNHSkpKQiucSkpKQvncdOac\nH6Cc6BmfzZsbU/iU8Ne/Bhdzq1aJFz1duviB0Cp8JN3peBldyl1iZ5wC1dDZpjZt2lBaWrqrsOre\nvTv9+vXjyCOP5PDDD+eII46gT58+oRVMFRIZo5XuysubX/g0cCvCAFTPW1ZWYgVP1cJHs1eL1E/H\ny+hS7kIunN544w2WLVsGVN5fruJbdR07duTwww+nX79+uwqkvn370q1bt9DizTTl5bBlS/MKnwSH\nrSUsK8vPJJ1I0dOli599WoWPiIjUJdTC6Z133qF9+/YcfPDB5OTk7CqQDj/8cHr27InpL1izlJU1\nr/DZsiX5hU/r1r7wSaTo6dLFzwGkXxsREWkpoRZOgwcP5txzz02LAqm4uJju3bsH+p5lZdWLnUQK\nn2Rr0yaxgqfi0b59cguflsibiNRP/S66lLsUGOOUDkUTwKhRo1ixYkW1daWlzSt8vvoq+fuxxx7N\nK3zatYvWGZ/a8iYiLUv9LrqUuxQonFLZzp1NKXzy+OEPq6/burXhzwjaHntUjvFpyqPi0ljUCp/m\nysvLCzsEkYyjfhddyl2aF047d+4+WLkpZ3y+/ropnxbMbR/atUu86KkofKTxdLsOkeRTv4su5S7F\nC6cdO5pX+IQx3UT79okXPXvuqcJHREQklSW1cHrpJVi/vvGFz7ZtyYzOy85uesFTdV3btsmPWURE\nRJIjqYXT2LEt/xkdOiRW9FSs32OPxD43Pz+fSy+9NNidkRanvIkkn/pddCl3KXiprmPHxIqeiufa\ntAkn7qKiooz/ZYoi5U0k+dTvoku5A2vqPeIS+hCzHKDw0ksL6dMnp97Cp3XKlXIikmqKiorIzc2l\nsLBQg1VFpNkqjilArnOuqL5tk1qmXHkl6BgnIiIiUaX7sIuIiIg0kgonERERkUZS4RSQWCwWdgiS\nAOVNJPnU76JLuVPhFJixyZhrQQKnvIkkn/pddCl3KpwCM2DAgLBDkAQobyLJp34XXcqdCicRERGR\nRlPhJCIiItJIKpwCsnz58rBDkAQobyLJp34XXcqdCqfAzJgxI+wQJAHKm0jyqd9Fl3Knwikwe++9\nd9ghSAKUN5HkU7+LLuVOhZOIiIhIo6lwEhEREWkkFU4iIiIijdQ6SZ/TDmDt2rVJ+rjke+WVVygq\nKgo7DGki5S2aKo4l6XxMSWfqd9GVrrmrcixp19C25pxr2WgAM7sA+F2Lf5CIiIhI4i50zj1U3wbJ\nKpy6AacC7wHftPgHioiIiDReO+BA4Ann3Of1bZiUwklEREQkHWhwuIiIiEgjqXASERERaSQVTiIi\nIiKNpMJJREREpJFUOImIiIg0kgqnFmZmvzKzcjNbEXYsUj8zO8nM8s3sn2b2tZn9y8weMLP9wo5N\nPDPbw8xmmNm/zazEzF42s5PDjkvqZ2b/a2Z3m9lbZrbVzN43s8Vm9l9hxyZNY2Y/i/9NWx12LGHR\ndAQtyMxygZeAncBfnHOxkEOSepjZq8BewFLgHeAgYBzwNdDPOfdpiOEJYGaLgHOAO4H1wAjgKOAE\n59yLIYYm9TCzpcD38X1rNbAfvm91BI52zv0jxPCkkcysJ/+/vbsJrasIwzj+f9W2VksXDSg1WkGK\nHwgqIlIxhhJdCEL9QHFTuuvCTaXqzlJRG9xpje6CG0GwLmxVXLQItWigiIuiUapIWjfGoqag1SoW\nHhczkdvrzc3Jzc2ZE/v8IITMhMtDhpPz3pkzc+E4IOCkpFsKRyrChdMSiogJ4GvgPuBLF07NFhFD\nkj5ta7sHOALskbS7TDIDiIg7gaPA05JeyW2rgEnglKShkvlsbhGxCfhc0rmWto2ksXtH0rZi4ayy\n/MZlgPRxbQMXauHkpbolEhHbgJuBZ0tnsWrai6bc9gkwA9xUfyJr8yhwDhifbZD0F/AGcFd+N2wN\nJOloa9GU274jFU6+tpaBiBgGHgF2ls5SmgunJRARa4CXgFEv7yxvEXE5aTnh59JZjNuAbyWdaWv/\nrKXflpcr8bXVeBFxETAGjEuaLJ2ntEtKB/ifeg44C+wtHcQWbSewAni7dBBjPTDdoX0aCOCqeuPY\nYkTEVmAQ2FU6i83rCWADMFI6SBO4cOoiIgJYWeV385IBEXE9sAN4XNLfSxjPuuhl7Dq8xjCwG9gn\n6Ugf41lvVgOdxurPln5bBiLiRuB1YAJ4s3Ac6yIi1gHPAy9Imimdpwm8VNfdMGnmaL6vP3LBBGmW\naULSgfrjWotexu5f+R/7u6QdQNtrymzdnQVWdWi/tKXfGi4irgA+BE4Dj8k7lJpuFPiFVOgannGa\nz3HSducqpiNiBLgfeDgirs3tQfo7r85tM5J+63tSa7egsWv9ISKuAQ6R/rE/IOn3/kazHk3TeTlu\nff7+Q41ZrAcRsRY4CKwFhiT9WDiSdZF3Pm4HngQG00Q+QXqzsiLf036VdLpcyvq5cOpC0ikWMI2c\nb7gC9re/FGktf4r0zMxYvzJaZwsdu1l5WvoQ6bmmzfl1rBmOAZsjYk3bA+KbSNfYsTKxrIp8dMQH\nwEbgXknfFI5k8xskFUpjwGsd+qeAV4Gn6gxVms9x6qOIuBq4vUPXOHAS2ANMSjpRZy6rJiIuAw4D\nN5CKJt+IG6TlHKdnJL2c21aStrT/JOnukvlsbnlX1n7SjPwWSQcLR7IKImIA6HRdjZJ2G+8ApiR9\nVWuwwlw41SAiTuADMBsvIg4AW0jnAn3c1n1G0nu1h7LzRMQ+4CHSs4SzJ4ffAYxImigYzbqIiL2k\nm+z7pNPDzyPprdpDWc8i4jAX8AGYLpxqEBFTpMLpwdJZbG65wN0wR/f3kq6rM4/9V55hehHYSvp4\nnC+AXZI+KhrMuso32uG5+iVdXGMcW6Q8nusk3Vo6SwkunMzMzMwq8nEEZmZmZhW5cDIzMzOryIWT\nmZmZWUUunMzMzMwqcuFkZmZmVpELJzMzM7OKXDiZmZmZVeTCyczMzKwiF05mZmZmFblwMjMzM6vI\nhZOZmZlZRS6czMzMzCr6Bzs9xtSfelHXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1064eb0f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "save_fig(\"leaky_relu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z<0, alpha*(np.exp(z)-1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure elu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGFCAYAAAALnnwfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xl8FdX5x/HPIyCLiOIKVbFataDWauKGu1hRUa5rQZRa\noba2Ai5twbYqoNYFWmsVsLUWaxEb0KpI3UDriq3VJvVnUXBXtOKCG0vYc35/nBuzkOVOkjvnzp3v\n+/WaV3Lmzs19hoczeTJz5ow55xARERGR5m0UOgARERGRpFDhJCIiIpIjFU4iIiIiOVLhJCIiIpIj\nFU4iIiIiOVLhJCIiIpIjFU4iIiIiOVLhJCIiIpIjFU4iIiIiOVLhJCIiIpIjFU4iIiIiOVLhJJJQ\nZrajmVWZ2a2hY6lWoDG1N7PxZvaqma0ys/VmlgkdVxSF+O9azcwqzOzh0HG0lJntZmZrzeyHoWOR\nZFDhJAWp1i+KppY3G9j+wRx+9uHN/RLKZZs41IpjbCObuOxSKPHEHlMOfgqMBf4H/Aq4HFgYNKJ6\nEvrvipl9F/gmcFnoWJpjZl81s7/WX++cexUoA8abWdf4I5OkaR86AJFmvA5Mb+S1z+MMpAD9D+gD\nfBE6kFoKMabjgWXAt5xz60MH00IF9+9qZgaMA55yzj0fOp6mmNnRwC3Am41sMhEYCowCrokrLkkm\nFU5S6F53zl0ROoiArLEXnHPrgFdjjAWaiAeCxdScrwCfFHjRlMR/1wHAV4ErA8fRKDMrxcf3DrCq\nse2cc/PN7EXg+6hwkmboUp1IK5lZBzMbZWYPm9mi7DiaD83sbjPbu4n3HWZms8zsg+x7FmXfc1D2\n9XHAY/jLM+NrXaJcb2a9Ghr3YmaHZtfd0shnbp0dz/F01Nibiye7TaNjccxsmJk9a2bLssuz2Us9\n9bf78rKVmZWa2SNmttTMPjeze8xsx+az4uM1syr8L/ev1r/Ea2ZnZ9tnNRVDW8TVRK4PTtq/ay3D\ngCrgnojvi41zrtw5N8A59yNgcTOb3wnsaGZHxhCaJJjOOIm03hbA9cBTwAPAZ8DOQAY4zswOdc6V\n136DmV0A/AaoBO4FFgHbAYcApwH/AJ4AdgTOzn7/RPbtDn+Zsnv9QJxzT5vZ28BpZjbCObem3iZn\n4P9gmtaC2B9vJp5GmdmNwEjgPeCP2dWnAn8ys72dcxc18Lb9gYvxRcXvgX2Ak4A9zWzPBvatvsez\nsV2U/Xo9/szOZ7XibsmYoUhxNZPrU4FZJOvftdoRwCvOuQYvH5pZR+AHQE/8/9UHnHP35/izQ/gn\n/v/HUfj/OyINc85p0VJwC/4XSRX+8sS4RpZjGtj+wRx+9uHZbW9tzTa1tt0Y6NnA+j7AUmBOvfV7\nAevwv0B3aOB9PRqIY2wT/0a31lt/BbAeOK2B9/wbWAls3sLYG42nsZiAQ7Pr/gt0rbV+M/wg7fXA\nwQ18xgb7APw5u35QhP9LbwFvNrD+u9mfdVYT+R/bmrhyzXXS/l2z/z+qgGmNvL4z8FL15wAd8H8M\nnJFttwPOyTWHbbHgi6HHmnh90+w+PR5nXFqSt+hSnRS6r+HviGpoOSZgXF9yzq1xzm1wGcA5twB/\nsD7MzNrVeumH+L9sL3XOvdvA+z5oZUi3Z3/+0NorzezrQAn+L//PWxh7S5xN9jKUc255rc/4An+H\nm2W3qe9J51z9u6BuzW6/Xytjao0oceUz12cT7t91++zXD+u/YP7OtEeBe6s/xzm3Fn9261fZzY6k\n5sxf/fffYX6Kg1yW/5jZoznG3CTn3DL8OKjtm9tW0k2X6qTQzXHODQgdRHPM7Jv4yx8HAz3wf2FX\nc8BW1PySqf7l9Eg+YnHOvWZmzwHHmtkWzrlPsy+dlY3l9lbE3hLVY6WebOC1x+ttU1tFA+vey37d\nvBXxtFaUuPKZ65D/rltmvzZ0KXEc/v9M/UHWLwM9zKwP/o+ecQ39YOfcmTnGkA+f4mMXaZQKJ0mj\nquzXps64Vr9W1cQ2AJgfzP13fJExF3gNWJ5tn4y/XNOx1ls2A1xDZ3ra0O34sSyDgd9l152B/yv/\ny7muWhB7S3QDqpxzSxp47cPsZ3Vr4LWlDaxbl/3a2rNgrRElrnzmOuS/68rs1061V5pZB/yg8VnO\nuRX13vMp/qzWAcBy51xljp8Vp874sWgijVLhJGlUPZh1yya2qf6rM5d5cy7BjxU6xDn3z9ovmFlf\nfPFR2+f+JeuZx+JpBn5A8lDgd2Z2OH6czO+yl01aGntLLAU2MrOtGvglvw3+l2lDv8zzrSr72Q0d\nBzdro8/IZ65D/rt+nP26Rb31X8uu+0cD76meDuJc4Lg8xdViZmb4vG9wSVWkNo1xkjR6BVgD7Gdm\njfWBg/B/sb+Yw8/bGfi0gcKjM35MUX3PZb/2z+FnV/+yiXSGxTn3CfAwcKCZ7YwvoBxwR71No8be\nknj+k/16RAOvHVlvmzhVj7HZroHXGtr3lsg110n7d30JX3h+vd766jNcbzfwnuo7GKdUj7FriJlN\nM7PnclyeN7M5rd4bb1f878T/ttHPkyKlwklSxzm3Gj9ny9Y08KgIM/sG8D38X+v35vAj3wG6Z8du\nVP+MjYDrsp9R3+/xv3R+WT1PT73P71GrWT0+aYcc4qiveizTOfgpDt5yztU/ExA19pbE82f82Y9x\nZrZprc/ZDD/OxVEzPUKcyrOffXr21vnquHYFzqdtHm+Sa64T9e+aHYD+IrBvvZcWAqupO06uWkk2\npibHyznnznLO7Z/jsp9zrq1uEjkg+7WhMWMiX9KlOil0u2QnCGzMNa7uvDPfMLM/NbLtQufchOz3\nP8GPARprZifgD5argN3wcxiBv3U6l0sdk/BnFJ4xszuzP+cI/IzVT+BvA/+S87MUXwjcALxkZrPw\nBUwP4DDgfuDH1TED7+N/ua/BD+J1wI05xPU3fPH3E3xf/21rY28qnuxdSRtwfm6pSfj5huab2d34\nX/in4s/23OCcm5fD/rQp59xiMysDhgDl5h9Uuw1+bNdD+GKztZ+Ra66T+O96L37CzgOdc89mY1pq\nZtcBZ5vZ/c45Z2bbAD/C73cFfjLSI4BNnHMP5DG++jrR9Azt/fFjvQp5rikpBKHnQ9CipaEFPx5n\nfQ5LtwjbP1bvMzbFT2tQgS8wVuEvMUwDvhkx3pOB5/HPRPsQ+At+xuo/4Q/GvRp4z2HAffjxIivx\nv1hmAgfW224//GSFn9fal1619nlqE3H9IbvNOmCXtoi9sXjq5WGDmPDzJj2b/Zxl2e8bm0NpPXBZ\nE/8vGt3nBt7zFvBGI691xE+M+T5+UPB/8APqN4ihNXHlkusE/rv2xF/yntzAa+cANwET8A9Z3jq7\nfndgKvDzfB4/asXRC5iDvzxf/W/6Gv4y9o61tuuMPwb8NY64tCR7Mefa4my0iIikjZlNwz+zbke3\n4V10iWFm5wA3A4c5554JHY8UNhVOIiLSItlxWwuAK51z14aOpyWyE7y+Cvyfc+6U0PFI4dMYJxER\naRHn3KLsA4W3DR1LK/TCD7S/vbkNRUBnnERERERypukIRERERHJU0JfqzGxL/DON3sbf8SQiIiLS\n1jrh7yae4/wEwo0q6MIJXzTVn+lYREREJB/OxE/J0qhCL5zeBpg+fTp9+vRpZtNkueiii7j++utD\nhyE5Ur6SY8GCBQwdOrQojxvFKo396623YPhwWJqdYvfoo+Gqq6BdyMdX56gY81V93KDhxwXVUeiF\n0yqAPn36UFLSVo+OKgybbbZZ0e1TMVO+kqcYjxvFKm3967334KSTaoqmfv3gb3+Djh2bfl+hKPJ8\nNTssSIPDRUREYvLZZ3DssfDuu769zz5w773JKZokpsLJzHY3szvN7A0zW2FmH5vZk9lnhKXSokWL\nQocgEShfIvmTlv61ciVkMvDSS769887w0EPQrVvYuKJKS74aE9eluh2BrsBt+GdCdcE/iHK2mf3A\nOffHmOIoGF988UXoECQC5Uskf9LQv9atg9NPh3nZxy5vsw3MmQPbJnDq0DTkqymxFE7OuYfwTxv/\nkplNxj9c9cdA6gqnq6++OnQIEoHyJZI/xd6/nIMf/Qhmz/btrl39maZddgkbV0sVe76aE2yMk/NT\nlr8LbB4qhpCGDBkSOgSJQPkSyZ9i719jx8Ifs6cHOnSAWbMgyWOriz1fzYn1rjoz6wJ0BjYDTgSO\nA8rijEFERCQukyfDL3/pvzeD22+Ho44KG5O0TtzTEVwHnJv9vgq4GxgVcwwiIiJ5d9ddcP75Ne0b\nboDBg8PFI20j7kt11wPfAs4CHgTaAam8CTOTyYQOQSJQvkTypxj712OPwdChfnwTwC9+AaOK5DRB\nMeYrilgLJ+fcq865x5xz051zGfyddvfHGUOhGDlyZOgQJALlSyR/iq1//ec/foLLNWt8e/jwmst1\nxaDY8hVV6Akw7wZKzWzXpjYaMGAAmUymztK3b19mzZpVZ7u5c+c2WAmPGDGCqVOn1llXUVFBJpNh\nyZIlddaPGzeOCRMm1Fm3aNEiMpkMCxcurLN+0qRJjB49us66yspKMpkM86rvOc0qKytj2LBhX7b7\n9+8PwODBgxO9H9WKfT8mT55cFPtRLPnQfhTXfjzzzDNFsR/jxo3j4osncNxxsGyZX3fUUYv46KMM\nr7ySrP1oKh/Vv7+Suh+lpaX069evTk0xaNCgDT6rMeaqzyMGYGbn4y/fHeCc+3cDr5cA5eXl5cU8\nvbuItKGKigpKS0vRcUPi9uGHcPDB8MYbvn3QQfDII9ClS9i4pHnVxw2g1DlX0dS2cc0cvnUD69oD\n3wVWAi/HEYeIiEg+LFsGAwbUFE277+6fP6eiqfjEdanuZjN71MzGmtn3zOwS4EVgb+AS51xlTHEU\njPqnJaWwKV8i+ZP0/rV6NZx8MlRkz1PssIOfFXyLLcLGlS9Jz1drxVU4zQDWAz8EbgIuwk9+mXHO\n3RBTDAWlrEzTVyWJ8iWSP0nuX1VV8N3vwt//7tvdu/uiafvtw8aVT0nOV1uI65ErdwJ3xvFZSTFz\n5szQIUgEypdI/iS1fzkHF10E1eF37gwPPAB9+oSNK9+Smq+2EvquOhERkUSaMAFuvNF/364d3Hkn\n9O0bNibJPxVOIiIiEf3pT/Dzn9e0//hHOOGEcPFIfFQ4iYiIRHD//fD979e0r7kGzj47WDgSMxVO\ngTQ0CZgULuVLJH+S1L/+8Q8YNAjWr/ftCy6Aiy8OG1PckpSvfFDhFEjtmVel8ClfIvmTlP718sv+\nctzKlb49ZAj85jdgFjauuCUlX/miwimQIUOGhA5BIlC+RPInCf3r3XfhmGPgs898+1vfgttug41S\n+Fs0CfnKpxSmXEREJHeffuqLpvfe8+3SUrjnHth447BxSRgqnERERBpRWQkDB8KCBb69yy7w4IOw\n6aZh45JwVDgFUv+p0VLYlC+R/CnU/rVuHQwe7AeEA2y7rZ8VfJttwsYVWqHmKy4qnAKZOHFi6BAk\nAuVLJH8KsX85Bz/4gZ96APwZpocfhp13DhtXISjEfMVJhVMgM2bMCB2CRKB8ieRPIfavSy7xk1yC\nH8t0332w995hYyoUhZivOKlwCqRLly6hQ5AIlC+R/Cm0/nXjjX5SS/BTDUyfDkceGTamQlJo+Yqb\nCicREZGsGTPgwgtr2pMnw7e/HS4eKTwqnERERIBHH4WzzvLjmwAuuwzOOy9sTFJ4VDgFMnr06NAh\nSATKl0j+FEL/Ki+Hk0+GtWt9+/vfh8svDxtToSqEfIWkwimQXr16hQ5BIlC+RPIndP96/XUYMACW\nL/ftE0+Em25K36NUchU6X6GpcApk1KhRoUOQCJQvkfwJ2b8++MDPCv7RR759yCFQVgbt2wcLqeCl\n/XiowklERFJp6VI47jh4803f3nNPmD0bOncOG5cUNhVOIiKSOqtX+zFNL7zg2716+Qkuu3cPG5cU\nPhVOgSxcuDB0CBKB8iWSP3H3r/Xr4Tvfgcce8+0tt/SPUtluu1jDSKy0Hw9VOAUyZsyY0CFIBMqX\nSP7E2b+cgwsugLvu8u0uXeCBB6B379hCSLy0Hw9VOAUyefLk0CFIBMqXSP7E2b+uvhqmTPHft2sH\nf/0rHHBAbB9fFNJ+PFThFEjab+dMGuVLJH/i6l9//CNcemlN+9Zb/eBwiSbtx0MVTiIiUvRmz4Zz\nz61pT5zoZwkXiUqFk4iIFLV582DwYKiq8u0f/xh++tOwMUlyqXAKZMKECaFDkAiUL5H8yWf/mj8f\nBg6EVat8+8wz4Ve/0qzgrZH246EKp0AqKytDhyARKF8i+ZOv/vXOO35W8M8/9+1jjvHjmjbSb75W\nSfvx0Fz1Y6ALkJmVAOXl5eWUlJSEDkdEEqCiooLS0lJ03Ei3Tz6Bgw+GV17x7f328/M2de0aNi4p\nTNXHDaDUOVfR1Laqu0VEpKisWAHHH19TNO26q5+rSUWTtAUVTiIiUjTWroVBg+Bf//Ltnj1h7lzY\neuuwcUnxUOEUyJIlS0KHIBEoXyL501b9yzk45xx48EHf7tbNP3/uq19tkx8vWWk/HqpwCmT48OGh\nQ5AIlC+R/Gmr/vWzn8G0af77jh393E177dUmP1pqSfvxUIVTIOPHjw8dgkSgfInkT1v0r+uv95Na\ngp9q4C9/gcMPb/WPlQak/XiowikQ3e2TLMqXSP60tn/dcYef1LLaTTfBKae0MihpVNqPhyqcREQk\nsebOhbPPrmmPHw8//GGoaCQNVDiJiEgiPf+8P7O0bp1vn3sujB0bNiYpfiqcApk6dWroECQC5Usk\nf1rSv159FQYM8HM2gS+gpkzRo1TikPbjoQqnQCoqmpyYVAqM8iWSP1H71+LF/vEp1XfFH3aYH+fU\nrl0egpMNpP14qMIpkClTpoQOQSJQvkTyJ0r/+uILOPZYePtt395rL7jvPujUKT+xyYbSfjxU4SQi\nIomwahWceCK8+KJv77gjPPQQbL552LgkXVQ4iYhIwVu/HoYOhSef9O2ttvJ31H3lK2HjkvRR4SQi\nIgXNORg5Eu6+27c32cQ/tHe33cLGJemkwimQTCYTOgSJQPkSyZ/m+teVV8Lvf++/b9/eF1D77x9D\nYNKgtB8PVTgFMnLkyNAhSATKl0j+NNW/br4Zxo2rad92m7+jTsJJ+/FQhVMg/fv3Dx2CRKB8ieRP\nY/3r3nvhvPNq2tddB2eeGVNQ0qi0Hw9VOImISMF56ikYMgSqqnx79Oi6z6MTCSWWwsnM9jWzyWY2\n38yWm9k7ZjbTzHaN4/NFRCQ5XnwRMhlYvdq3zzoLrr02bEwi1eI643QxcDLwKHA+cDNwGFBhZrvH\nFENBmTVrVugQJALlSyR/avevt9/2E1x+8YVvH3cc/PGPsJGujxSMtB8P4/qveB2wo3PuQufcrc65\nq4FDgQ7Az2KKoaCUlZWFDkEiUL5E8qe6fy1Z4gd+L17s1x9wANx1F3ToEDA42UDaj4ft4/gQ59yz\nDax73czmA33iiKHQzJw5M3QIEoHyJZI/M2fOZPlyOP54//BegK9/He6/38/ZJIUl7cfD0Cc/twWW\nBI5BREQCWrsWTjsNnnvOt7/yFZgzx88OLlJoghVOZjYU2A6YESoGEREJq6oKhg/3hRL4587NmeOf\nQydSiIIUTmbWG5gMPANMCxGDiIiEd/HFMH26/75jR5g9G/bcM2xMIk2JvXAys22AB4DPgG8751zc\nMRSCYcOGhQ5BIlC+RNrer3/tFxjGRhvBjBlw6KGho5LmpP14GGvhZGbdgDlAN+BY59wHubxvwIAB\nZDKZOkvfvn03uCVy7ty5DT5DZ8SIEUydOrXOuoqKCjKZDEuW1B1iNW7cOCZMmFBn3aJFi8hkMixc\nuLDO+kmTJjF69Og66yorK8lkMsybN6/O+rKysjr/2apnXh08eHCi96Nase/Hiy++WBT7USz50H4k\nfz9uv91Paukt5aSTJnDSScnbj9qSnI8o+1F75vAk7kdpaSn9+vWrU1MMGjRog89qjMV1wsfMOgJz\ngRLgKOfcczm8pwQoLy8vp6SkJN8hikgRqKiooLS0FB03CtfDD8PAgbBunW9feSVcemnYmCTdqo8b\nQKlzrqKpbWOZjsDMNgLuBA4EMrkUTSIiUnz+9S849dSaomnECLjkkrAxiUQRS+EE/AYYCMwGtjKz\nOo9pdM7dEVMcIiISyCuv+LmaKit9+7TT4IYbwCxsXCJRxDXG6ZuAwxdP0xpYUqf+tV8pbMqXSOv8\n73/Qvz988olvH3mkv5uuXTv1r6RJe75iKZycc0c659o1tsQRQ6GZOHFi6BAkAuVLpOU+/9w/f27R\nIt/ee2+4914//QCofyVN2vMVeubw1JoxQ/N+JonyJdIyK1dCJgPz5/v2TjvBQw/BZpvVbKP+lSxp\nz5cKp0C6dOkSOgSJQPkSiW79ejjjDHj6ad/eems/K3iPHnW3U/9KlrTnS4WTiIi0OefgvPOgenqe\nrl39maZddw0bl0hrqXASEZE2N348/OEP/vsOHeCee8BPkyOSbCqcAqk/06oUNuVLJHe/+x1ccUVN\ne9o0OProxrdX/0qWtOdLhVMgvXr1Ch2CRKB8ieTmr3/1k1pW++1v4fTTm36P+leypD1fKpwCGTVq\nVOgQJALlS6R5TzwBZ57pxzcB/OxncMEFzb9P/StZ0p4vFU4iItJqL7wAJ54Ia9b49rBhcPXVYWMS\nyQcVTiIi0ipvvQXHHQdLl/r28cf7geF6lIoUIxVOgSxcuDB0CBKB8iXSsI8+8o9S+eAD3z7wQLjz\nTmgf4Umo6l/JkvZ8qXAKZMyYMaFDkAiUL5ENLVvmzy69/rpv9+kD998PUedHVP9KlrTnS4VTIJMn\nTw4dgkSgfInUtWYNnHoq/Pvfvr399n5W8C23jP6z1L+SJe35UuEUSNpv50wa5UukRlUVnH02PPKI\nb3fvDg8/DDvs0LKfp/6VLGnPlwonERHJmXPwk59AWZlvd+rkL8/tsUfYuETiosJJRERy9qtf+Ukt\nAdq18wPBDzoobEwicVLhFMiECRNChyARKF8icNttcPHFNe0//AEGDmz9z1X/Spa050uFUyCVlZWh\nQ5AIlC9JuwcegHPOqWlffTUMH942P1v9K1nSni9z1XPjFyAzKwHKy8vLKSkpCR2OiCRARUUFpaWl\n6LjRdp59Fvr1g5UrfXvUKLjhBk1wKcWj+rgBlDrnKpraVmecRESkUQsW+LmaqoumQYP8GCcVTZJW\nKpxERKRB770HxxwDn37q20cdBdOmwUb6zSEppv/+gSxZsiR0CBKB8iVp89lncOyx8O67vl1SAvfc\nAx07tv1nqX8lS9rzpcIpkOFtNapSYqF8SZqsXOnvlnvpJd/+2tfgwQehW7f8fJ76V7KkPV8qnAIZ\nP3586BAkAuVL0mLdOjj9dHjmGd/eZhv/KJVtt83fZ6p/JUva86XCKRDd7ZMsypekgXPwwx/C7Nm+\nvemm/lEqX/tafj9X/StZ0p4vFU4iIgLAZZfB1Kn++403hlmzYJ99wsYkUmhUOImICJMnw1VX+e/N\n4Pbb/dxNIlKXCqdAplb/WSeJoHxJMbvzTjj//Jr2DTf4+Zriov6VLGnPlwqnQCoqmpyYVAqM8iXF\n6u9/h6FD/fgmgEsu8TODx0n9K1nSni8VToFMmTIldAgSgfIlxeg//4GTT4a1a337e9+DK6+MPw71\nr2RJe75UOImIpNAbb8Bxx8GyZb6dycDvf69HqYg0R4WTiEjKfPihf5TKhx/69sEHQ1kZtG8fNi6R\nJFDhJCKSIkuX+jNNb7zh23vs4edt6tIlbFwiSaHCKZBMJhM6BIlA+ZJisHo1nHKKH9sEsMMOfoLL\nLbYIG5f6V7KkPV8qnAIZOXJk6BAkAuVLkq6qCr77XX8XHfhiac4c2H77sHGB+lfSpD1fKpwC6d+/\nf+gQJALlS5LMObjwQpg507c7d4b774c+fcLGVU39K1nSni8VTiIiRe7aa2HSJP99u3Zw113Qt2/Y\nmESSSoWTiEgRu/VW+MUvatpTp8Lxx4eLRyTpVDgFMmvWrNAhSATKlyTR3/4GP/hBTfvaa/04p0Kj\n/pUsac+XCqdAysrKQocgEShfkjT/+Id/3tz69b594YUwZkzYmBqj/pUsac+XCqdAZlaP0pREUL4k\nSV56CU44AVat8u0hQ+C66wp3VnD1r2RJe75UOImIFJF334Vjj4XPPvPto4+G226DjXS0F2kT6koi\nIkXi00/9o1Tee8+3990X7r4bNt44bFwixUSFk4hIEais9JfnFizw7V12gQcegE03DRuXSLFR4RTI\nsGHDQocgEShfUsjWrvUDwf/5T9/u0QPmzoVttgkbV67Uv5Il7flS4RRI2mdeTRrlSwqVc3Duuf7s\nEkC3bvDQQ7DTTmHjikL9K1nSni8VToEMGTIkdAgSgfIlheqSS+BPf/Lfb7wx3Hcf7L132JiiUv9K\nlrTnK7bCycw2MbPLzewhM/vEzKrM7Ky4Pl9EpNjccANcc43/3gzuuAOOOCJoSCJFL84zTlsBlwG9\ngRcAF+Nni4gUlRkz/KSW1aZMgdNOCxePSFrEWTi9D/Rwzu0EjAEKdCq2eMybNy90CBKB8iWF5JFH\n4Kxa5+vHjoUf/ShcPK2l/pUsac9XbIWTc26tc+6juD6v0E2cODF0CBKB8iWForwcTjnF30kH/ll0\n48cHDanV1L+SJe350uDwQGbMmBE6BIlA+ZJC8NprcNxxsHy5b590Etx0U+E+SiVX6l/JkvZ8qXAK\npEuXLqFDkAiULwntgw/8rOAff+zbhx4Kf/kLtGsXNq62oP6VLGnPlwonEZEC98UX/vlzb73l29/4\nBsyeDZ07h41LJI1UOImIFLDVq/0luf/7P9/ecUd4+GHYfPOwcYmkVSIKpwEDBpDJZOosffv2Zdas\nWXW2mzt3LplMZoP3jxgxgqlTp9ZZV1FRQSaTYcmSJXXWjxs3jgkTJtRZt2jRIjKZDAsXLqyzftKk\nSYwePbrOusrKSjKZzAZ3HZSVldWZpr76fYMHD070flQr9v3YZZddimI/iiUfadmP9eth6FB44okK\nIEP37kvvgj4GAAAgAElEQVSYMwe+8pVk7Ue1xvJx0EEHFcV+FEs+mtuP2nEncT9KS0vp169fnZpi\n0KBBG3xWo5xzsS9AKVAFnNXMdiWAKy8vd8XmxhtvDB2CRKB8JUd5ebkrhuNGVZVzI0Y45x+q4lyX\nLs49+2zoqPJD/StZijFf1ccNoMQ1U8Mk4oxTMRo1alToECQC5UvidtVVflJLgPbt4e674YADwsaU\nL+pfyZL2fLWP88PMbASwObBddlXGzHbIfn+jc25ZnPGIiBSiW26Byy6rad96qx8cLiLhxVo4AT8F\nemW/d8DJ2QXgdkCFk4ik2n33wQ9/WNP+1a/gO98JF4+I1BXrpTrn3E7OuXaNLIvijCW0+gMFpbAp\nXxKHp5+G00+Hqirf/slP4Kc/DRtTHNS/kiXt+dIYp0DGjBkTOgSJQPmSfPvvfyGTgVWrfHvoUEjL\nky3Uv5Il7flS4RTI5MmTQ4cgEShfkk/vvOPHMH3+uW8fe6wf17RRSo7Q6l/JkvZ8paRbFp5evXo1\nv5EUDOVL8mXJEv8olfff9+399oO77oIOHcLGFSf1r2RJe75UOImIBLJiBZxwArzyim/vths88AB0\n7Ro2LhFpnAonEZEA1q6Fb38b/vUv3+7ZE+bMga23DhuXiDRNhVMg9aeBl8KmfElbcg7OOQceesi3\nN9vMP3/uq18NGlYw6l/JkvZ8qXAKpLKyMnQIEoHyJW3pZz+DadP89x07wuzZsNdeYWMKSf0rWdKe\nL3P+mXAFycxKgPLy8nJKSkpChyMiCVBRUUFpaSmFetz4zW/8/Ezg75q76y445ZSwMYmkXfVxAyh1\nzlU0ta3OOImIxOSOO2qKJoCbblLRJJI0KpxERGIwZw6cfXZN+/LL4dxzg4UjIi2kwimQJUuWhA5B\nIlC+pDWefx5OPRXWrfPtH/2o7kN80079K1nSni8VToEMHz48dAgSgfIlLfXqqzBggJ+zCXwBNWkS\nmIWNq5CofyVL2vOlwimQ8ePHhw5BIlC+pCXefx/69/ezgwMcfjhMnw7t2oWNq9CofyVL2vOlwimQ\nQrzbRxqnfElUX3wBxx3nn0MH8M1vwn33QadOYeMqROpfyZL2fKlwEhFpY6tWwYknwosv+vZXv+on\nu9xss6BhiUgbUOEkItKG1q+HM8+EJ5/07a228nfU9ewZNi4RaRsqnAKZOnVq6BAkAuVLcuEcjBgB\n99zj25tsAg8+6B/eK41T/0qWtOdLhVMgFRVNTkwqBUb5klxccQXcfLP/vn17X0Dtt1/YmJJA/StZ\n0p4vFU6BTJkyJXQIEoHyJc25+WaofbPRn//s76iT5ql/JUva86XCSUSkle65B847r6b9m9/AGWeE\ni0dE8keFk4hIKzz5pC+Sqqp8e8wYuOiisDGJSP6ocBIRaaEXX4RMBlav9u3vfheuvTZsTCKSXyqc\nAslkMqFDkAiUL6nv7bfh2GNh6VLfHjAAbrlFj1JpCfWvZEl7vlQ4BTJy5MjQIUgEypfU9vHHcMwx\nsHixbx9wANx5J3ToEDaupFL/Spa050uFUyD9dbtNoihfUm35cjj+eP/wXoDeveGBB/ycTdIy6l/J\nkvZ8qXASEcnRmjVw2mnw/PO+vd12flbwLbcMG5eIxEeFk4hIDqqq4Hvf84USwOabw8MPQ69eYeMS\nkXipcApk1qxZoUOQCJQvGTMGpk/333fqBH/7G+y5Z9iYioX6V7KkPV8qnAIpKysLHYJEoHyl269/\nDddd57/faCOYMQMOOSRsTMVE/StZ0p4vFU6BzJw5M3QIEoHylV7TpsHo0TXtm2+GE08MF08xUv9K\nlrTnS4WTiEgjHnoIhg+vaf/yl3DOOeHiEZHwVDiJiDTgX//yd9CtX+/bI0bAL34RNiYRCU+Fk4hI\nPQsX+rmaKit9+9vfhhtu0KzgIqLCKZhhw4aFDkEiUL7S43//87OCf/KJb/frB7ffDu3ahY2rmKl/\nJUva86XCKZC0z7yaNMpXOnz2mX/+3KJFvr333nDvvdCxY9i4ip36V7KkPV8qnAIZMmRI6BAkAuWr\n+K1c6e+Wmz/ft3fayQ8O79YtbFxpoP6VLGnPlwonEUm9devgjDPg6ad9e+utYe5c6NEjbFwiUnhU\nOIlIqjkH550H1ZMhd+3qzzTtskvYuESkMKlwCmTevHmhQ5AIlK/iNW4c3HKL/75DBz+mqbQ0bExp\no/6VLGnPlwqnQCZOnBg6BIlA+SpON90EV15Z0542Db71rXDxpJX6V7KkPV8qnAKZMWNG6BAkAuWr\n+Pz1rzByZE37hhvg9NPDxZNm6l/JkvZ8qXAKpEuXLqFDkAiUr+Ly+ONw5pl+fBPAz38O558fNqY0\nU/9KlrTnS4WTiKTKCy/4aQfWrPHt4cPhqqvCxiQiyaHCSURS48034bjjYNky3z7hBLj5Zj1KRURy\np8IpkNGjR4cOQSJQvpLvo4/8o1Q++MC3+/aFmTOhffuwcYn6V9KkPV+xFU5mtrGZTTCz98ys0sye\nNbPU3r/Sq1ev0CFIBMpXsi1bBgMGwOuv+3afPnD//ZDyoRoFQ/0rWdKerzjPOE0DLgSmA+cD64AH\nzeygGGMoGKNGjQodgkSgfCXXmjVwyilQXu7b228Pc+bAFluEjUtqqH8lS9rzFctJajPbHxgE/MQ5\nd3123e3AfGAicEgccYhIulRVwdlnw6OP+nb37r5o2mGHoGGJSILFdcbpNPwZpluqVzjnVgNTgb5m\ntl1McYhISjgHP/4xlJX5dufO/vLc7ruHjUtEki2uwmlv4FXn3PJ665+r9XqqLFy4MHQIEoHylTx/\n/rOf1BKgXTs/EPygVA4MKHzqX8mS9nzFVTj1BBY3sH4xYMBXYoqjYIwZMyZ0CBKB8pU8kybVfH/L\nLTBwYLhYpGnqX8mS9nzFVTh1BlY3sH5VrddTZfLkyaFDkAiUr+R4/PG67WuugWHDwsQiuVH/Spa0\n5yuuGUxWAh0bWN+p1uuNWrBgQZsHVAiWLFkSOgSJQPkqfM89BxdfXH28WMDpp8PRR0NFRdCwJAfq\nX8lSbPmKUmeYq35YUx6Z2VzgK865Peut7wc8Cgx0zj3QwPtKgPK8BygiIiICpc65Jv/UiuuM0wvA\nEWbWtd4A8QMBl329UdOnT6dPnz75jE9EEuzNN+F734OlSwEWAEP585+ns+eeOm6ISPMWLFjA0KFD\nc9o2rsLpr8BPgR8AvwE/kzhwNvCsc+5/Tb25T58+lJSU5DvGWE2YMIGLL744dBiSI+WrcL31Flxw\nQXXRBKWlfrLLPfcsvuNGsVL/Spa05yuWwsk595yZ3QVcY2bbAq/ji6YdgVQO26ysrAwdgkSgfBWm\n99/3Y5jef9+3990XrrsODj88bFwSjfpXsqQ9X7GMcYIvzzBdCQwFugMvApc65x5t4j0lQHl5ebn+\nchSROt59F/r1q3n+XO/e8PTTsGhRBaWlpei4ISK5qqjwxw1yGOMU27PqnHNrnHMXO+e2c851cc4d\n2FTRJCLSmLfegsMOqymadtoJHnkEttoqbFwiUvzifMiviEirvf66vxT39tu+veuu8NRT/uG9IiL5\npsIpkGKbA6PYKV+FYeFCXzS9+65v9+4NTz6poinp1L+SJe35UuEUyPDhw0OHIBEoX+G99BIccUTN\nQPA994QnnoCePUNGJW1B/StZ0p4vFU6BjB8/PnQIEoHyFdZTT8Ehh8CHH/r23nv7R6tsu23YuKRt\nqH8lS9rzpcIpEN3tkyzKVzh33umnHPj8c9/ebz947DENBC8m6l/JkvZ8qXASkYLknJ+TafBgWLPG\nrzv2WF80de8eNjYRSS8VTiJScNavhwsvhJ/+tGbd974Hs2dD167h4hIRUeEUyNSpU0OHIBEoX/FZ\nsQK+/W248caadZdfDrfcAh06hItL8kf9K1nSni8VToFUVDQ5MakUGOUrHm+8AX37wr33+nb79nDr\nrTB2LJiFjU3yR/0rWdKer7ge8iv1TJkyJXQIEoHylX8PPwxDhtQMAt90U7jrLjjmmLBxSf6pfyVL\n2vOlM04iEpRzcM01MGBATdHUuzc8/7yKJhEpPDrjJCLBLFsGw4bB3XfXrDvxRJg2Dbp1CxeXiEhj\ndMZJRIJ4/nkoKakpmszgiivgnntUNIlI4VLhFEgmkwkdgkSgfLWdqiqYOBEOOsg/sBd8oTR7Nlx2\nGWyko1LqqH8lS9rzpUt1gYwcOTJ0CBKB8tU23n8fzjoL/v73mnX77w9/+Qt87Wvh4pKw1L+SJe35\n0t92gfTv3z90CBKB8tV6s2fDXnvVFE1m8ItfwLx5KprSTv0rWdKeL51xEpG8+vhjuOACKCurWbfd\ndnD77XDkkeHiEhFpCZ1xEpG8cA5mzIDdd69bNJ10Evzf/6loEpFkUuEUyKxZs0KHIBEoX9H873++\nQBoyBJYs8eu6d4c//9nfNbfllmHjk8Ki/pUsac+XCqdAymr/CS4FT/nKzbp1MGkS7LGHH9NU7dRT\n4eWX/cBwPTpF6lP/Spa050tjnAKZOXNm6BAkAuWreU88AaNGwfz5Neu23RamTPGFk0hj1L+SJe35\n0hknEWmV996D00/3Y5ZqF01nn+3PMqloEpFiojNOItIiy5fD9dfDtddCZWXN+tJSf7mub99wsYmI\n5IsKJxGJZM0a+MMf4Mor4aOPatZvtZV/WO/w4Zr9W0SKlw5vgQwbNix0CBKB8uUflTJ9OvTu7ccy\nVRdN7dr59quvwjnnqGiS6NS/kiXt+dIZp0DSPvNq0qQ5X+vX+wfx/vKX8N//1n1t0CB/5mm33cLE\nJsUhzf0ridKeLxVOgQwZMiR0CBJBGvO1dq1/htw118Arr9R97eij4eqrYd99w8QmxSWN/SvJ0p4v\nFU4iUseqVXDbbTBhArz9dt3X9tvPF1JHHRUiMhGR8FQ4iQgAixfD734Hv/+9f75cbYcdBpdc4s80\naQJLEUkzDeMMZN68eaFDkAiKOV/l5fCd78COO/rxSrWLpmOOgaeegiefhP79VTRJfhRz/ypGac+X\nCqdAJk6cGDoEiaDY8lVZ6S/HHXywH6c0fbof0wT+LrnTT4fnn4eHH4ZDDw0aqqRAsfWvYpf2fOlS\nXSAzZswIHYJEUCz5euEFuOUWXygtXVr3tS22gHPPhfPOg+23DxOfpFOx9K+0SHu+VDgF0qVLl9Ah\nSARJztdHH8HMmTBtGvz73xu+vscecP75MHQoJHg3JcGS3L/SKO35UuEkUoSWL4f77vNnlh55xM/F\nVFuXLv5y3DnnwIEHauySiEiuVDiJFInly+Ghh/xklX/7W93nx1UrLYXvfx+GDIFu3eKPUUQk6TQ4\nPJDRo0eHDkEiKNR8ffaZvwR30kmw9dZ+Ju+ZM+sWTTvuCD//Ocyf7y/VnXuuiiYpLIXav6Rhac+X\nzjgF0qtXr9AhSASFki/nYMECeOABv8ybt+FlOIDu3X0RdeaZ/s45PT9OClmh9C/JTdrzpcIpkFGj\nRoUOQSIIma/PP/dzKc2ZAw8+uOFs3tW23RZOPhlOPRUOPxw6dIg1TJEW0/EwWdKeLxVOIgVmxQp/\nJumxx/xSUQFVVQ1vu8sucMIJvljq29fPwSQiIvmjwkkksKVL4bnn4OmnfaH0r3/VTEZZX4cO/vEn\nxx/vl912izdWEZG0U+EUyMKFC+ndu3foMCRHbZWvqip45RV49ln45z/98tJLfuxSY77xDejXD448\n0n/ddNNWhyFSUHQ8TJa050uFUyBjxoxh9uzZocOQHLUkX87BO+/Af/7jZ+x+7jlfMH3+edPv2223\nmiLpiCNgm21aHrdIEuh4mCxpz5cKp0AmT54cOgSJoLl8rVsHCxf6Iqm6UHrhBT9dQFPatYO99vLj\nk/r29YWSHnciaaPjYbKkPV8qnAJJ++2cSVOdrzVr4LXX4OWX6y6vvupfa87WW9cUSX37+gfsbrJJ\nnoMXKXA6HiZL2vOlwkmknnXrYNEieOONmuX11/0Zpddea3jepIb07An77OOXvfeGkhLYaSc93kRE\nJMlUOEnqOOfHGb33Hrz1li+KahdJb7/ti6dctW/vxyXtuWdNkbTPPn5eJRERKS6xFE5m1gO4ENgf\n2BfoChzhnHsqjs8vRBMmTODiiy8OHUZRWrYM3n23Znnvvbrtd9/1cyVFM4GOHS+md2/Yffe6y9e+\npskmRVpDx8NkSXu+4jrj9HVgNPAa8CLQN6bPLViVDT2BVRq1ahV8+KFfPvig5mvt76u/Ll/e8s/Z\nZBNfCO2yi/9avcyeXcn112uCSZF80PEwWdKer7gKp38DWzrnPjezU1HhxOWXXx46hCCqqvxlsk8+\n8cunn9Z831S7NcVQbZ07ww471Cy9etUtkrbZpuExSN/6VjrzJRKHtB4Pkyrt+YqlcHLORb4wIoVl\nzRqorPQFzNKldZcvvmi8Xf+1ZcuanuyxNTbfHHr08Et1YbT99nULpe7dNThbRERaToPDE8g5WL3a\nX75avbru9819XbnSF0ArVvil+vv6X+uvizJYuq20bw9bbAFbbumXHj38gOuGvm6zDXTqFH+MIiKS\nLokunJzzl36qqvwt4uvX1/2+fjuX75t7be1aX0SsXVuztKS9fPkSNtpoqya3b6z4yWW+oEKyySbQ\nrVvNUrsYqv197WWLLfy2hXJ2aMmSJWy11VahwxApSupfyZL2fEUunMzMgI1z2dY5tzpyRA04/HD/\ntX5B09gT45NhOFCYU9Z37OiLnS5dGv7arRtstllNIVT7+/rtTTf1Z46Sbvjw4al+xIBIPql/JUva\n89WSX2mHAY/nsJ0zsz7OuVdb8Bl1LF++oIlXOwG7N/MTXgZWNfF6z+zSmJVAUzEA9AE6N/H64uxS\nbTBQUatddz/at/eXnjp29Iu/DPUyHTqsYuON+XLp0MG/3qEDdO/ek+7de365fe2vHTvCRhut5JNP\nFtCpk1/fuXPN1+rv99mnD127Nr4fixcvZvHixY2+3qlTJ3bfvel8vPzyy6xa1Xg+evbsSc+ejedj\n5cqVLFjQdD769OlD585ttx/jx4/fYJsk7kdDim0/qmOpH1PS9qMxxbgfDfWvJO5HQ4pxPwYPHkxF\nRcUG2yRtP2prLkd1OOciLcC2wFk5Lps28P5TgfXAYTl8Vgngmlo6d97dHXqoc4cf7tw3vznHbbXV\nQHfMMc4NGODcwIHOnXiicx06dG/yZ5SWjnPnnefcqFHOHXDAWHfIIde60aOd+9nPnPvFL5w744xH\nmnw/4K69dr6bOdO5srIVbv/9B7pf//pp9+ijzj35pHP/+IdzRx55SpPv7917d1dZ6dzatc49/PAc\nN3DgQFdf9+5N78e4ceO+3Hbs2LHu2muvrfP+Rx5pfj/mz5/vnHNuxYoVbuDAge7pp5+u8zNOOaXp\n/dh9992/3HbOHO2H9kP7of3Qfmg/ErUfJa6Z2sRcvm5xakR2OoI7gSNdMxNgmlkJUD59+nT69OnT\n4Db6i6GG9qOG9sNL434sWLCAoUOHUv+4kbT9aIz2o4b2w9N+1GjNGaehQ4cClDrnNjydVksiCqfy\n8nJKSkpiiU9Ekq2iooLS0lJ03BCRXFUfN8ihcNoonpDAzC41s0vwl+oMOMvMLsmuS52pU6eGDkEi\nUL5E8kf9K1nSnq/YCifgiuwyGH8dcVi2ncopSBsaWCeFS/kSyR/1r2RJe75iu1HcORdnkVbwpkyZ\nEjoEiUD5Eskf9a9kSXu+VMyIiIiI5EiFk4iIiEiOVDiJiIiI5EiFUyCZTCZ0CBKB8iWSP+pfyZL2\nfKlwCmTkyJGhQ5AIlC+R/FH/Spa050uFUyD9+/cPHYJEoHyJ5I/6V7KkPV8qnERERERypMJJRERE\nJEcqnAKZNWtW6BAkAuVLJH/Uv5Il7flS4RRIWVlZ6BAkAuVLJH/Uv5Il7flS4RTIzJkzQ4cgEShf\nIvmj/pUsac+XCicRERGRHKlwEhEREcmRCicRERGRHKlwCmTYsGGhQ5AIlC+R/FH/Spa050uFUyBp\nn3k1aZQvkfxR/0qWtOdLhVMgQ4YMCR2CRKB8ieSP+leypD1fKpxEREREcqTCSURERCRHKpwCmTdv\nXugQJALlSyR/1L+SJe35UuEUyMSJE0OHIBEoXyL5o/6VLGnPlwqnQGbMmBE6BIlA+RLJH/WvZEl7\nvlQ4BdKlS5fQIUgEypdI/qh/JUva86XCSURERCRHKpxEREREcqTCKZDRo0eHDkEiUL5E8kf9K1nS\nni8VToH06tUrdAgSgfIlkj/qX8mS9nypcApk1KhRoUOQCJQvkfxR/0qWtOdLhZOIiIhIjlQ4iYiI\niORIhVMgCxcuDB2CRKB8ieSP+leypD1fKpwCGTNmTOgQJALlSyR/1L+SJe35UuEUyOTJk0OHIBEo\nXyL5o/6VLGnPlwqnQNJ+O2fSKF8i+aP+lSxpz5cKJxEREZEcqXASERERyZEKp0AmTJgQOgSJQPkS\nyR/1r2RJe75UOAVSWVkZOgSJQPkSyR/1r2RJe77MORc6hkaZWQlQXl5eTklJSehwRCQBKioqKC0t\nRccNEclV9XEDKHXOVTS1rc44iYiIiORIhZOIiIhIjlQ4BbJkyZLQIUgEypdI/qh/JUva86XCKZDh\nw4eHDkEiUL5E8kf9K1nSni8VToGMHz8+dAgSgfIlkj/qX8mS9nypcApEd/ski/Ilkj/qX8mS9nyp\ncBIRERHJUSyFk5n1M7OpZvaKma0wszfM7BYz6xHH54uIiIi0hbjOOE0ADgfuAUYBZcAgoMLMtokp\nhoIyderU0CFIBMqXSP6ofyVL2vMVV+F0kXNuF+fcz51ztzrnLgVOAHoAI2OKoaBUVDQ5MakUGOVL\nJH/Uv5Il7fmKpXByzs1rYN3TwKdAnzhiKDRTpkwJHYJEoHyJ5I/6V7KkPV/BBoeb2SZAVyDdM2mJ\niIhIYoS8q+4ioAMwI2AMIiIiIjlrH/UNZmbAxrls65xb3cjPOAwYC8x0zj0ZNQYRERGREFpyxukw\nYGUOS6WZ7Vb/zWbWG3933YvA91sWdvJlMpnQIUgEypdI/qh/JUva89WSwmkhcHYOyzBgce03mtkO\nwFzgM+B459yKXD5wwIABZDKZOkvfvn2ZNWtWne3mzp3bYEJHjBixwe2TFRUVZDKZDR5WOG7cOCZM\nmFBn3aJFi8hkMixcuLDO+kmTJjF69Og66yorK8lkMsybV3c8fFlZGcOGDfuyPXKkv5lw8ODBid6P\nasW+Hx9//HFR7Eex5EP7UVz70a1bt6LYj2LJR3P7Uf37K6n7UVpaSr9+/erUFIMGDdrgsxpjzrmc\nN24NM9sCeAbYHDjYOfdmDu8pAcrLy8tTP8W7iOSmoqKC0tJSdNwQkVxVHzeAUudck/MtRB7j1BJm\n1gV4COgJHJFL0SQiIiJSaGIpnIC/APsBU4E9zGyPWq8td87dF1McIiIiIi0W13QE3wQcMByYVm+5\nPqYYCkr967lS2JQvkfxR/0qWtOcrrpnDd3LOtWtk2TmOGApN/QFrUtiUL5H8Uf9KlrTnK+QEmKm2\n9dZbhw5BIlC+RPJH/StZ0p4vFU4iIiIiOVLhJCIiIpIjFU4iIiIiOYprOoKW6gSwYMGC0HG0ueee\ne46Kiibn2JIConwlR/XxohiPG8VK/StZijFftY4XnZrbNraZw1vCzM4A7ggdh4iIiKTCmc65vzS1\nQaEXTlsCxwBvA6vCRiMiIiJFqhPwVWCOc+6TpjYs6MJJREREpJBocLiIiIhIjlQ4iYiIiORIhZOI\niIhIjlQ4iYiIiORIhVMBMrM/mlmVmc0OHYtsyMz6mdlUM3vFzFaY2RtmdouZ9QgdW5qZ2cZmNsHM\n3jOzSjN71sy+FTou2ZCZ7Wtmk81svpktN7N3zGymme0aOjbJjZldmv099WLoWOKmu+oKjJmVAv8E\n1gJ/d85lAock9ZjZ80B34C7gNWBnYBSwAtjbOfdRwPBSy8xmACcD1wOvA2cD+wNHOOf+ETA0qcfM\n7gIOwvehF4Ee+D7UFTjAOfdywPCkGWa2HbAQcMDbzrm9AocUKxVOBcbMngFeBr4F/FeFU+Exs0Oc\nc/PqrTsUeBL4pXNubJjI0svM9geeBX7inLs+u64jMB/40Dl3SMj4pC4zOxD4t3NuXa11u+Dzdadz\n7qxgwUmzsn+kbIl/+siWaSucdKmugJjZWcAewCWhY5HG1S+asuueBj4F+sQfkQCnAeuAW6pXOOdW\nA1OBvtm/kKVAOOeerV00Zde9ji+c1IcKmJkdBpwCXBQ6llBUOBUIM+sKXANcpUs9yWNmm+AvMywJ\nHUtK7Q286pxbXm/9c7Vel8K3LepDBcvMNgJuBG5xzs0PHU8ohf6Q3zQZB6wEfhs6EGmRi4AOwIzQ\ngaRUT2BxA+sXAwZ8Jd5wJCozGwpsB1waOhZp1I+AXkC/0IGEpMKpjZmZARvnsm32UgJmthtwPjDY\nObc2j+FJPS3JVwM/4zBgLDDTOfdkG4YnuesMNJSfVbVelwJlZr2BycAzwLTA4UgDzGwL4HLgCufc\np6HjCUmX6treYfgzR80tldmCCfxZpmecc7PiDzf1WpKvL2UP+Pfg7wz6fkwxy4ZWAh0bWN+p1utS\ngMxsG+AB4DPg2053LBWqq4BP8AVuqumMU9tbiL8NOheLzawfcCxwspntmF1v+Nx0zq771Dm3rM0j\nFYiYr9oNM9sBmIs/4B/vnFvRtqFJBItp+HJcz+zX92OMRXJkZt2AOUA34BDn3AeBQ5IGZO94/D5w\nAbCdP1GP4f8w6ZD9PbXUOfdZuCjjo8KpjTnnPiTCqebsL18H3Fv/R+Gv97+JHz9zY1vFKDWi5qta\n9rT1XPy4piOyP0fCeQE4wsy61hsgfiC+L70QJixpTHa6iL8BuwBHOedeCRySNG47fKF0IzCpgdff\nBG4AfhxnUKFoHqfAzGx7oKSBl24B3gZ+Ccx3zr0VZ1zSODPrAjwOfB1fNOmXcmC15nH6qXPuN9l1\nG+dXoJcAAAE1SURBVONvb//YOXdwyPikruzdWffiz7ZnnHNzAockTTCzLYGG+tBV+LuJzwfedM69\nFGtggahwKlBm9haaALMgmdksIIOfI+iJei8vd87dF3tQgpnNBE7Cjxmsnjl8X6Cfc+6ZgKFJPWb2\nW/wv29n42cPrcM7dEXtQEpmZPU4KJ8BU4VSgzOxNfOF0YuhYpK5sUdurkZffcc7tHGc84mXPMF0J\nDMU/EudF4FLn3KNBA5MNZH/hHtbY6865djGGIy2UzeMWzrlvho4lTiqcRERERHKk6QhEREREcqTC\nSURERCRHKpxEREREcqTCSURERCRHKpxEREREcqTCSURERCRHKpxEREREcqTCSURERCRHKpxERERE\ncqTCSURERCRHKpxEREREcqTCSURERCRHKpxEREREcvT/DnMLv3xCVbkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x106582fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"elu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, name=None):\n",
    "  return tf.maximum(0.01 * z, z, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = fully_connected(X, n_hidden1, activation_fn=leaky_relu, scope=\"hidden1\")\n",
    "    hidden2 = fully_connected(hidden1, n_hidden2, activation_fn=leaky_relu, scope=\"hidden2\")\n",
    "    logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.initialize_all_variables()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.71 Test accuracy: 0.689\n",
      "1 Train accuracy: 0.74 Test accuracy: 0.8156\n",
      "2 Train accuracy: 0.84 Test accuracy: 0.849\n",
      "3 Train accuracy: 0.81 Test accuracy: 0.8676\n",
      "4 Train accuracy: 0.75 Test accuracy: 0.8769\n",
      "5 Train accuracy: 0.88 Test accuracy: 0.8865\n",
      "6 Train accuracy: 0.86 Test accuracy: 0.8909\n",
      "7 Train accuracy: 0.85 Test accuracy: 0.8962\n",
      "8 Train accuracy: 0.9 Test accuracy: 0.9003\n",
      "9 Train accuracy: 0.87 Test accuracy: 0.9035\n",
      "10 Train accuracy: 0.9 Test accuracy: 0.9072\n",
      "11 Train accuracy: 0.87 Test accuracy: 0.9095\n",
      "12 Train accuracy: 0.91 Test accuracy: 0.9112\n",
      "13 Train accuracy: 0.9 Test accuracy: 0.9118\n",
      "14 Train accuracy: 0.96 Test accuracy: 0.914\n",
      "15 Train accuracy: 0.92 Test accuracy: 0.9148\n",
      "16 Train accuracy: 0.88 Test accuracy: 0.9175\n",
      "17 Train accuracy: 0.94 Test accuracy: 0.9189\n",
      "18 Train accuracy: 0.94 Test accuracy: 0.9186\n",
      "19 Train accuracy: 0.87 Test accuracy: 0.9211\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 100\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import fully_connected, batch_norm\n",
    "from tensorflow.contrib.framework import arg_scope\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "learning_rate = 0.01\n",
    "momentum = 0.25\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "    batch_norm_params = {\n",
    "        'is_training': is_training,\n",
    "        'decay': 0.9,\n",
    "        'updates_collections': None,\n",
    "        'scale': True,\n",
    "    }\n",
    "\n",
    "    with arg_scope(\n",
    "            [fully_connected],\n",
    "            activation_fn=tf.nn.elu,\n",
    "            weights_initializer=he_init,\n",
    "            normalizer_fn=batch_norm,\n",
    "            normalizer_params=batch_norm_params):\n",
    "        hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
    "        hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
    "        logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.initialize_all_variables()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.9 Test accuracy: 0.8601\n",
      "1 Train accuracy: 0.88 Test accuracy: 0.8915\n",
      "2 Train accuracy: 0.9 Test accuracy: 0.9074\n",
      "3 Train accuracy: 0.86 Test accuracy: 0.9141\n",
      "4 Train accuracy: 0.9 Test accuracy: 0.923\n",
      "5 Train accuracy: 0.86 Test accuracy: 0.9255\n",
      "6 Train accuracy: 0.9 Test accuracy: 0.9315\n",
      "7 Train accuracy: 0.96 Test accuracy: 0.9352\n",
      "8 Train accuracy: 0.94 Test accuracy: 0.9391\n",
      "9 Train accuracy: 0.98 Test accuracy: 0.9429\n",
      "10 Train accuracy: 0.96 Test accuracy: 0.9451\n",
      "11 Train accuracy: 1.0 Test accuracy: 0.9458\n",
      "12 Train accuracy: 0.96 Test accuracy: 0.9506\n",
      "13 Train accuracy: 0.98 Test accuracy: 0.9527\n",
      "14 Train accuracy: 0.98 Test accuracy: 0.9528\n",
      "15 Train accuracy: 1.0 Test accuracy: 0.9551\n",
      "16 Train accuracy: 0.92 Test accuracy: 0.9563\n",
      "17 Train accuracy: 0.96 Test accuracy: 0.9579\n",
      "18 Train accuracy: 0.98 Test accuracy: 0.9567\n",
      "19 Train accuracy: 0.98 Test accuracy: 0.9586\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "    batch_norm_params = {\n",
    "        'is_training': is_training,\n",
    "        'decay': 0.9,\n",
    "        'updates_collections': None,\n",
    "        'scale': True,\n",
    "    }\n",
    "\n",
    "    with arg_scope(\n",
    "            [fully_connected],\n",
    "            activation_fn=tf.nn.elu,\n",
    "            weights_initializer=he_init,\n",
    "            normalizer_fn=batch_norm,\n",
    "            normalizer_params=batch_norm_params,\n",
    "            weights_regularizer=tf.contrib.layers.l1_regularizer(0.01)):\n",
    "        hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
    "        hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
    "        logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
    "    reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"base_loss\")\n",
    "    loss = tf.add(base_loss, reg_losses, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.initialize_all_variables()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 1.0 Test accuracy: 0.897\n",
      "1 Train accuracy: 0.98 Test accuracy: 0.9054\n",
      "2 Train accuracy: 0.98 Test accuracy: 0.8917\n",
      "3 Train accuracy: 1.0 Test accuracy: 0.856\n",
      "4 Train accuracy: 1.0 Test accuracy: 0.8384\n",
      "5 Train accuracy: 0.94 Test accuracy: 0.8679\n",
      "6 Train accuracy: 0.98 Test accuracy: 0.8621\n",
      "7 Train accuracy: 1.0 Test accuracy: 0.886\n",
      "8 Train accuracy: 0.98 Test accuracy: 0.8748\n",
      "9 Train accuracy: 0.94 Test accuracy: 0.8113\n",
      "10 Train accuracy: 0.96 Test accuracy: 0.8763\n",
      "11 Train accuracy: 0.98 Test accuracy: 0.8973\n",
      "12 Train accuracy: 0.96 Test accuracy: 0.8802\n",
      "13 Train accuracy: 0.98 Test accuracy: 0.9116\n",
      "14 Train accuracy: 0.98 Test accuracy: 0.8795\n",
      "15 Train accuracy: 0.98 Test accuracy: 0.9038\n",
      "16 Train accuracy: 0.98 Test accuracy: 0.9253\n",
      "17 Train accuracy: 0.98 Test accuracy: 0.9082\n",
      "18 Train accuracy: 1.0 Test accuracy: 0.8855\n",
      "19 Train accuracy: 1.0 Test accuracy: 0.9014\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/weights:0',\n",
       " 'hidden1/BatchNorm/beta:0',\n",
       " 'hidden1/BatchNorm/gamma:0',\n",
       " 'hidden1/BatchNorm/moving_mean:0',\n",
       " 'hidden1/BatchNorm/moving_variance:0',\n",
       " 'hidden2/weights:0',\n",
       " 'hidden2/BatchNorm/beta:0',\n",
       " 'hidden2/BatchNorm/gamma:0',\n",
       " 'hidden2/BatchNorm/moving_mean:0',\n",
       " 'hidden2/BatchNorm/moving_variance:0',\n",
       " 'outputs/weights:0',\n",
       " 'outputs/BatchNorm/beta:0',\n",
       " 'outputs/BatchNorm/gamma:0',\n",
       " 'outputs/BatchNorm/moving_mean:0',\n",
       " 'outputs/BatchNorm/moving_variance:0',\n",
       " 'train/hidden1/weights/Momentum:0',\n",
       " 'train/hidden1/BatchNorm/beta/Momentum:0',\n",
       " 'train/hidden1/BatchNorm/gamma/Momentum:0',\n",
       " 'train/hidden2/weights/Momentum:0',\n",
       " 'train/hidden2/BatchNorm/beta/Momentum:0',\n",
       " 'train/hidden2/BatchNorm/gamma/Momentum:0',\n",
       " 'train/outputs/weights/Momentum:0',\n",
       " 'train/outputs/BatchNorm/beta/Momentum:0',\n",
       " 'train/outputs/BatchNorm/gamma/Momentum:0']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.all_variables()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"\", default_name=\"\", reuse=True):  # root scope\n",
    "    weights1 = tf.get_variable(\"hidden1/weights\")\n",
    "    weights2 = tf.get_variable(\"hidden2/weights\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0.    0.]\n",
      " [   3.    4.]\n",
      " [  30.   40.]\n",
      " [ 300.  400.]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.constant([0., 0., 3., 4., 30., 40., 300., 400.], shape=(4, 2))\n",
    "c = tf.clip_by_norm(x, clip_norm=10)\n",
    "c0 = tf.clip_by_norm(x, clip_norm=350, axes=0)\n",
    "c1 = tf.clip_by_norm(x, clip_norm=10, axes=1)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    xv = x.eval()\n",
    "    cv = c.eval()\n",
    "    c0v = c0.eval()\n",
    "    c1v = c1.eval()\n",
    "\n",
    "print(xv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.        ]\n",
      " [ 0.05969927  0.07959903]\n",
      " [ 0.59699273  0.79599035]\n",
      " [ 5.96992731  7.95990324]]\n"
     ]
    }
   ],
   "source": [
    "print(cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0.            0.        ]\n",
      " [   3.            3.48245788]\n",
      " [  30.           34.82457733]\n",
      " [ 300.          348.24578857]]\n"
     ]
    }
   ],
   "source": [
    "print(c0v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 301.51119995  350.        ]\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(c0v, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.        ]\n",
      " [ 3.          4.        ]\n",
      " [ 6.          8.        ]\n",
      " [ 6.00000048  8.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(c1v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0.   5.  10.  10.]\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(c1v, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\", collection=\"max_norm\"):\n",
    "    def max_norm(weights):\n",
    "        clip_weights = tf.assign(weights, tf.clip_by_norm(weights, clip_norm=threshold, axes=axes), name=name)\n",
    "        tf.add_to_collection(collection, clip_weights)\n",
    "        return None # there is no regularization loss term\n",
    "    return max_norm\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    with arg_scope(\n",
    "            [fully_connected],\n",
    "            weights_regularizer=max_norm_regularizer(1.5)):\n",
    "        hidden1 = fully_connected(X, n_hidden1, scope=\"hidden1\")\n",
    "        hidden2 = fully_connected(hidden1, n_hidden2, scope=\"hidden2\")\n",
    "        logits = fully_connected(hidden2, n_outputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "clip_all_weights = tf.get_collection(\"max_norm\")\n",
    "        \n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    threshold = 1.0\n",
    "    grads_and_vars = optimizer.compute_gradients(loss)\n",
    "    capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "                  for grad, var in grads_and_vars]\n",
    "    training_op = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.initialize_all_variables()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.84 Test accuracy: 0.8401\n",
      "1 Train accuracy: 0.9 Test accuracy: 0.8855\n",
      "2 Train accuracy: 0.9 Test accuracy: 0.8979\n",
      "3 Train accuracy: 0.9 Test accuracy: 0.9078\n",
      "4 Train accuracy: 0.92 Test accuracy: 0.9146\n",
      "5 Train accuracy: 0.96 Test accuracy: 0.9144\n",
      "6 Train accuracy: 0.9 Test accuracy: 0.9203\n",
      "7 Train accuracy: 0.92 Test accuracy: 0.9234\n",
      "8 Train accuracy: 0.92 Test accuracy: 0.9237\n",
      "9 Train accuracy: 0.92 Test accuracy: 0.9287\n",
      "10 Train accuracy: 0.94 Test accuracy: 0.931\n",
      "11 Train accuracy: 0.94 Test accuracy: 0.9331\n",
      "12 Train accuracy: 0.98 Test accuracy: 0.9329\n",
      "13 Train accuracy: 0.98 Test accuracy: 0.9342\n",
      "14 Train accuracy: 0.92 Test accuracy: 0.9373\n",
      "15 Train accuracy: 0.96 Test accuracy: 0.9386\n",
      "16 Train accuracy: 0.94 Test accuracy: 0.9399\n",
      "17 Train accuracy: 0.9 Test accuracy: 0.939\n",
      "18 Train accuracy: 0.88 Test accuracy: 0.941\n",
      "19 Train accuracy: 0.96 Test accuracy: 0.9433\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.14695850554017442&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;is_training&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;X&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/unpack&quot;\\n  op: &quot;Unpack&quot;\\n  input: &quot;dnn/hidden1/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;axis&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;num&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;hidden1/weights/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;Div&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;hidden1/weights/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;hidden1/biases/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/unpack&quot;\\n  op: &quot;Unpack&quot;\\n  input: &quot;dnn/hidden2/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;axis&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;num&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;hidden2/weights/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;Div&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;hidden2/weights/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;hidden2/biases/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/unpack&quot;\\n  op: &quot;Unpack&quot;\\n  input: &quot;dnn/outputs/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;axis&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;num&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/weights&quot;\\n  input: &quot;outputs/weights/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/weights/read&quot;\\n  input: &quot;outputs/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/weights/read&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;Div&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/weights/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/weights&quot;\\n  input: &quot;outputs/weights/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/biases&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/biases/Initializer/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/biases&quot;\\n  input: &quot;outputs/biases/Initializer/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;Div&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;Div&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value/Minimum&quot;\\n  input: &quot;train/clip_by_value/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_1/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_1/Minimum&quot;\\n  input: &quot;train/clip_by_value_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_2/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_2/Minimum&quot;\\n  input: &quot;train/clip_by_value_2/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_3/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_3/Minimum&quot;\\n  input: &quot;train/clip_by_value_3/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_4/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_4/Minimum&quot;\\n  input: &quot;train/clip_by_value_4/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_5/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_5/Minimum&quot;\\n  input: &quot;train/clip_by_value_5/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 784\\n          }\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/weights/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/weights/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden1/weights/Momentum&quot;\\n  input: &quot;train/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/weights/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/hidden1/weights/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/biases/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/biases/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden1/biases/Momentum&quot;\\n  input: &quot;train/zeros_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden1/biases/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/hidden1/biases/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/weights/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/weights/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden2/weights/Momentum&quot;\\n  input: &quot;train/zeros_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/weights/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/hidden2/weights/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_3&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/biases/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/biases/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden2/biases/Momentum&quot;\\n  input: &quot;train/zeros_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/hidden2/biases/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/hidden2/biases/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_4&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/weights/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/weights/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/outputs/weights/Momentum&quot;\\n  input: &quot;train/zeros_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/weights/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/outputs/weights/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_5&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/biases/Momentum&quot;\\n  op: &quot;Variable&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/biases/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/outputs/biases/Momentum&quot;\\n  input: &quot;train/zeros_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/outputs/biases/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/outputs/biases/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/momentum&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.25\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden1/weights/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;train/hidden1/weights/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden1/biases/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;train/hidden1/biases/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_1&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden2/weights/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;train/hidden2/weights/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_2&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden2/biases/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;train/hidden2/biases/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_3&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_outputs/weights/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;outputs/weights&quot;\\n  input: &quot;train/outputs/weights/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_4&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_outputs/biases/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;outputs/biases&quot;\\n  input: &quot;train/outputs/biases/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_5&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/Momentum/update_hidden1/weights/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden1/biases/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden2/weights/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden2/biases/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_outputs/weights/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_outputs/biases/ApplyMomentum&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/weights/Assign&quot;\\n  input: &quot;^hidden1/biases/Assign&quot;\\n  input: &quot;^hidden2/weights/Assign&quot;\\n  input: &quot;^hidden2/biases/Assign&quot;\\n  input: &quot;^outputs/weights/Assign&quot;\\n  input: &quot;^outputs/biases/Assign&quot;\\n  input: &quot;^train/hidden1/weights/Momentum/Assign&quot;\\n  input: &quot;^train/hidden1/biases/Momentum/Assign&quot;\\n  input: &quot;^train/hidden2/weights/Momentum/Assign&quot;\\n  input: &quot;^train/hidden2/biases/Momentum/Assign&quot;\\n  input: &quot;^train/outputs/weights/Momentum/Assign&quot;\\n  input: &quot;^train/outputs/biases/Momentum/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/save/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;hidden1/biases&quot;\\n        string_val: &quot;hidden1/weights&quot;\\n        string_val: &quot;hidden2/biases&quot;\\n        string_val: &quot;hidden2/weights&quot;\\n        string_val: &quot;outputs/biases&quot;\\n        string_val: &quot;outputs/weights&quot;\\n        string_val: &quot;train/hidden1/biases/Momentum&quot;\\n        string_val: &quot;train/hidden1/weights/Momentum&quot;\\n        string_val: &quot;train/hidden2/biases/Momentum&quot;\\n        string_val: &quot;train/hidden2/weights/Momentum&quot;\\n        string_val: &quot;train/outputs/biases/Momentum&quot;\\n        string_val: &quot;train/outputs/weights/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/save/shapes_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/save&quot;\\n  op: &quot;SaveSlices&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/save/tensor_names&quot;\\n  input: &quot;save/save/shapes_and_slices&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;outputs/biases&quot;\\n  input: &quot;outputs/weights&quot;\\n  input: &quot;train/hidden1/biases/Momentum&quot;\\n  input: &quot;train/hidden1/weights/Momentum&quot;\\n  input: &quot;train/hidden2/biases/Momentum&quot;\\n  input: &quot;train/hidden2/weights/Momentum&quot;\\n  input: &quot;train/outputs/biases/Momentum&quot;\\n  input: &quot;train/outputs/weights/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/save&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;hidden1/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice/tensor_name&quot;\\n  input: &quot;save/restore_slice/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;save/restore_slice&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_1/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_1/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_1&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_1/tensor_name&quot;\\n  input: &quot;save/restore_slice_1/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;save/restore_slice_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_2/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;hidden2/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_2/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_2&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_2/tensor_name&quot;\\n  input: &quot;save/restore_slice_2/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;save/restore_slice_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_3/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_3/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_3&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_3/tensor_name&quot;\\n  input: &quot;save/restore_slice_3/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;save/restore_slice_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_4/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;outputs/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_4/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_4&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_4/tensor_name&quot;\\n  input: &quot;save/restore_slice_4/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/biases&quot;\\n  input: &quot;save/restore_slice_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_5/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;outputs/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_5/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_5&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_5/tensor_name&quot;\\n  input: &quot;save/restore_slice_5/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/weights&quot;\\n  input: &quot;save/restore_slice_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_6/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/hidden1/biases/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_6/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_6&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_6/tensor_name&quot;\\n  input: &quot;save/restore_slice_6/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_6&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden1/biases/Momentum&quot;\\n  input: &quot;save/restore_slice_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_7/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/hidden1/weights/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_7/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_7&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_7/tensor_name&quot;\\n  input: &quot;save/restore_slice_7/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_7&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden1/weights/Momentum&quot;\\n  input: &quot;save/restore_slice_7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_8/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/hidden2/biases/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_8/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_8&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_8/tensor_name&quot;\\n  input: &quot;save/restore_slice_8/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_8&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden2/biases/Momentum&quot;\\n  input: &quot;save/restore_slice_8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_9/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/hidden2/weights/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_9/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_9&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_9/tensor_name&quot;\\n  input: &quot;save/restore_slice_9/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_9&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/hidden2/weights/Momentum&quot;\\n  input: &quot;save/restore_slice_9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_10/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/outputs/biases/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_10/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_10&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_10/tensor_name&quot;\\n  input: &quot;save/restore_slice_10/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_10&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/outputs/biases/Momentum&quot;\\n  input: &quot;save/restore_slice_10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_11/tensor_name&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;train/outputs/weights/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_11/shape_and_slice&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_slice_11&quot;\\n  op: &quot;RestoreSlice&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/restore_slice_11/tensor_name&quot;\\n  input: &quot;save/restore_slice_11/shape_and_slice&quot;\\n  attr {\\n    key: &quot;dt&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;preferred_shard&quot;\\n    value {\\n      i: -1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_11&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;train/outputs/weights/Momentum&quot;\\n  input: &quot;save/restore_slice_11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n  input: &quot;^save/Assign_6&quot;\\n  input: &quot;^save/Assign_7&quot;\\n  input: &quot;^save/Assign_8&quot;\\n  input: &quot;^save/Assign_9&quot;\\n  input: &quot;^save/Assign_10&quot;\\n  input: &quot;^save/Assign_11&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.14695850554017442&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import dropout\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "initial_learning_rate = 0.1\n",
    "decay_steps = 10000\n",
    "decay_rate = 1/10\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n",
    "                                           decay_steps, decay_rate)\n",
    "\n",
    "keep_prob = 0.5\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "    with arg_scope(\n",
    "            [fully_connected],\n",
    "            activation_fn=tf.nn.elu,\n",
    "            weights_initializer=he_init):\n",
    "        X_drop = dropout(X, keep_prob, is_training=is_training)\n",
    "        hidden1 = fully_connected(X_drop, n_hidden1, scope=\"hidden1\")\n",
    "        hidden1_drop = dropout(hidden1, keep_prob, is_training=is_training)\n",
    "        hidden2 = fully_connected(hidden1_drop, n_hidden2, scope=\"hidden2\")\n",
    "        hidden2_drop = dropout(hidden2, keep_prob, is_training=is_training)\n",
    "        logits = fully_connected(hidden2_drop, n_outputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss, global_step=global_step)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.initialize_all_variables()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.92 Test accuracy: 0.8804\n",
      "1 Train accuracy: 0.96 Test accuracy: 0.8983\n",
      "2 Train accuracy: 0.9 Test accuracy: 0.9074\n",
      "3 Train accuracy: 0.88 Test accuracy: 0.9004\n",
      "4 Train accuracy: 0.86 Test accuracy: 0.9164\n",
      "5 Train accuracy: 0.92 Test accuracy: 0.9149\n",
      "6 Train accuracy: 0.9 Test accuracy: 0.9174\n",
      "7 Train accuracy: 0.96 Test accuracy: 0.9266\n",
      "8 Train accuracy: 0.9 Test accuracy: 0.9204\n",
      "9 Train accuracy: 0.96 Test accuracy: 0.9259\n",
      "10 Train accuracy: 0.9 Test accuracy: 0.9253\n",
      "11 Train accuracy: 0.98 Test accuracy: 0.9296\n",
      "12 Train accuracy: 0.9 Test accuracy: 0.9346\n",
      "13 Train accuracy: 0.96 Test accuracy: 0.9334\n",
      "14 Train accuracy: 0.92 Test accuracy: 0.9321\n",
      "15 Train accuracy: 0.98 Test accuracy: 0.9363\n",
      "16 Train accuracy: 0.92 Test accuracy: 0.9375\n",
      "17 Train accuracy: 0.96 Test accuracy: 0.932\n",
      "18 Train accuracy: 0.9 Test accuracy: 0.9342\n",
      "19 Train accuracy: 0.9 Test accuracy: 0.9366\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                               scope=\"hidden[2]|outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training_op2 = optimizer.minimize(loss, var_list=train_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:0\n",
      "hidden1/weights:0\n",
      "hidden1/biases:0\n",
      "hidden2/weights:0\n",
      "hidden2/biases:0\n",
      "outputs/weights:0\n",
      "outputs/biases:0\n",
      "train/hidden1/weights/Momentum:0\n",
      "train/hidden1/biases/Momentum:0\n",
      "train/hidden2/weights/Momentum:0\n",
      "train/hidden2/biases/Momentum:0\n",
      "train/outputs/weights/Momentum:0\n",
      "train/outputs/biases/Momentum:0\n"
     ]
    }
   ],
   "source": [
    "for i in tf.all_variables():\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden1/weights:0\n",
      "hidden1/biases:0\n",
      "hidden2/weights:0\n",
      "hidden2/biases:0\n",
      "outputs/weights:0\n",
      "outputs/biases:0\n"
     ]
    }
   ],
   "source": [
    "for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden2/weights:0\n",
      "hidden2/biases:0\n",
      "outputs/weights:0\n",
      "outputs/biases:0\n"
     ]
    }
   ],
   "source": [
    "for i in train_vars:\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:\n",
      "Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.\n",
      "WARNING:tensorflow:Given features: Tensor(\"input:0\", shape=(?, 784), dtype=float32), required signatures: TensorSignature(dtype=tf.float32, shape=TensorShape([Dimension(None), Dimension(784)]), is_sparse=False).\n",
      "WARNING:tensorflow:Given targets: Tensor(\"output:0\", shape=(?,), dtype=int64), required signatures: TensorSignature(dtype=tf.int64, shape=TensorShape([Dimension(None)]), is_sparse=False).\n",
      "WARNING:tensorflow:Given features: Tensor(\"input:0\", shape=(?, 784), dtype=float32), required signatures: TensorSignature(dtype=tf.float32, shape=TensorShape([Dimension(None), Dimension(784)]), is_sparse=False).\n",
      "WARNING:tensorflow:Given targets: Tensor(\"output:0\", shape=(?,), dtype=int64), required signatures: TensorSignature(dtype=tf.int64, shape=TensorShape([Dimension(None)]), is_sparse=False).\n",
      "WARNING:tensorflow:Given features: Tensor(\"input:0\", shape=(?, 784), dtype=float32), required signatures: TensorSignature(dtype=tf.float32, shape=TensorShape([Dimension(None), Dimension(784)]), is_sparse=False).\n",
      "WARNING:tensorflow:Given targets: Tensor(\"output:0\", shape=(?,), dtype=int64), required signatures: TensorSignature(dtype=tf.int64, shape=TensorShape([Dimension(None)]), is_sparse=False).\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-070bc81138b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0msteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m40000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m             \u001b[0mmonitors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvalidation_monitor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m     )\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/dnn.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, input_fn, steps, batch_size, monitors, max_steps)\u001b[0m\n\u001b[1;32m    433\u001b[0m     result = self._estimator.fit(x=x, y=y, input_fn=input_fn, steps=steps,\n\u001b[1;32m    434\u001b[0m                                  \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmonitors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m                                  max_steps=max_steps)\n\u001b[0m\u001b[1;32m    436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmonitors\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, input_fn, steps, batch_size, monitors, max_steps)\u001b[0m\n\u001b[1;32m    331\u001b[0m                              \u001b[0msteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m                              \u001b[0mmonitors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmonitors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m                              max_steps=max_steps)\n\u001b[0m\u001b[1;32m    334\u001b[0m     \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Loss for final step: %s.'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py\u001b[0m in \u001b[0;36m_train_model\u001b[0;34m(self, input_fn, steps, feed_fn, init_op, init_feed_fn, init_fn, device_fn, monitors, log_every_steps, fail_on_nan_loss, max_steps)\u001b[0m\n\u001b[1;32m    706\u001b[0m           \u001b[0mfail_on_nan_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfail_on_nan_loss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    707\u001b[0m           \u001b[0mhooks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhooks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 708\u001b[0;31m           max_steps=max_steps)\n\u001b[0m\u001b[1;32m    709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    710\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extract_metric_update_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/graph_actions.py\u001b[0m in \u001b[0;36m_monitored_train\u001b[0;34m(graph, output_dir, train_op, loss_op, global_step_tensor, init_op, init_feed_dict, init_fn, log_every_steps, supervisor_is_chief, supervisor_master, supervisor_save_model_secs, keep_checkpoint_max, supervisor_save_summaries_steps, feed_fn, steps, fail_on_nan_loss, hooks, max_steps)\u001b[0m\n\u001b[1;32m    283\u001b[0m       \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msuper_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m         _, loss = super_sess.run([train_op, loss_op], feed_fn() if feed_fn else\n\u001b[0;32m--> 285\u001b[0;31m                                  None)\n\u001b[0m\u001b[1;32m    286\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/monitored_session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    366\u001b[0m                           \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m                           \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 368\u001b[0;31m                           run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m    369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mshould_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/monitored_session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    519\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                               \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m                               run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m    522\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAbortedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/monitored_session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    486\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    487\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/monitored_session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    617\u001b[0m                                   \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    618\u001b[0m                                   \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 619\u001b[0;31m                                   run_metadata=run_metadata)\n\u001b[0m\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/monitored_session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    486\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    487\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    715\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 717\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    718\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    913\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 915\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    916\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m--> 965\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    970\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    971\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 972\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    973\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    974\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m    952\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m    953\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 954\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m    955\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    956\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "X_train = mnist.train.images\n",
    "y_train = mnist.train.labels.astype(\"int\")\n",
    "X_val = mnist.test.images[8000:]\n",
    "y_val = mnist.test.labels[8000:].astype(\"int\")\n",
    "\n",
    "feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(\n",
    "        feature_columns = feature_columns,\n",
    "        hidden_units=[300, 100],\n",
    "        n_classes=10,\n",
    "        model_dir=\"/tmp/my_model\",\n",
    "        config=tf.contrib.learn.RunConfig(save_checkpoints_secs=60)\n",
    "    )\n",
    "\n",
    "validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(\n",
    "        X_val,\n",
    "        y_val,\n",
    "        every_n_steps=50,\n",
    "        early_stopping_metric=\"loss\",\n",
    "        early_stopping_metric_minimize=True,\n",
    "        early_stopping_rounds=2000\n",
    "    )\n",
    "\n",
    "dnn_clf.fit(x=X_train,\n",
    "            y=y_train,\n",
    "            steps=40000,\n",
    "            monitors=[validation_monitor]\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Coming soon**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}