10_introduction_to_artificial_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 10 – Introduction to Artificial Neural Networks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ann\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Perceptrons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "iris = load_iris()\n",
    "X = iris.data[:, (2, 3)]  # petal length, petal width\n",
    "y = (iris.target == 0).astype(np.int)\n",
    "\n",
    "per_clf = Perceptron(random_state=42)\n",
    "per_clf.fit(X, y)\n",
    "\n",
    "y_pred = per_clf.predict([[2, 0.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure perceptron_iris_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAEYCAYAAABBfQDEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XdcleX/x/HXBQo4UZy5M0dOcpQ5cqappQ2tNMVE/aoN\n7Wc5cg8QFUXUMs1c5R5p2ddyTyxF6luau9yaI/dCGffvD/S4QBHh3Iz38/HgIedz7uu+P+dA9OHm\nuq6PsSwLERERERGJ4WJ3AiIiIiIiyYkKZBERERGRO6hAFhERERG5gwpkEREREZE7qEAWEREREbmD\nCmQRERERkTuoQBYRERERuYNTC2RjjJsxZrIx5qAx5oIx5ldjTMMHHN/NGPOPMebczXHpnZmviIiI\niKQ9zr6DnA44DLxgWZYnMACYb4wpdO+BxpiXgJ5AHaAI8BQw2HmpioiIiEhaZOzupGeM+QMYZFnW\n4nvis4ADlmX1u/m4LjDLsqwnbEhTRERERNIIW+cgG2PyAMWBHbE8XQb4447HfwC5jTHZnZGbiIiI\niKRNthXIxph0wExgumVZe2M5JDNw4Y7HFwADZHFCeiIiIiKSRqWz46LGGENMcXwd6BLHYZeBrHc8\nzgpYwKVYzmfvPBERERERSREsyzIPO8aWAhmYAuQEGluWFRXHMTsAb2DhzcfPACctyzoX28E3bnyX\n6EkmtZMnz9OhwziWL//NEXN3z0zLll/w/PM+NmaWcv3wwyCaNBlkdxpiM30fyC36XhDQ90FqEhQ0\niH37Bt0XL158EJ98cn/8/jEPrY0BG6ZYGGMmAk8DTS3LuvGAQ78B2htjSt2cd9wXmOaMHJ0lT55s\nfP99PwIDfUmfPuZ3levXLzN9ehumTfPh2rWLNmcoIiIikvY49Q7yze3cOgLhwMmYmRZYQCcghJi7\nxqUtyzpqWdZyY0wgsBbwIOZO8iBn5usMLi4u/N//vUqtWmVp1SqIv/46DsCWLTPZv/8X2refQ5Ei\nz9qcpYiIiIj9cuf2ILZyMCb+8DH79sXvOrZv85YYjDFWSpxica/Ll6/Rrdtkvv56tSOWLp0rTZoE\nUL9+d1xc1PjwYfbsWUfJkrXtTkNspu8DuUXfCwL6PpDbOnc28ZqDrAI5GZo7dwMffDCBS5euOWJP\nP/0ivr7f4OmpbaBFREREEiK+BbJuSSZDLVrUZOvWYJ57roQjtnv3Kvz9vdm+/UcbMxMRERFJ/VQg\nJ1NFi+Zl7doAevZsxs252ly6dJrx419m/vxuRERctzlDERERkdRJUyxSgLVrt9G2bTD//HN7h7uC\nBZ+hffu55M1b0sbMRERERJK3GTOGc+pUOAD79g3WFIvUok6d8vz661gaN67siB058jsBARXZtGkq\nqeGXHBEREZGkcOpUOPv2xb5/clxUIKcQOXNmZfHivowZ8x/c3dMDcOPGVWbMaM+UKS25evW8zRmK\niIiIpA4qkFMQYwzvv/8yISGBlCxZwBEPC5vH0KEV2L//FxuzExEREUkdVCCnQN7eT7J58yjat6/v\niJ05c5BRo17gxx+HEh0dV/duEREREXkYFcgpVKZMHkyY8AFz5vQkW7ZMAERHR7FkST/GjHmRc+eO\n2ZyhiIiISMqkAjmFa9asGlu3BlOtWilHbO/edfj7l+ePP5bYmJmIiIiI/XLn9qB48UEULz4o3mO0\nzVsqERkZxdCh8xk2bAHR0dGOeK1aH9Cs2Ujc3DLYmJ2IiIiI/dRJL41Jl86VgQNbsnKlHwUK5HDE\n168fz4gRVTh+fKeN2YmIiIikHCqQU5kXXihDWNgYXn31eUfs2LHtDBtWmY0bJ2nPZBEREZGHUIGc\nCnl5ZWH+/F58/nlnPDzcAIiIuMasWZ2YNKk5V66ctTlDERERkeRLc5BTuT//PISPTxA7dhx2xLJn\nL0i7drMoXvwFGzMTERGRtKxbt/qEh+e4L+7hcYbg4JWxjhk4sAUXLnjcF/f0DGfw4LkPvWZ85yCn\ne+iZJEUrW7YwP/88kl69pjNx4k8AnDt3hNGja9O4cX8aN+6Hq6u+DURERMS5wsNzYFn3F7Xh4S3i\nHHPhggfh4dNjeaZtouUFmmKRJmTI4M64cZ1YsOBTvLyyAGBZ0SxdOpjg4DqcPXv4IWcQERERSTtU\nIKchr776PGFhwdSsWcYR++uvEPz9vfntt29tzExEREQk+VCBnMYUKJCT5cuHMGjQO7i6xnz5r149\nz6RJzZk1qzM3bly1OUMRERERe6lAToNcXV3p0+ctVq8eSqFCuRzxjRu/ZNiwZzl6dJuN2YmIiIjY\nS6uz0rBq1UoRFhbMe+99wbff/gzAP//sZPjw52jePIhatd7HmIcu9BQRERF5ZB4eZ2JdkOfhcSbO\nMZ6e4cS2IC8mnni0zZtgWRbTp6+iW7fJXL163REvX74pbdpMIXPmnDZmJyIiIpI41Gpa4s0Yg69v\nfTZvDqJ8+SKO+LZtS/Dz82bPnrX2JSciIiLiZCqQxeHppwsQEhJIly6vOGIXLhxnzJh6fP99P6Ki\nImzMTkRERMQ5VCDLXTw83AgK6sDixX3JmTMrEDMF46efhhIUVIt//z1ob4IiIiIiSUxzkCVOx4+f\npV27MaxZc3tXCw+PrLRuPYnKld+2MTMRERGJjxkzhnPq1P0L2HLn9sDH51MbMnK+O9+DffsGq9W0\nPJ58+bz48cdBBAV9x8CBs4iMjCI8/CKTJ7dg584VvP32ONzdM9mdpoiIiMTh1Klw9u0bFMszscVS\np7vfg8HxGqMpFvJALi4u9OjxBuvWDePJJ/M44j//PJWAgIocPvw/G7MTERERSXwqkCVennuuBFu3\nBtOiRU1H7OTJvQQGPs/q1WNIDVN1REREREAFsjyCrFkz8vXX3Zgy5SMyZfIAIDLyBgsWdGP8+Fe4\nePGUzRmKiIiIPD4VyPJIjDH4+NRhy5YgKlQo6oj/+eeP+Pt7s3PnShuzExEREXl8WqQnCVKiRH42\nbhxB//4zCQ7+HoCLF08wblwDGjToSdOmfqRL52ZzliIiImlb7twexLYgLyaeNtz5HuzbF78x2uZN\nHtuKFf+jXbsxnDp1wRErXPhZOnSYQ65cT9mYmYiIiMhtajUtTtOgQQV+/XUsDRpUcMQOHdqKv/8z\nbNky08bMRERERB6dCmRJFHnyZGPJkv4EBvqSPn3MzJ3r1y8zbZoP06a1ITz8ks0ZioiIiMSPCmRJ\nNC4uLvzf/73Kxo3DKVbsCUd8y5YZDB1agUOHwmzMTkRERCR+VCBLoqtYsRhbtozGx6eOI3b69N+M\nGFGVFStGEh0dbWN2IiIiIg+mRXqSpObMWc+HH07k0qVrjlipUvVp2/YbPD3z2piZiIhI6jdjxnBO\nnQq/L547twc+Pp+muOs8rvgu0tM2b5KkWrasRZUqJfHxCWLr1pi9VXbtWom/f3neffdrypZtZHOG\nIiIiqdepU+Hs2zcolmdiiyX/6ziLplhIkitaNC/r1g2jR49mGBPzS9ulS6f5/PPGLFjwMRER123O\nUEREROQ2FcjiFOnTp2PoUB9++mkQTzyR3RFfvTqYwMCqnDy518bsRERERG5TgSxOVbeuN2FhY2jc\nuLIjduTI/wgM9Obnn6eTGubEi4iISMqmAlmcLlcuTxYv7svo0R1wc4uZBn/lSjjffOPLlCnvcO3a\nhYecQURERCTpaJGe2MIYw4cfvkKNGqXx8RnNnj1HAQgLm8uBA5tp334ORYs+b3OWIiIiKVvu3B7E\ntlAuJp7yruMs2uZNbHflSjiffDKFqVNXOmIuLq40aTKEl17qhYuLq43ZiYiISGoR323eNMVCbJcp\nkwcTJ37A7Nk98PTMCEB0dBTff9+XsWPrc+7cMZszFBERkbTE6QWyMeYDY8xWY0y4MWbqA4571xgT\naYy5aIy5dPPfms7MVZyrefPqhIWNoWrVpx2xPXvW4u/vzbZtP9iYmYiIiKQldtxBPgb4AVPicezP\nlmVltSwry81/NyRxbmKzwoVzs3r1UPr0ecuxZ/KVK2f44oumzJvXlYiI+7v0iIiIiCQmpy/Ssyzr\nOwBjzLNAfmdfX5K/dOlcGTToHerUKUfbtmM4duwMAGvXfsbevetp334O+fKVtjlLERGR5N9i+b33\nqmFZ95dbxhxjwoSfYx2TkNfkrPdh4MAWXLhw/8I/T89wBg+e+0i5PUhy38WigjHmFHAWmAkEWJYV\nbXNO4iS1apUjLCyYTp3Gs2TJFgCOHdvGsGGVeeutsdSo0cFxl1lERMQOyb3FckxxvCCW+JtxjknI\na3LW+3Dhggfh4dNjeaZtnGPuzm1wvK6TnBfprQfKWpaVG2gGtAR62JuSOFuOHFlZsOBTPvusEx4e\nbgBERFxj1qyOTJr0JleunLM5QxEREUltku0dZMuyDt7x+Q5jzBCgOzAituOHDJnj+LxWrbLUqlUu\nqVMUJzHG0KlTI6pXL03r1kHs3HkYgP/971sOHgylffvZFCtWw+YsRUREJLnZs2cdZ86s41HvZCfb\nAjkOcf49fcCAls7MQ2xQtmxhfvllJD17TuPLL5cBcO7cEYKCavHKKwNp1Kiv9kwWERERh5Ila5Mj\nR23Onh10M5JMp1gYY1yNMR6AK5DOGONujLmvqjHGNDTG5L75+dNAP0DdQNK4DBnc+eyzzsyf/ynZ\ns2cGwLKi+eGHgQQH1+Xs2SM2ZygiIiIpnR13kPsBA4FbLfxaAYONMdOAnUApy7KOAvWA6caYTMBJ\nYAYwzIZ8JRl67bXnqVTpKdq2HcPGjTsA2LdvA/7+3vj4TKZChTdszlBERNKC5N5i2ZhjsS7IMybu\nJlwJeU3Oeh88PcOJbUFeTDx2d+a2b1/8rqNW05KiRUVFMXz4Qvz85hEdfXuDkxde6MSbb47GzS2j\njdmJiIhIcqJW05ImuLq60rfv26xZM5RChXI54hs3fsmwYc9y7Nh2G7MTERGRlEgFsqQK1aqVYuvW\nYJo1q+aI/fPPToYNe5Z168aTGv5SIiIiIs6hAllSjezZMzN7dg8mTvyADBli9kyOjLzO3LkfMnHi\n61y+fMbmDEVERCQl0BxkSZV27TpC69ZBbN9+0BHLli0/vr4zKVmytm15iYjIo0vu7ZydJSFtlhMi\nIe93YraATsqva3znIKe0fZBF4qVUqYJs2hRInz7f8Pnn/wXg/PljjBlTl0aN+vLyywNxddW3v4hI\nSpDc2zk7S0LaLCdEQt7vx28BHb/rOIumWEiq5eHhxujRHVi0qA85cmQBwLIsfvzRn6Cgmvz770F7\nExQREZFkSQWypHqvvPIcv/46ljp1brcf37//F4YOfYawsHk2ZiYiIiLJkQpkSRPy5fPixx8H4e/v\ng6trzLf9tWsXmDy5BTNmdOD69Ss2ZygiIiLJhQpkSTNcXV3p2bMZ69YN48kn8zjimzZNISCgEkeO\n/G5jdiIiIpJcaJWSpDlVqpQkNHQ0H344kXnzNgJw8uQeRoyowuuvB1K3bleMeegCVxERcZLk3s7Z\nWRLSZjkhEvJ+P24L6Phex1m0zZukWZZlMWPGWj76aBJXrtz+D7hcuZdp02YaWbLkesBoERERSWnU\nalrkIYwxtGlTly1bgqhQoagjvn37Uvz8yrNr1yobsxMRERG7qECWNK9Eifxs2DCCbt1edcQuXjzB\nuHENWLz4U6KiImzMTkRERJxNBbII4O6enhEjfPnhhwHkzu0JxEzBWL58BCNH1uD06b9tzlBERESc\nJd5zkI0xGYFngNzcU1hblrUo8VOLP81BlsR04sQ52rcfy8qVt3e18PDIQsuWE6hSpZWNmYmIJJ3U\n1s65W7f6hIfnuC/u4XGG4OCVsY5JSLtkZ42BhH2NnDUmpUjUVtPGmBeBOcD932lgAa6Plp5I8pU3\nb3Z++GEAY8YsoX//mURERBIefolp01qza9cKWrT4HA+PLHanKSKSqJJz29+ECA/PgWXdX2yGh7eI\nc0xC2iU7awwk7GvkrDGpTXynWIwFlgIFLMtyuedDxbGkOi4uLnz88Wts2DCcYsWecMQ3b/6GoUMr\ncuhQmI3ZiYiISFKKb4FcBPCzLOt4EuYikuxUqlSMLVtG07p1HUfs9Om/CAysxooVo4iOjrYxOxER\nEUkK8S2QNwElkzIRkeQqS5YMTJ36EdOndyNLlgwAREVFsGhRDz7/vBEXLpywOUMRERFJTHEWyMaY\nirc+gInAKGNMB2NMlTufu/m8SKr3zju1CA0dTeXKxR2xnTtX4O/vzY4dy2zMTERERBLTgxbphRGz\nAO/OlX6TYjlOi/QkzXjqqSdYty6AQYPmMGpUzOYtly6d4rPPGvHiix/z6qsBpE/vbnOWIiKPLjm3\n/U0ID48zsS7I8/A4E+eYhLRLdtYYSNjXyFljUps4t3kzxhSO70ksyzqUaBklgLZ5EzusXv0Hvr5j\nOHHinCNWqFBF2refQ548JWzMTERERGLz2K2mLcs6dOsDKAwcuzN2M37s5nMiaU69et78+usYGjWq\n5IgdPvwbAQEV+fnn6cR3j3ERERFJXuK7SG8t4BVL3PPmcyJpUq5cnnz3XT+Cgtrj5hYzY+n69St8\n840vU6e24tq1CzZnKCIiIo8qvgWyIWau8b1yAFcSLx2RlMcYQ5cuTQgJCaREifyO+Natcxg6tAL7\n92+2MTsRERF5VA9sNW2MWXLz05eBVcD1O552BcoCuyzLaphkGcaD5iBLcnHlSjgffzyZadNWOWIu\nLq40bepHgwa9cHGJ7++kIiJyS3Jul+zMtszJuQV0cs7tTonVavrWUk8DnAOu3fHcDSAE+CpBGYqk\nQpkyefDllx9Sr94zvP/+F1y8eJXo6Ci++64Pu3atwtd3Btmy5bM7TRGRFCU5t0t2Zlvm5NwCOjnn\nlhAPLJAty/IFMMYcBEZZlqXpFCLx8NZbNXjuueK0aTOazZv3ALBnzxr8/Mrz7rvTKV/+FZszFBER\nkbjE6++9lmUNVnEs8miKFMnDmjUB9O79JsbE/DXnypUzfPFFE+bN60pExIP3uxQRERF7PKiT3gFj\nzP74fDgzYZGUJF06VwYPbsWKFUPInz+HI7527WcMH16Ff/7ZZWN2IiIiEpsH3UH+HBh/8+NrYnas\n+BuYefPj75ux6UmbokjKV6tWOcLCgmnS5DlH7NixbQQEVCIkZLL2TBYREUlG4pyDbFlW0K3PjTHT\ngRGWZQXceYwxpjdQJsmyE0lFcuTIysKFvfnyy5/o0WMa169HEBFxjZkz/8POnSto3XoSGTNmsztN\nEZFkJzm3S3ZmW+bk3AI6OeeWEA/c5s1xkDEXgYqWZf11T7wY8JtlWVmTKL940TZvktJs336Q1q2D\n2LXriCPm5VWIdu1mU6xYdRszExERSb0eu9X0Pa4AtWOJ1wauxj8tEQEoV64Iv/wyio4db28hfvbs\nYUaPrsnSpX5ER0fZmJ2IiEjaFt87yD0BP2AacKst2PPAu8Agy7JGJFmG8aA7yJKSLV78C507j+fc\nucuOWPHiNfH1nYmXV0EbMxMREUld4nsHOV4FMoAx5i3gI6DUzdAuYKxlWfMTnGUiUYEsKd2RI6d5\n991gQkJ2OmIZM2bHx2cKFSq8bmNmIiIiqUeiF8jJmQpkSQ2ioqIYNmwh/v7ziI6OdsRr1nyP5s2D\ncHPLYGN2IiIiKZ8KZJEUatOmnbRpM5ojR/51xPLlK0P79nPJn7+sjZmJpAwzZgzn1Kn7G/Hkzu2B\nj8+nNmQkd0rI10dfU0ks8S2Q49zm7ebOFUUty/rXGHMJiLOStnsXC5HUpHr10oSFjaFz5/EsXvwL\nAMeP72D48Gdp3nw0NWt2dnTmE5H7nToVzr59g2J5JraYOFtCvj76moqzxVkgA12AS3d8nvJvNYuk\nENmzZ2bu3J5MnbqSjz+ezLVrN4iICGfOnPfZuXMFPj6TyZw5x8NPJCIiIo/sQY1Cvr7j8+lOyUZE\nHIwxtG/fgKpVS9G69Sj+/PMQAH/88R2HDm2lXbtZlChRy+YsRUREUp947YNsjOltjHneGOOa1AmJ\nyN1Kly7Izz+P5IMPXnbEzp8/RnBwHZYsGUBUVKSN2YmIiKQ+8W0U8jKwHjhvjFl+s2CuqoJZxDk8\nPNwIDv4Pixb1IUeOLABYlsWPP/oRFFSLM2cO2ZyhiIhI6vGgOcgOlmXVMMZkAGoAtYgpmAcCEcaY\nTZZlNXzgCUQkUbzyynOEhY3B13cM69ZtB2D//p/x9/emdeuvqFTpTZszFLFf7twexLZ4KyYudkvI\n10dfU3G2R97mzRiTF6hDTJH8NhBhWVbGJMjtUXLSNm+SpkRFRTFy5CIGD55DVNTtPZOrV+/AW2+N\nwd09k43ZiYiIJE/x3eYtvnOQ3zTGfGGM2QX8DXQE/gLqA9kfK1MReWSurq58+umbrFs3jCJFcjvi\nmzZNZtiwyhw9+oeN2YmIiKRs8Z2DPA9oBkwDclmWVceyrEGWZa2zLOv6o1zQGPOBMWarMSbcGDP1\nIcd2M8b8Y4w5Z4yZbIxJ/yjXEkntqlQpydatwbz11guO2IkTuwkMrMyaNeNIDY2AREREnC2+BXIn\nYCUx+yEfN8b8YIz5xBhT0Tx6x4JjgB8w5UEHGWNeAnoSM52jCPAUMPgRryWS6nl6ZmLGjI+ZPLkL\nmTLFzMe7cSOS+fM/4osvmnDp0mmbMxQREUlZEjIHuRhQm5jpFa8Dly3L8nrkCxvjB+S3LKtdHM/P\nAg5YltXv5uO6wCzLsp6I5VjNQRYB9uw5ho9PEL//vt8R8/R8Al/fGTz9dD0bMxMRgIEDW3Dhwv0L\nyzw9wxk8eG6Ku05ybgGtltYSm8duNX0vY4wL8CwxxXFdoPrNp/YkJMF4KAPcWfX+AeQ2xmS3LOtc\nEl1TJEUrWTI/GzeOoF+/GYwduwSACxf+YezY+jRo0IumTYfg6qqZSiJ2uXDBg/Dw6bE80zZFXic5\nt4BWS2t5HPFdpPcjcA7YSMxd4/8BzYHslmVVTaLcMgMX7nh8ATBAliS6nkiq4O6enpEj27FkSX9y\n5fIEYvZMXr58OCNH1uD06f0POYOIiEjaFt87yNuAccBGy7KuJGE+d7oMZL3jcVbAAi7FdvCQIXMc\nn9eqVZZatcolaXIiyV3DhpX49dcxtGs3hlWrYna1OHgwlMDAsrz55mSee+4dmzMUERFJWnv2rGPv\n3nWPPC6+jULsmHizA/AGFt58/AxwMq7pFQMGtHRWXiIpRt682fnvfwcyZswS+vWbQWRkFJcuXWPq\n1Fbs3LmCFi0+w8NDf5QREZHUqWTJ2pQsWdvxeOnS+O33EN9dLBKNMcbVGOMBuALpjDHucbSs/gZo\nb4wpZYzJDvQlZps5EXkELi4ufPzxa2zYMJynnsrriG/e/DUBAZU4dOhXG7MTERFJfuK9SC8R9SOm\nTfWt7TNaAYONMdOAnUApy7KOWpa13BgTCKwFPIi5kzzIhnxFUoXKlYsTGhpM165fMmvWOgBOndpH\nYGBVXnttGPXqdcPFxem/M4ukKZ6e4cS2UC4mnvKuk5xbQKultTyOR97mLTnSNm8ij2bWrHV06TKR\ny5dv/8+ydOmXaNv2a7JmzWNjZiIiIkknUVtNi0jq0qpVbUJDg6lcubgjtnPncvz8yrNjx3IbMxMR\nEbGfCmSRNKpYsSdYty6ATz553RG7dOkUn33WkIULuxMZecPG7EREROwT5xQLY8wlbs8TfiDLsrI+\n/KikoykWIo9n1arfadduLCdO3N4kplChSrRvP4c8eYo/YKSIiEjKEd8pFg8qkN+N78Usy/r6EXJL\ndCqQRR7fqVPn6dBhHMuW/eaIubtnomXLL6hSxQdjHvrzREREJFl77AI5JVGBLJI4oqOj+fzz/9K7\n9zdEREQ64s8++w7vvDOBDBls/WORiIjIY9EiPRF5ZC4uLnTt2pSQkECKF8/niG/dOpuhQytw4MAW\nG7MTERFxjngVyMYYN2PMYGPMXmNMuDEm6s6PpE5SRJyrQoWibNkSRNu29Ryxf//dz8iRNVi2bDjR\n0dE2ZiciIpK04nsH2Q94FwgCooEewHjgDPB+0qQmInbKnDkDkyZ1YebM7mTNmhGA6OhIvvuuN+PG\nNeD8+eM2ZygiIpI04lsgvwV0tizrSyAK+N6yrK7EdMSrn1TJiYj93nqrBlu3BlOlSklHbPfu1fj7\ne7Nt239tzExERCRpxLdAzkNMG2iAy0C2m58vAxokdlIikrw8+WQe1qwZyqefvunYzeLy5X/54osm\nzJv3ERER123OUEREJPHEt0A+DNxasfMX8NLNz6sC1xI7KRFJftKnT8eQIa1YvnwI+fJ5OeJr145j\nxIgqnDix28bsREREEk98C+TFwK3VOmOBwcaYA8B0YHIS5CUiyVTt2uUICxvDK68854gdPfoHAQGV\n2LRpCqlh60gREUnbErQPsjGmClAd2GtZlu2TELUPsojzWZbFxIk/0bPnNK5fj3DEK1Z8k9atJ5Ex\nY7YHjBYREXG+RG0UYoypCfxsWVbkPfF0QDXLsjYkONNEoAJZxD7bth3ExyeIXbuOOGJeXoVp3342\nTz1VzcbMRERE7pbYjULWAl6xxD1vPiciaVT58kX45ZdR/Oc/LzliZ88eIiioJkuX+hEdra3SRUQk\nZYlvgWyA2G415wCuJF46IpISZczozvjx7zF3bk+yZcsEQHR0FD/8MIDg4HqcO3fU5gxFRETi74FT\nLIwxS25++jKwCrhzLydXoCywy7KshkmWYTxoioVI8nH48GnefXc0mzbtcsQyZfLCx2cKzzzzmo2Z\niYhIWpdYUyzO3PwwwLk7Hp8BjgITgdaPl6qIpCaFCuVi5Up/+vdvgYtLzI+YK1fOMnHi68yZ8wE3\nbmhnSBERSd7iu0hvIDDKsqxkOZ1Cd5BFkqeQkB28+24wR47864jly1eW9u3nkD9/WRszExGRtChR\nF+lZljXYsqwrxpjKxpi3jTGZAIwxmW7uZCEicp8aNcoQFjaG11+v6ogdP/4nw4c/y4YNE7VnsoiI\nJEvxKpCNMXmMMVuAUGA2Ma2nAUYDQUmUm4ikAtmzZ2bu3J588cV7eHi4ARAREc7s2e/x5ZfNuHLl\nrM0Zioift/M7AAAgAElEQVSI3C2+u1gEAyeI2bXi6h3xBUCDxE5KRFIXYwwdOrzEL7+MomzZwo74\n778vxt/fm337bN1KXURE5C7xLZDrAX0tyzp3T/xvoFDipiQiqVWZMoXYtCmQ995r7IidO3eU0aPr\n8MMPA4mKinzAaBEREeeIb4GcAbgRSzwXEJ546YhIapchgztjx3Zk4cLeeHllAcCyolm6dAijR9fm\nzJlDNmcoIiJpXXwL5A1A2zseW8YYV6AXsDqxkxKR1K9p0yr8+usYatW6vZvF339vwt/fm19/XWBj\nZiIiktbFt0DuCfzHGLMScCdmYd5OoDrQO4lyE5FULn/+HCxbNpghQ1rh6hrz4+jatQt89dVbzJzZ\nkevXk+XOkiIiksrFd5u3nUB54BdgBeBBzAK9CpZl/Z106YlIaufq6sqnn77J2rUBFC6cyxEPCfmK\nYcMqc/ToHzZmJyIiaVG8GoUkd2oUIpI6nD9/mQ8+mMiCBSGOWLp07jRrNpLatT/EmIfu7S4iIhKn\nRGkUYozJaIwZb4w5Zow5ZYyZbYzJmXhpiojcli1bZmbO/IRJkz4kY0Z3ACIjrzNvXlcmTHiVy5f/\nfcgZREREHt/DplgMJmZx3lJgLlAfmJDEOYlIGmaMoW3bF9myZTTe3k864tu2/YCfX3l2715jY3Yi\nIpIWPKxAfgNob1lWR8uyugIvA6/d3MFCRCTJlCyZn5CQQLp2beKIXbjwD2PHvsh33/UhKirCxuxE\nRCQ1e1iBXBDYeOuBZVmhQCSQLymTEhEBcHdPz6hR7fn++37kyuUJgGVZLFs2jFGjXuD06f02Zygi\nIqnRwwpkV+5vEBIJpEuadERE7teoUWXCwoKpV8/bETtwYAtDhz7D1q1zbMxMRERSowfuYmGMiQZW\nAtfvCDcC1gNXbwUsy2qaVAnGh3axEEkboqOjGT36OwYMmEVkZJQjXrVqW95++zM8PDLbmJ2IiCR3\n8d3F4mEF8rT4XMyyLN9HyC3RqUAWSVu2bt1Lmzaj+fvvE45Y7tzF6dBhLoUKVbQxMxERSc4SpUBO\nKVQgi6Q9Fy9epWvXL5k9e70j5uqantdfH07duv+Hi0t8G4WKiEhakSj7IIuIJFdZs2Zk+vRuTJ36\nEZkzewAQFRXBwoWfMH78y1y8eNLmDEVEJKXSHWSRNOzgwRNMmjSb69fP4u7uRceO71CkSF6703pk\nf/31Dz4+Qfz661+OWNaseWjb9htKl25gY2YiIpKc6A6yiDzQwYMn8PcfRN26G3jjjT+pW3cD/v6D\nOHjwxIMHJkPFij3B+vXD+Pjj1xyxixdPMm7cS3z7bQ8iI+/djEdERCRuKpBF0qhJk2bTosUJMmSI\neZwhA7RoEXNHOSVyc0vP8OFtWbp0IHnyZHPEV64cxciR1Tl5cp+N2YmISEqiAlkkjbp+/ayjOL4l\nQwa4fv2cPQklkvr1K/Drr2N46aXbu1kcOhRGQEBFNm+eYWNmIiKSUqhAFkmj3N29uHbt7ti1a+Du\nnt2ehBJR7tzZ+P77fowc2Y706WP6Gl2/fpnp09swdWprrl27aHOGIiKSnKlAFkmjOnZ8h7lz8zqK\n5GvXYO7cvHTs+I69iSUSFxcXPvqoKSEhIyhePJ8jHho6i6FDK3DgQKiN2YmISHKmXSxE0rDbu1ic\nw909e4rdxeJhLl++Rrduk/n669WOmItLOl59dSj163fXnskiImmEGoWIiNxj7twNfPjhRC5evOqI\nPf30i/j6foOn5xM2ZiYiIs6gbd5ERO7RokVNtm4NpkqVko7Y7t2r8PMrz/btP9qYmYiIJCdOL5CN\nMdmNMYuNMZeNMQeMMS3jOG6gMeaGMeaiMebSzX+LODdbEUltnnwyD2vWDKVXr+YYE3MT4fLlfxk/\n/mXmz/8/IiKu25yhiIjYzY47yF8A4UAuoDUwwRhTKo5j51qWldWyrCw3/z3orCRFJPVKnz4dfn6t\nWbZsMPnyeTnia9aMJTDweU6c2G1jdiIiYjenzkE2xmQEzgGlLcv6+2bsG+CoZVl97jl2IPCUZVlt\n4nFezUEWcZLU0p76ln//vch//vMZS5dudcTc3DLy9tvjqFatneMus4iIpHzJdQ5yCSDyVnF80x9A\nmTiOb2KM+dcYs90Y0znp0xORB0lN7alvyZkzK4sW9WHMmP/g7p4egBs3rjJjRgemTGnJ1avnbc5Q\nRESczdkFcmbgwj2xC0CWWI6dB5QiZipGR2CAMebtpE1PRB4ktbWnvsUYw/vvv0xISCAlSxZwxMPC\n5uHv/wx///2zjdmJiIizpXPy9S4DWe+JZQUu3XugZVl3TgL8xRgzFmhOTOF8nyFD5jg+r1WrLLVq\nlXvsZEXkbqm1PfUt3t5PsmVLEN27T2Hy5BUAnD17iNGjX+Dll4fQsOGnuLi42pyliIjE154969i7\nd90jj3N2gbwXSGeMeeqOaRbewI54jLWAOOeMDBgQ62YYIpKIbrWnvrNITi3tqW/JmNGdL754n3r1\nnuG998Zz/vwVoqKiWbKkH7t3r8LXdwbZsxd4+IlERMR2JUvWpmTJ2o7HS5cOjtc4p06xsCzrKrAI\nGGKMyWiMqQ40BWbce6wxpqkxJtvNz58DugJaiSdio9TenvpOzZpVIyxsDNWr395kZ+/edfj7e/P7\n79/bmJmIiCQ1p3fSM8ZkB6YC9YF/gV6WZc0zxtQAfrQsK+vN42YDDQA34Cgw3rKs8XGcU7tYiDhJ\nWmlPfUtkZBQBAfMJCFhAdHS0I16r1vs0azYKN7cMDxgtIvcaMKAIp04dsjsNScVy5y7MkCEHY31O\nraZFRBLRxo07ePfd0Rw9esYRy5evLB06zCVfvrg24hGRe90sUOxOQ1IxYwwTJ8b+PZZct3kTEUmR\nXnihDGFhY3j11ecdsePH/yQwsCIbNnyp/+GLiKQiKpBFROLJyysL8+f3Yvz49/DwcAMgPPwGs2d3\nZtKk5ly5ctbmDEVEJDGoQBYReQTGGP7zn5f45ZdRlClTyBH/3/8W4e/vzb59G2zMTkREEoOzt3kT\nSXOc1Zp54cL1BAV9QZYsEVy6lJ5PPnmf5s1rJXpuzno9yb2ldZkyhfj555F8+unXTJjwIwDnzh1l\n9Og6NG7cn8aN++Hqqh+xIiIpkRbpiSShW62Zb3Wfu7UtWr9+gxK12Fu4cD3TpgXTrRuO6wQHg69v\ntziL5ITk5qzX46zrJJYlS7bQsePnnD17u+fRU09Vp127WeTIUdjGzESSHy3SS3x16tShXLlyjBs3\nzu5UkgUt0hNJ5pzVmjko6AtHcXzrOt26xcQTMzdnvZ6U1tK6adMqhIUFU7Pm7d0s/v57E0OHPsNv\nv31rY2Yikljatm2Li4sLAQEBd8XXr1+Pi4sLZ8/Gfw1CnTp16Nq160OP8/X1pWnTpg89bvHixQwb\nNize17/XtWvX6NOnD8WLFydDhgzkypWLGjVqMG9erM2LY3Xo0CFcXFz47bffEpxHcqK//4kkIWe1\nZs6SJSLW62TOHJGouTnr9aTEltYFCuRk+fIhjBjxLX5+c4mKiubq1fNMmtScF17oyJtvBuPmltHu\nNEWStY4dh7N3b/h98RIlPJg06VPbzgUxdyUzZMhAYGAgnTp1IkeOHHc9Z4eIiAjSp09PtmzZHus8\nnTp14pdffmHcuHGUKVOGs2fPsmXLlkcq+i3Lsu19SAq6gyyShG61Zr5TUrRmvnQpfazXuXw5faLm\n5qzX46zrJDZXV1f69HmLNWsCKFw4lyO+ceMkAgIqc/ToNhuzE0n+9u4NZ/36Qfd9xFboOvNct9Sp\nU4ciRYowZMiQBx63YcMGnn/+eTJkyEDevHn5+OOPiYyMBGLuCq9fv57x48fj4uKCq6srhw8fjtf1\nfX19adKkCYGBgRQsWJCCBQsCULt27bvuSC9atAhvb28yZsxIjhw5qFOnDqdPn47zvD/88AO9e/em\nUaNGFCpUiGeeeYZOnTrx3nvv3XVcYGAgxYoVI2PGjHh7ezNr1izHc0WLFgWgcuXKuLi4ULduXSCm\ncPbz86NQoUJ4eHhQvnx5lixZctd5hwwZQpEiRfDw8OCJJ56gbdu2jueWL19OzZo18fLyIkeOHDRs\n2JDdu3fH6/16HCqQRZKQs1ozf/LJ+wQHc9d1goNj4omZm7NeT0pvaV216tNs3RpM8+bVHbETJ3Yx\nfPhzrF37ueZfiqRQLi4uDB8+nIkTJ3LgwIFYjzl+/DiNGzemUqVK/P7770ydOpU5c+bQu3dvAMaO\nHUvVqlXx9fXl5MmT/PPPP45CNz7Wr1/P9u3bWb58OatXrwbuvoN98uRJWrZsia+vL7t372bjxo34\n+Pg88Jx58+Zl2bJlXLx4Mc5j+vbty7Rp05gwYQK7du2id+/edO7cmZ9++gmA0NBQLMtixYoVnDhx\ngkWLFgEwZswYgoKCGDlyJH/++Sevv/46b7zxBtu2xdww+PbbbwkKCmLixIn89ddfLF26lOeee85x\n3StXrtCtWzfCwsJYv3492bJlo0mTJo5fOJKKpliIJKEiRWIWlt3Zmrlfv8TfjeHWQrz+/b8gc+YI\nLl9++C4WCcnNWa/HWddJStmyZWbWrO40aFCB//u/r7h69TqRkdeZN68Lu3atoE2bqWTOnNPuNEXk\nETVs2JDq1avTt29fZs++f13E+PHjyZcvH+PHjwegZMmSDB8+nM6dO+Pn50fWrFlxc3MjY8aM5MqV\n677xD5MhQwamTZtGunSxl3DHjx8nMjKSZs2aOQrv0qVLP/CckyZNonXr1uTMmZNy5cpRrVo1Xn31\nVV588UUArl69SnBwMCtXrqR69Zhf/AsXLsyWLVsYP348jRo1crwWLy8vcufO7Th3UFAQPXr04O23\n3wZg8ODBbNiwgVGjRvHNN99w+PBh8uXLR/369XF1daVAgQJUrFjRMf6NN964K9cpU6bg6elJaGgo\n1apVe5S37pGoQBZJYkWK5CUg4OMkv07z5rUeuq3bvRKSm7Nej7Ouk5SMMbRt+yLPP/80rVuPYtu2\ngwBs2/YDfn7etGs3k5Il69ibpIg8ssDAQKpWrUr37t3ve2737t1UrVr1rliNGjW4ceMGf/31F2XL\nln2sa5ctWzbO4hjA29ubevXqUaZMGRo0aMCLL75I8+bNyZkzJ0eOHHEUy8YY+vTpw6effsoLL7zA\n/v372bx5M5s2bWLNmjU0aNCATp06MWHCBHbu3El4eDgNGza861qRkZE8+eSTceZy6dIljh8/fl8h\nW6NGDced5zfffJOxY8dSpEgRXnrpJRo2bEjTpk1xc4tpxrR//3769etHaGgop0+fJjo6GsuyOHz4\ncJIWyJpiISKSxJ5+ugAhIYF06fKKI3bhwnHGjKnHd9/1JSoq7sWUIpL8VK5cmTfeeINevXrd91xc\ni9USaxFbpkyZHvi8i4sLK1asYOXKlXh7ezNlyhSKFy/O9u3byZ8/P3/88Qd//PEHv//+O507d3aM\nc3V1pXr16vTs2ZNly5bh5+fHpEmTOHz4MNHR0QD897//dYz/448/2LFjB8uXL39ozrG97luxAgUK\nsHfvXiZNmoSnpyfdu3enUqVKXLs5x+6VV17hzJkzTJo0idDQUH7//XdcXV25ceNGvN+zhNAdZBER\nJ/DwcCMoqAP16j1Dhw7j+Pffi1iWxbJlAezZs4b27WeTM2fcd2JE0oISJTyAQXHE7TtXbAICAihd\nujTLli27K166dGkWLFhwV2zjxo24u7vz1FNPAeDm5kZUVFSi5BGXKlWqUKVKFfr370+ZMmWYN28e\n/v7+jsV0D1OqVCkALl++TOnSpXF3d+fgwYPUqhX7Xypv3fG983VlyZKFfPnyERISQu3atR3xkJCQ\nu6Z9uLm50ahRIxo1akSvXr3ImzcvmzZtomLFiuzevZsJEyY4rvvbb78l+fxjUIEsIuJUjRtX5tdf\nx+DrO4Y1a2IWqRw4sBl//2do1epLnn22hc0ZitgnIduvOeNcsXnqqafo1KkTY8eOvSv+/vvvM3bs\nWN577z0++ugj/v77b3r37k2XLl3w8IgpzosUKUJoaCiHDh0ic+bMeHl5JdoWaVu2bGHVqlW89NJL\n5MmTh99++42jR49SpkyZOMfUqVOHli1bUrlyZXLkyMGOHTvo27cvTz/9NKVKlcIYQ/fu3enevTvR\n0dHUrFmTy5cvs3nzZlxdXenQoQO5c+cmQ4YMLF++nMKFC+Ph4UHWrFnp0aMHAwcOpFixYlSqVIkZ\nM2YQEhLi2C/566+/JjIykipVqpA5c2bmzp2Lm5sbJUqUIHv27OTMmZOvvvqKAgUKcPToUXr27En6\n9HHv0JRYNMVCJImFhGynceOOvPrqOzRu3JGQkO0PHbNw4XqqVn2bBg3eoGrVt1m4cP1Dxxw8eII+\nfUbzySf96NNnNAcPnkiM9G27Tmr2xBNe/PjjIIYObUO6dK4AhIdfZMqUlnzzTTvCwy/bnKGIxEf/\n/v1Jly7dXcVtvnz5+Omnn/j999+pUKECHTp0oFWrVgwdOtRxTPfu3XFzc6N06dLkzp2bI0eOPFYe\nd17f09OTTZs20aRJE0qUKEGPHj0YMGAALVu2jHN8w4YNmTlzJg0bNqRUqVJ8+OGH1KpVixUrVjjO\n7efnx6BBgwgKCqJs2bI0aNCARYsWOeYgu7q68tlnnzF58mTy58/Pa6+9BkDXrl3p0aMHvXr1oly5\ncnz//fcsWrSIcuXKAZAtWzamTJlCzZo1KVeuHIsXL2bx4sUUKlQIYwzz589n27ZtlCtXji5duuDv\n74+7u/tjvV/xoVbTIkkoJGQ7gYGD6No1ytEyedw4V3r2HESNGuViHeOsttEJkdJaQKcEW7fuxccn\niP37TzpiefKUoH37ORQqVPEBI0VSJrWalqSmVtMiyVxAwGeO4hhiCt6uXaMICPgszjHOahudECmt\nBXRK8OyzJQgNDaZFi5qO2MmTexkx4nlWrQpWISEiYgMVyCJJKH36y7G2TE6f/kqcY5zVNjohUmIL\n6JQga9aMfP11N6ZM+YhMmWLmKEZFRbBw4cd8/vnLXLx4yuYMRUTSFhXIIkkoIiJzrC2TIyLi3qbH\nWW2jEyKltoBOCYwx+PjUITR0NBUrPuWI79jxE/7+5dm5c6WN2YmIpC0qkEWSUJ8+XRg3zvWulsnj\nxrnSp0+XOMc4q210QqT0FtApQfHi+diwYTgff/yaI3bx4knGjWvAt9/2JDIyaff+FBERLdITSXIh\nIdsJCPiM9OmvEBGRiT59usS5QO+WhQvXExQU/7bRELOA7s7WzB07Jk1rZmddR2DFiv/Rvv1YTp48\n74gVLlyZ9u3nkDt3MRszE0k4LdKTpJYYi/RUIIuIJGMnT56nQ4dxLF/+myPm7p6Zli2/4PnnfWzM\nTCRhVCBLUtMuFiIiqVyePNn4/vt+BAb6kj59TG+n69cvM316G6ZN8yE8/JLNGYqIpD4qkEVEkjkX\nFxf+7/9eZePG4RQrls8R37JlJkOHVuDgwa02ZicikvqoQBYRSSEqVixGaGgQbdrUdcROn/6bwMBq\nrFgxkujoaBuzExFJPdLZnYBIYri9cOws7u5eSbZw7PaCu8tERGSO14K7CROWMHnydLJli+b8eRc6\ndGjLe+81feCYVq0C2LEjlBw54MwZKFPmOWbN6vPAMT16fMnKlT85xtSv34iRIzs9cMzQobNYuHAB\nXl5w9iw0b/4mffu2euAYZ73XzrpOSpM5cwYmT+5KvXrefPjhRC5dukZ0dCSLFvVk166VtG37NZ6e\nT9idpkiaVKdOHcqVK8e4cePsTkUekxbpSYrnrPbHCWkbPWHCEv7736n3tY1+5ZV2cRbJrVoFcP58\n6H1jsmWLu0ju0eNLdu786b4xpUvHXSQPHTqLn39ecN+YatXiLpLV0jp52b//BG3ajCY0dK8jliVL\nLtq0mU65co1tzEwkbil1kZ6vry9nzpxhyZIlcR5z/vx50qdPT6ZMce91/yDXrl3Dz8+PBQsWcPTo\nUTJnzkzJkiXp0qULb7/9drzOcejQIZ588knCwsKoWDFttqvXIj0RnNf+OCFtoydPnh5r2+jJk6fH\nOWbHjtBYx+zYERrnmJUrf4p1zMqVP8U5ZuHCBbGOWbhwQZxj1NI6eSlaNC9r1wbQs2czjIn5eX/p\n0mnGj3+Z+fO7ERFx3eYMRR7dwYMH6NWrNR99VIdevVpz8OCBZHGuB4mIiOl0mi1btgQXxwCdOnVi\nwYIFjBs3jj179rBy5Up8fHw4e/ZsvM9hWZbj54EknApkSfGc1f44IW2js2WLjnVMtmxxzxXNkYNY\nx3h5xZ1bQsZ4ecU+JvsDmuKppXXykz59Ovz9ffjpp0E88cTtL96aNWMIDHyeEyf22JidyKM5ePAA\nAwfWp3btWbz++jpq157FwIH1E1TYJua57uXr60uTJk0IDAykYMGCFCxYEIDatWvTtWtXx3GLFi3C\n29ubjBkzkiNHDurUqcPp06fjPO8PP/xA7969adSoEYUKFeKZZ56hU6dOvPfee3cdFxgYSLFixciY\nMSPe3t7MmjXL8VzRokUBqFy5Mi4uLtStG7NmwbIs/Pz8KFSoEB4eHpQvX/6+u+FDhgyhSJEieHh4\n8MQTT9C2bVvHc8uXL6dmzZp4eXmRI0cOGjZsyO7duxP2BqYAKpAlxXNW++OEtI0+f94l1jHnz8f9\nn96ZM8Q65kE3EBIy5uzZ2Mece0ANqpbWyVfdut6EhY2hcePKjtiRI78TEFCRn3+eliL/pC1pz4QJ\n/WnR4u97/nr0NxMm9Lf1XLFZv34927dvZ/ny5axevRrgrju3J0+epGXLlvj6+rJ79242btyIj8+D\n9y7Pmzcvy5Yt4+LFi3Ee07dvX6ZNm8aECRPYtWsXvXv3pnPnzvz0U8xfDENDQ7EsixUrVnDixAkW\nLVoEwJgxYwgKCmLkyJH8+eefvP7667zxxhts27YNgG+//ZagoCAmTpzIX3/9xdKlS3nuuecc171y\n5QrdunUjLCyM9evXky1bNpo0aUJkZGTC3sBkTgWypHjOan+ckLbRHTq0jbVtdIcObeMcU6bMc7GO\nKVPmuTjH1K/fKNYx9es3inNM8+ZvxjqmefM34xyjltbJW65cnixe3Jfg4A64ucWswb5x4yrffNOO\nKVPe4dq1CzZnKPJg4eHHYv3rUXj4cVvPFZsMGTIwbdo0SpcuTZkyZe57/vjx40RGRtKsWTMKFSpE\n6dKladeuHbly5YrznJMmTWLLli3kzJmTSpUq0aVLF1atWuV4/urVqwQHBzN58mTq169P4cKFadGi\nBR06dGD8+PEAjvN7eXmRO3dusmXLBkBQUBA9evTg7bffplixYgwePJgXXniBUaNGAXD48GHy5ctH\n/fr1KVCgABUrVuT99993XPuNN97g9ddfp2jRopQtW5YpU6Zw4MABQkPjnv6XkmkXC0nxihSJWbx1\nZ/vjfv0Sf8eDmIV4gx6pbfSthXi9esV/F4tZs/rQqlUA3buHOnaXeNguFiNHdqJHD+je/SfHmIft\nYtG3byuGDoUePRaQPXvMneOH7WLhrPfaWddJjYwxfPDBK9SoUYbWrYPYs+coAGFhczlwYDPt28+m\naNGqNmcpEjsPj/xcu3b39K9r18DDI1/cg5xwrtiULVuWdOniLqO8vb2pV68eZcqUoUGDBrz44os0\nb96cnDlzcuTIEUqXLg3E/Dfbp08fPv30U1544QX279/P5s2b2bRpE2vWrKFBgwZ06tSJCRMmsHPn\nTsLDw2nYsOFd14qMjOTJJ5+MM5dLly5x/PhxqlWrdle8Ro0ajjvPb775JmPHjqVIkSK89NJLNGzY\nkKZNm+Lm5gbA/v376devH6GhoZw+fZro6Ggsy+Lw4cP3nTc10C4WIiKp1JUr4XTvPoUpU1Y6Yi4u\nrrzyymAaNvwUFxdXG7OTtOpBu1jcmjd8a2pEzF+PnmLw4JUUKRJ3AZjU54K7d7GIa0eL2LZ527Jl\nCytWrGDJkiX89ddfbNiwgTJlynDw4EHHMV5eXo47vfcaOnQoAwYM4MCBA5w4cYLnn3+etWvXOuY9\n35I+fXoKFiwY6y4Wly5dwtPTkzVr1lC7dm3HmP79+7Ns2TK2bo1pNnTjxg1Wr17NqlWr+Pbbb8mS\nJQuhoaFkyJCB0qVLU7BgQXr16kX+/PlJly4dpUqVYvLkybRp0+aR38+kpF0sREQkTpkyeTBhwgfM\nnt0DT8+MAERHR7FkST/Gjq3PuXPHbM5Q5G5FijzJ4MErWbeuFYsX12HdulYJLmgT81yPo0qVKvTv\n35+tW7eSL18+5s2bh4uLC0WLFnV8xFUcA5QqVQqAy5cvU7p0adzd3Tl48OBd44sWLeoomG/d8Y2K\ninKcI0uWLOTLl4+QkJC7zh0SEuK4k31rbKNGjQgKCiI0NJQdO3awadMmzp49y+7du+nTpw9169al\nZMmSXLhwIdXOPwZNsRARSfWaN6/Os88Wp02b0fzyS8yq8z171uLv7827706jfPkmNmcocluRIk8y\nYsTMZHeuR7VlyxZWrVrFSy+9RJ48efjtt984evRorPOVb6lTpw4tW7akcuXK5MiRgx07dtC3b1+e\nfvppSpUqhTGG7t270717d6Kjo6lZsyaXL19m8+bNuLq60qFDB3Lnzk2GDBlYvnw5hQsXxsPDg6xZ\ns9KjRw8GDhxIsWLFqFSpEjNmzCAkJITffvsNgK+//prIyEiqVKlC5syZmTt3Lm5ubpQoUYLs2bOT\nM2dOvvrqKwoUKMDRo0fp2bMn6dOnd9bb6XS6gywikgYULpyb1auH0rfv27i4xPzov3LlDF980ZS5\nc7sQERFuc4Yiyd/D9he+83lPT082bdpEkyZNKFGiBD169GDAgAG0bNkyzvENGzZk5syZNGzYkFKl\nSvHhhx9Sq1YtVqxY4Ti3n58fgwYNIigoiLJly9KgQQMWLVrkmIPs6urKZ599xuTJk8mfPz+vvfYa\nALig8owAAAxTSURBVF27dqVHjx706tWLcuXK8f3337No0SLKlYtZR5MtWzamTJlCzZo1KVeuHIsX\nL2bx4sUUKlQIYwzz589n27ZtlCtXji5duuDv74+7u/tjvZ/JmeYgi4ikMRs2/EnbtsEcPXrGEcuf\nvxzt288lX77SDxgp8vhSaic9STkSYw6yplhImnXw4ImbuyScxd3di44dk88uCQnJbcKEJUyeHP/d\nMiTtqlmzLGFhY+jUaTzff78ZgGPHtjNyZAXeeOMzatT4jzpxiUiapgJZ0qSDB0/g7z/I0c742jXw\n999Lv36DbC+SE5LbhAlL+O9/pzJiBDfHRBMcPBVARbLEyssrC/Pn9+Krr5bTvftUwsNvcO3aDWbN\n6sTOnctp3forMmV6QCtGEZFUTHOQJU2aNGm2owCFWx2WYu7a2i0huU2ePJ1u3bhrTLduMXGRuBhj\n6NixIT//PJLSpQs54v/73yL8/7+9ew/Sqq7jOP7+wMoKEkJ5LwMzHW8hkWmlRJGZaQNN6WjmhdQg\nxHKGJrtomLhpqFwaU1FTy0pJx0zdtIs3UtFycxJ0IDQhTExAXBGBVdZvf5yzcNqeXfbG89tn9/Oa\nYWbP7/zO4XNmHni+z29/z+9XM4Jnn304YTozs3RcIFuv1NCwpuQOSw0NreyzXCYdyTZ48Nslrxk8\n+O1tkNB6moMOGspjj13GxIlbNh949dUXmDnzE9TWXkhjY89dysnMrBQXyNYrVVe/c/M2xk02bIDq\n6iFpAhV0JFt9fZ+S19TX+5+4tU3//tVcccXXuO227zBkyEAAIt6mtvYHzJo1hjVrlidOaGZWPn73\ntF5pwoSTmDt3t81FZbbD0m5MmHBS2mB0LNuZZ45n1iz+55pZs7J2s/YYN+4j1NXNYtSoLWu1Pvfc\nw9TUHMyTT96eMJmZWfl4mTfrtbasFPEq1dVDuukqFm3P5lUsrCs1NjYyffrtXHTRXBobt0zVGTVq\nIscfP5N+/QYkTGeVzMu82bbWFcu8uUA2M7MWzZ+/iFNPncny5as2t+2++wGcccYtvOc9wxMms0o1\ndeowVq78V+oY1oPtsstQpk1bVvKcC2QzM+sS9fXrmDTpKm6/ff7mtqqqao47bgajR5/lNZPNrGK0\ntUAu+xxkSUMk3SFpnaSlklrcc1HSdEmrJa2SNL2cOc3MLDN48EBuvvlbzJkzmf79+wGwaVMDc+ee\nzdVXf551617Zyh3MzCpLii/pXQVsBHYGTgaulrR/806SJgJjgQ8Aw4HPSZpQzqBWeebNW5g6gnUD\nfh10PUmcfvqnefzxGQwfPmxz+4IFd1FTczD/+MdDybK1prvmsvLy68Daq6wFsqQBwBeA8yNiQ0Q8\nCtwFnFKi+6nAjIh4KSJeAmYA48sW1irSvHlPp45g3YBfB9vO/vvvySOPXMrZZ39uc1t9/YvMnj2G\nO+88n8bGtxKm+39LljyUOoJ1A34dWHuVewR5X2BTRPyz0PYUcGCJvgfm57bWz8zMymj77fsxc+aZ\n3HHHeey00yAAIoJ77/0hM2aMZvXqZWkDmpl1UrkL5IHAa83aXgPe0Ya+r+VtZmbWDRx77Iepq5vN\nmDFbVrN4/vnHqKk5mLq6XydMZmbWOWVdxULSCOCRiBhYaJsCjI6Icc361gNHRkRdfjwSeDAidixx\n38pfisPMzMzMtrm2rGJRVY4gBUuAKkl7F6ZZHAw8U6LvM/m5uvx4RAv92vSgZmZmZmZtUdYpFhGx\nHvgNME3SAEmHk61U8YsS3W8CpkjaQ9IewBTgxvKlNTMzM7PeKMUyb5OBAcBK4FfA1yJikaQjJK1t\n6hQR1wB3AwuBBcDdEXFdgrxmZmZm1ov0iJ30zMzMzMy6SooRZDMzMzOzbquiC+T2bFttPZekyZKe\nkLRR0g2p81gakvpJ+qmkZZJek/Q3SUenzmVpSPqFpBX5a2GxpDNSZ7J0JO0jaYOkm1JnsTQkPZS/\nBtZKel3Sotb6V3SBTBu3rbYe70XgIuD61EEsqSpgOTAqXw5yKnCrpPemjWWJXAwMzV8LY4EaSR9M\nnMnS+Qnw19QhLKkAzoqIQRHxjohotV6s2AK5ndtWWw8WEb+NiLuANamzWDoRsT4ipkXEC/nx74Cl\nwIfSJrMUImJRRDTtey2yN8e9E0ayRCSdCLwK3J86iyXX5mWBK7ZApn3bVptZLyNpV2AfWlg/3Xo+\nSVdKegNYBKwA7kkcycpM0iDgQuCbtKM4sh7rEkkrJT0saXRrHSu5QG7PttVm1otIqgJ+CfwsIpak\nzmNpRMRksveKI8jW4G9Im8gSmAZcFxEvpg5iyZ0LvA94N3AdcLekvVrqXMkF8jpgULO2QcDrCbKY\nWTchSWTFcQPw9cRxLLHIzAf2BCalzmPlI2kEcCQwO3UWSy8inoiINyLirYi4CXgUOKal/uXearor\ntWfbajPrPa4HdgKOiYjG1GGs26jCc5B7m9HAUGB5/sF5INBX0gERcUjaaNYNBK1Mu6nYEeR2bltt\nPZikvpK2B/qSfWiqltQ3dS4rP0lzgP2AsRHxZuo8loaknSWdIGkHSX0kfQY4EX9Jq7e5huxD0Qiy\nAbQ5QC1wVMpQVn6SdpR0VFN9IOnLwCjgDy1dU7EFcq7kttVpI1kC5wPrgW8DX85/Pi9pIiu7fDm3\nCWRvhi/n61yu9frovVKQTad4gWx1m0uBcyKiNmkqK6uI2BgRK5v+kE3N3BgRXvGo99kOqCGrF1eR\n1Y/jIuLZli7wVtNmZmZmZgWVPoJsZmZmZtalXCCbmZmZmRW4QDYzMzMzK3CBbGZmZmZW4ALZzMzM\nzKzABbKZmZmZWYELZDMzMzOzAhfIZmYVRNJpkl7fSp+lkqaUK1NrJA2V9LakkamzmJm1lQtkM7N2\nknRjXvQ1SnpT0j8lXSZpQDvvcVcHI3TLHZ5aeaZumdfMrCVVqQOYmVWoPwEnA/2AUcD1wACyLUzt\nfyl1ADOz9vAIsplZxzRExKqIeDEi5gK/Aj7fdFLSAZJqJa2V9LKkmyXtmp+7ADgNOLYwEv3x/Nwl\nkhZLWp9PlZguqV9ngkoaJOnaPMdaSQ9K+lDh/GmSXpc0RtJCSeskPSBpaLP7fFfSf/J7/EzSVElL\nt/ZMuWGS/ijpDUnPSDqyM89kZrYtuUA2M+saG4HtACTtDswDFgCHAJ8CdgCaph9cDtwK3AfsCuwO\nzM/PrQPGA/sBk4ATgPM6me0eYDfgGGAE8Gfg/qaCPVcNfCf/uz8CDAbmNJ2UdCIwFfguMBJYDExh\ny/SJ1p4JoAaYDQwHngBuac+UFDOzcvIUCzOzTpJ0KPAlsmkXkBW2f4+I7xX6jAdekXRIRNRJ2gAM\niIhVxXtFxA8Lh8slXQJ8E7igg9nGkBWlO0dEQ958gaSxwClkhS1AX+CsiHguv+5y4IbCrb4B3BAR\nN+bHP5L0SWCfPPcbpZ5J2jy7YmZE3JO3fQ84laxYLxbRZmbdggtkM7OO+Wy+mkRV/ue3ZEUkZCOs\no0usNhHA3kBdSzeVdBxwDvB+YCBZ4dqZ3/aNJBu9Xl0oViEbMd67cNzQVBznVgDbSRocEfVkI9rX\nNrv3X8gL5DZY2PRDRKzIs+zSxmvNzMrKBbKZWcfMA74KbAJWRERj4VwfoJZs5Lf5F9RebumGkg4D\nbiEbLf4DUA+MAy7rRM4+wH+AI0pkWVv4eVOzc01TJ/qUaOuIt1rIZmbW7bhANjPrmPURsbSFc08C\nxwPLmxXORW+SjQ4XHQ78OyIubmqQNKyTOZ8kmxMcreRti8XAocDPC22HNetT6pnMzCqOP72bmXW9\nK4EdgVslHSppL0lHSrpG0g55n2XAQZL2lfQuSVXAEuDdkk7Kr5kEnNiZIBFxH/AocKekoyUNk/RR\nST+QdPhWLi+OOP8YGC/pK5LeL+lcsoK5OKpc6pnMzCqOC2Qzsy4WES+RjQY3AvcCTwNXkK100fRF\nueuARWTzkVcCH4uIWrLpFLOAp8hWv/h+RyI0Oz4GeIBsDvFiYC6wL9k84zbdJyJ+DVwEXEI2Kn0A\n2SoXGwv9/++ZWsjTUpuZWbegCP8fZWZm7SfpN0DfiBiXOouZWVfyr7/MzGyrJPUnW77u92Qj418E\nxgJfSJnLzGxb8AiymZltlaTtgbvJ1i7uDzwLTM93ETQz61FcIJuZmZmZFfhLemZmZmZmBS6QzczM\nzMwKXCCbmZmZmRW4QDYzMzMzK3CBbGZmZmZW8F8oWao9tA0FUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6c0183bf98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]\n",
    "b = -per_clf.intercept_ / per_clf.coef_[0][1]\n",
    "\n",
    "axes = [0, 5, 0, 2]\n",
    "\n",
    "x0, x1 = np.meshgrid(\n",
    "        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),\n",
    "        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),\n",
    "    )\n",
    "X_new = np.c_[x0.ravel(), x1.ravel()]\n",
    "y_predict = per_clf.predict(X_new)\n",
    "zz = y_predict.reshape(x0.shape)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.plot(X[y==0, 0], X[y==0, 1], \"bs\", label=\"Not Iris-Setosa\")\n",
    "plt.plot(X[y==1, 0], X[y==1, 1], \"yo\", label=\"Iris-Setosa\")\n",
    "\n",
    "plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], \"k-\", linewidth=3)\n",
    "from matplotlib.colors import ListedColormap\n",
    "custom_cmap = ListedColormap(['#9898ff', '#fafab0'])\n",
    "\n",
    "plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)\n",
    "plt.xlabel(\"Petal length\", fontsize=14)\n",
    "plt.ylabel(\"Petal width\", fontsize=14)\n",
    "plt.legend(loc=\"lower right\", fontsize=14)\n",
    "plt.axis(axes)\n",
    "\n",
    "save_fig(\"perceptron_iris_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure activation_functions_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAAEYCAYAAADMNRC5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd0FNXbwPHvzaaRBoQeEAEB6U2KFCF0BRHBgiIKYkOx\nIPb2w/basGLBCijFggIiWEAhAtIEpCMdpENIIJCe3fv+cXdDSDYhZWvyfM7Zszuzd2buPpns7J3b\nlNYaIYQQQgghhCiMAG9nQAghhBBCCOE/pAAhhBBCCCGEKDQpQAghhBBCCCEKTQoQQgghhBBCiEKT\nAoQQQgghhBCi0KQAIYQQQgghhCg0KUAIn6aU2quUGuuB44xTSm30wHGUUuoTpVS8UsqqlOrq7mNe\nID+TlVJzvZkHIYTwJ0qp4UqpJA8dy6aUGuyJYwlRFErmgRCuopRqBawBVmitryjituOA67XWzXOt\nrwQka63TXJTHi4G9QFut9boc68OAEK11oiuOU8Dx+wM/AN3s+UjQWme585j243YDFgOVtdYJOdZH\nYr4HPHIxFEIId1NKTQaGAxqwAonAFuB74NOSfucqpUKASK11fEnzmmOfk4FKWutrcq2vCiRqrTNd\ndSwhXEFqIIQr3QV8CDRTSl1ajO3zlGa11iddVXiwU/kcJ8XdhQe7BsARrfUqrfVxTxQe7ByfW+Vc\nqbU+I4UHIUQptBCoDlwM9AbmAi8AS5VS5Yq7U6VUoNY63ZWFh4LYrxNSeBA+RwoQwiWUUqHAUOAz\nzF2eO52kqaGUmm5vvpOslFqnlOqmlBoOjAOa2qtrrUqp2+zbZDdhUkrNUEp9n2ufSin1n1LqIfty\nX6XUEqVUglLqpFLqV6VUoxyb7LE/r7Efa5F9u+eVUpty7fc5+77TlFIblVLX5Hj/YkfVslJqgf3z\nbFFK9SogRpOBt4Ha9m332NfHKaUm5E6bs2mRUmqxUupDpdT/KaVOKKWOKaXG59omSCn1ilJqnz3P\nu5RS99trXRbZk52wx3eSfZspuY4TrJR6Vyl1VCmVqpRaoZTqnOP9bva891BKrbR/7r+VUq1zpIlS\nSk215zHVno8H84uLEEK4QbrW+oTW+ojWeqPW+l0gFmgDPA7Z35mvK6UOKKXOKqVWKaX6OHaQ4/vu\nKvt7aUAfexOmM/Y0DexpmuY8uFLqbvt3tUUpFaCU+lwptUcplaKU2qGUeixH2nGYGpP+Oa6BXe3v\nZTdhUkotd/K9H2nf58BCfqZApdQEpdQh+3Viv1LqFRfGXZQRUoAQrnIDsE9rvRmYBtymlLI43lSm\nidASoDYwEGgGvGh/+xvgLWA7UA2oAXzr5BjTgH5Kqagc62Ixd5lm2JfDgXeAtphmQqeAn5RSgfb3\n22Puwvexb+doW6o5v2ZiDPAI8Jg9r7OBWUqpFrny9DLwLtAC+Bv42v5ZnXnQ/pkP2j9nuxzHLoyh\nQCbQERgNjFFKDcnx/lfAMHveGwF3YD7/f8B19jSNMfF9KJ9jj8f8LUcArYBNwK9KqWq50r2CuQi3\nBk5i/jYO/wc0BfoBlwIjgUOF/IxCCOEWWustwK+c+z6cAlwB3IT5nv8SmKuUap5r09eAZzDfq6sc\nu7Pvcyfmu/+WXNsMBb7WWlsxv7UOAtfb9/E08JRS6nZ72jeB74DfOXcNXO7kI0yz5zWn64EUYH4h\nP9NDmGvwjUB9YAjm2itE0Wit5SGPEj+AOODhHMt7gEE5lu8CTgMV89l+HLDRyfq9wFj7awtwDLg9\nx/ufAb8UkK9wIAvoZF++GLABbQo6PubL/plcaRYDX+Xaz5053o+xr+tUQH4eAfY42e+EXOsmA3Nz\npfkrV5oFmPa8YJpG2YDe+Ry3G6YtcHR+xwHCgHTglhzvBwC7gBdz7McG9MqRppN93zH25R+BL7x9\nTspDHvIom4/c35+53nsVOAvUs39v1cr1/mzgA/trx/fdtbnSDAeSciw/COzNsVzLvu/2BeTxVWDB\nhfJsP/5g++to+3d09xzvLwQm2l9fUojP9B6w0Nt/I3n4/0NqIESJKaXqA52Br3OsnoEpNDi0wvxA\nL3Y/A23u5HyL/U6PUioYcydpao681FOmqdMupdRp4CimxqF2YY+jTMfiGPLeAVoGNMm1LrvZk9b6\nsP1l1cIeq4hyjxJ1OMexWmMuHHEl2P8lQCA5PrfW2gas4PzPrcnxue35UDnyMhEYopRar5Qar7w8\n0pQQQuTg6A/Wxv56q1LqjOOBqTm9JEd6Day9wD6/BmoqpbrYl28BdmutV2cfVKlR9uaex+3HeZgi\nXJcAtBkA4zfOXQNrAN05dw1sXYjPNAVobW9G9YFSqp9S6ry+cUIURuCFkwhxQXdi7lQfyP09pJSq\nqbU+RK7OuyUwDfjL/sXZEQgC5uR4fx5wALgb02wmC9gGBBfjWM6aFuVe56xzW1EL5jbyxifISbrc\nx9LFOFZBHHko6ud2vBcAoLX+VSlVG7gK6AnMV0p9p7W+w4V5FUKI4miCqSEPwHz3tsVcJ3JKzbWc\nXNAOtdYnlFK/Y37YL8M0X8pu1mlvavoOMBZzQyYJuB+4thj5nwZ8opS6D7gZ+E9r7bjpc8HPpLX+\nx94v7kqgB6aJ03pMR3MhCk1qIESJ2Ps53AY8CbTM9dgIONp4rgNaKKWi89lVBqaJUoHsd3R2Y76g\nhwJztNYp9rxEY9qXvqK1XqS13g6U5/yCcob9Od9jaa3PYO6qd8n1Vhdg64XyWAwnMG1ec2pZxH2s\nw/w/d8/n/Qt+bkxTpUxyfG6lVACmoLalKJnRWidoradrrUdi+mIMV0o5KxQJIYRHKKWaYX44zwT+\nwXxn1tBa78n1OFKM3U8DblBKtQGaA9NzvNcZWKm1nqi1Xq+13oPpf5BToa6BmCaiAAMw18Ccx/kH\ncyOowM+ktU7WWv+gtR4N9Ad62lsSCFFoUgMhSupqoBLwee7mSUqpb4F7MR2NZwBPAHOUUk9j+hg0\nx7Qj/RPYB1xsH83nP+CM1joD52Zgaj0uBgblWJ8IxAN3KaUOYtqhvsH5d8uPY+7E9FVK7QfStPNh\nTMcDLyildmGqr2/F/LBuc8GIFN0i4B2l1ABMZ7Z7gIsw/T8KRWu9Syk1E/hcKTUGU6CoBdTRWk8D\n9mNqCvorpeYBqVrr5Fz7SFFKTQReU0qdtB9/LKZp0sQcSQusTVJKvWA//hZMTcp1mOp8GYpQCOEp\nIfbBHwKAKkAv4ClMh+e3tNapSqnpwBSl1KOY76xozMAcu7XWjprtwtaezwY+Br4AVmmtd+d4bwfm\nJsqVmBs1NwNdgYQcafYBVyqlGmIGpjhlb7Z7Hq11ulJqNvAsZvCOW3K8t1MpNaOgz6SUehg4gql1\nyLJvfxpzTRai0KQGQpTUSGBRPn0bZmKGLO1lryXohmlWNBfYDDzPueYvPwA/A39gfuQ7Rppw1pxm\nGmZ0n1OYDmQmodYaM7JEC0wb/fcxX7LpOdJYgQcwBZBDnN/8KacJmELE6/Z9DcR0ZMvZ9r8wTX0K\nY5L98QWm+vsMMKsY+70VU7h6D9NsazIQBdn9M8ZhRkg6iomNM09gRgOZhLmb1Qzoq7U+doG85FyX\njik0rgeWYjqyX+NkGyGEcJdemJrk/ZjRja7GfAd201o7miiNwHxPvo75zvwJM4LR/hz7KdR3un2f\nszHXn6m53v4E8706HViN6fvwZq40n9nzsAZzDXQMn+3s+FPtx1lrr2nP6UKf6QxmdMFV9mO1AK7U\nrp1vSZQBMhO1EEIIIYQQotCkBkIIIYQQQghRaFKAEEIIIYQQQhSaFCCEEEIIIYQQhebzozAppaST\nhhBCeIDW2m8mlJJrgxBCeIaza4Nf1EB4e7pux2P48OFez4OvPSQm/heX+PnxLGYxqxqtwmazSUy8\n+PClmPgjb8fM1/6GvvSQuEhMJC6lIyb58YsChBDCdU7OPwlAtWHVyD1zuBBCCCHEhfh8EyZfUqdO\nHW9nwedITJzz5bg0eL8BVYdUpVz9ch49ri/HxFskJv5P/obOSVzykpg4J3HJyx9iIgWIIoiNjfV2\nFnyOxMQ5X46LClBU6FrB48f15Zh4i8TE/8nf0DmJS14SE+ckLnn5Q0ykCZMQQgghhBCi0KQAIYQQ\nQgghhCg0VVAPa1+glNK+nkch/MHpFacJLB9IeJNwb2dF+CClFNrPhnGVa4MQQrhXftcGqYEQoow4\n+89ZNvTZkD0KkxBCCCFEcUgBogji4uK8nQWfIzFxzhfjUvO+mnTc35GKfSt65fi+GBNvk5j4P/kb\nOidxyUti4pzEJS9/iImMwiREGaIsCoXftFIRQgghhA+SPhBCCCGkD4QQQog8pA+EEEIIIYQQosSk\nAFEE/tAmzdMkJs75Ulz2v7afvc/tJe1gmlfz4Usx8RUSE/8nf0PnJC55SUyck7jk5Q8xkT4QQpRi\ntkwbB98+SOaJTCpfW5nQWqHezpIQQggh/JxL+0AopUYDI4DmwAyt9cgC0j4MPA6EAj8A92qtM52k\nk3auQhTTyV9OsqnfJspdWo7229qjlN80cRceJn0ghBBC5JbftcHVBYhrARvQFyiXXwFCKdUXmAJ0\nB44Ac4AVWuunnaSVi4QQxWRNs5IwPwGtNVWvr+rt7AgfJgUIIYQQuXmkE7XWeo7Wei6QcIGktwFf\naK3/1VqfBl4CbndlXtzBH9qkeZrExDlfiYsl1EKV66r4ROHBV2LiSyQm/k/+hs5JXPKSmDgnccnL\nH2LirT4QTTG1Dg4bgKpKqYpa60Qv5UkI1zl2DHbs8HYufMuBA6U6JlqD1QpWm8JmM68dN8g16txr\nfW590rZDnCy364LpznttT1PYdM7yKYQQQpSEW+aBUEq9BNQsoAnTLuA+rfUC+3IgkAHU0Vr/lyut\nVFML//LBB/DAA97ORammgSSiSCCaJKI4SwTJhOf7nE4I6YSQQTAZBOf72rGcRSBWLOc9bAQUuE77\n/aB20oRJCCHE+fJrwuStGoizQFSO5SjMb4IzzhKPGDGCOnXqAFChQgVatWpFbGwscK6aR5Zl2WeW\nf/uNWIAqVYgLDjbvh4WZ91NSPLLcJaQSVh3MiowjXjl+cZaztIXZZwM5aa1I9cBmHM6qyvLU08Rb\nK2BR3Uiwledg5jaSdATJtn5YCQRM/DERL8Fy5xJub5YVfxCAjSB1BQHYsOk/AbAEXAForCwhQCuC\nVBcUmky9FAUEq86kBtrIsi0DBQGWTmjAaltKVFYAwaozCk2GXgZAaEAnDpfLAutyc/jAy0FpyFpB\ntVQL5QIuByDNtgKlNOUCOrInIhMyV5r0QR0gcxWk/0BEVgBBqhaJNoQQQohC8VYNxHRgj9b6Ofty\nD2Ca1jrGSVqfucsUFxeX/UNRGBITJ+66i7jPPyf2s8/gzju9koWE3xPYesNWaj5Qk7ov1vVKHnKL\ni4uja9dY9u6Fbdtg585zjx074OBBsBXhR2xkJFSsCOXLQ0QEhIefe879ulw5CA42j5CQC78ODASL\n5fyHCrCRlH6KxIx4EtJOkJB2gtMZCdzZdiQWCwQEgGOQqyxbFnXfq0tiaiLJmcnZeVYorP+zZo+G\n5fj/sWkblhctTj+n9X9WAtT5tRtaawJeDCAwIJDQwNDzHtvv3+40/ZDvhxBsCSbIEkRwQHD267f6\nvIVSSjpRF5N8BzoncclLYuKcxCUvX4qJR2oglFIWIAiwAIFKqRAgS2ttzZX0K2CyUmoGcBR4Bpjs\nyrwI4TVW++ke4L0mLdG9oul4uCOZ8XlGRvYYrWH7dlizBtatg0WLYO9eSEpynl4pqFYNYmKgZs3z\nn6tVg0qVIDraPCpWhKAg1+TTarNyLPkYB5MOsvfMYQZeOjDPcLc2bSPs/8JIt6bn2X5k+6EEW86f\nXyMwIJCTKSdJzUolQAVQIbQCFUIrUD6kPBnWDEICQ85LH6ACGN97PKGBoYQHhRMRHEFEcAThweFO\n86yUIvO5TAIDCvcVrpTiuxu+K1RaIYQQ4kJcPYzrOGAcpjmSwwuYwsFWoLHW+qA97RjgScw8EN8j\n80CI0mLECPjyS5g82bwuIxwFht9/h7g4WLIETpzIm65GDWjaFBo2hAYNzKNhQ7j4YnP331N6fNmD\nXQm7OHzmMNYc9zgSHk+gYrmKedJXf7M6aVlpVA2vSpXwKlQNr0rVsKqM7zOeqJCoPOkPJh0kKiSK\nyOBIv5h/Q2oghBBC5OaRGgit9QuYAoMzkbnSvgu868rjC+ETHO1wLM6bpJQmVqspKMydC/Pmwa5d\n579fowZcfjm0aWMerVubde6QmJrIlhNb2HpiK//G/8uuhF3sTtzNotsWUS2iWp70/53+jwNJBwCo\nGl6VmpE1qRVVi9SsVCqStwBx4OEDBFkKX+1RK6pW8T+MEEII4cO81YnaL/lSmzRfITFxwmolDoj1\nYhMmd9u4EaZOhRkz4PDhc+ujo+HKK6F7d+jWDerXP9cvIC4ujho1Yt2Wp45fdGT7ye151u9K2OW0\nADFryCwigyOJiYzJ06TImaIUHgpL/n/8n/wNnZO45CUxcU7ikpc/xEQKEEK4mqMGwgsFiLObz3Jq\n8SmqDqlKcFXXtgfKyIDvvoP33jP9Ghzq1YMbboCrr4aOHV1b8ZKamcqGYxtYe3gt646sY+2RtXx8\n9cdcXuvyPGnb12xPRHAETas2pXHlxtSPrk/96Po0qtzI6b5bVGvhuowKIYQQZYhbRmFyJWnnKvzO\nkCHml/a338KNN3r00DvH7OTQe4eo+VBNGrzbwCX7PH0a3n8fPvwQjh416ypWhJtugmHDTKHBHU38\nH/j5ASaumXhe/wSACVdO4IEOMs+Gq0kfCCGEELn52jwQQpReXhqFyZZl4/jXxwGoNixvk52iSkkx\nc+K99hok2ueHb9YMxoyBoUPN0Kglte/UPqw2K5dEX5LnvWoR1dBomlVtxmU1LjOPmMtoWa1lyQ8s\nhBBCiGIrvY203cAxaZg4R2LihM1mphfzdBMmK9R9sS5Vb6lK5GWRF06fD5sNJk0y/ReeeMIUHrp2\nNaMrbdwId9xRvMJDXFwcZzPOMm/HPB74+QEu/eBS6r5Xl7dWvOU0/f3t7yfpySQ23buJKddO4YEO\nD9Dpok75Dm3qj8rC/49SarRS6m+lVJpSatIF0j6slDqilEpUSn2ulHJ9xxMXKwt/w+KQuOQlMXFO\n4pKXP8REaiCEcDUv9YEICAkg5p4YYu7JMx9joW3dCqNGwdKlZvmyy+CVV6B375I3U1p9aDV9Xu9D\npu3caM3lQ8pTLtB5aaRCaIWSHVD4ikPAS0BfIN+ip1KqL/A40B04AszBjOr3tAfyKIQQogikD4QQ\nrnbNNfDTT2Zs0wEDvJ2bQsnIgJdfNs2VMjOhalV4+23TVMlV/RuOnj1Krbdr0TamLX0v6Uvf+n1p\nX7N9oSdDE+7l7j4QSqmXgJpa65H5vD8d2Ku1fta+3AOYrrV2OvCvXBuEEML9pA+EEJ7iAzNRF8W+\nfabf9+rVZvnuu01BomLeqRDyZdM2VhxYwYxNM1i8bzEbRm3IM+xp9YjqnHz8JOVDy7su86I0aYqp\ndXDYAFRVSlXUWid6KU9ClFhmJuzdaybbjImByBwtTDNOZJBxJIOIFhHnbZNxIoPkTckABFUOcvq+\nbCfbuXu7gvjHLxwf4Q9t0jxNYuKEF/pA2DJsxdpuzhwzudvq1VC7tpkU7pNPCl942HRsE0/9/hT1\n3qtHl8ld+GjNR2yL38ZfB/7KkzYuLk4KD7nI/895IoDTOZZPA4pck5D6GvkbOidxMQWHjz4y/cku\nvRQaNYqjRg145BE4bsa7IGlFEnuf3Ztn26QVSWzouYENPTfk+35p2e67Ud/5RT49ud2XPb/0iXwW\nRGoghHA1D89EnXUmi1X1VhHdL5pGUxqhCtHmyGaDJ5+E8ePN8jXXwOTJZiK4onhkwSMs3LMQMDMv\n39zsZm5qdhOtq7cu6scQ4iwQlWM5CtDAmfw2GDFiBHXq1AGgQoUKtGrVKnvyJccPWHcvO3jqeP6y\nvH79ep/Kj6eXf/stjnHjYNUqs1yx4mJuSJzFgGQLA97uwvTpf/LWW3B13daENw/Ps/3K/1ZytNVR\n2lZsm+/7SVFJNKf5ecdvXbk1FbpXYE3iGg5HHXb6vq8dLzQmtFR/vuIc79B/h9gUtckrny8uLo4p\nU6YAZH+/OiN9IIRwtV694I8/YOFC89oDMk5kkLQiicrXVL5g2rQ0GD7cTFURGGgKEQ89VLy+DjO3\nzOSPvX8wtPlQutTuQoCSSk1/5SN9IPZorZ+zL/cApmmtnY4KINcG4avS0qB/f1i0CCpVgo8/hkED\nNUuD/4QAePyyWP7+G2rUgD//hAaumbJHCLeQPhBCeIoXRmEKrhJcqMJDYiJce61pqhQVBbNmQc+e\nBaRPTWTSP5M4m3GWcbHj8rx/Q9MbuKHpDSXJuijllFIWIAiwAIFKqRAgS+tcMwTCV8BkpdQM4Cjw\nDDDZo5kVwgWeesoUHqpXN8NfN20KtgxT2FUWxR9/mFrfuDgz1+jKlRAS4t08C1FUcruwCHJXWQuJ\niVOOPhAeasJUWEeOQJcupvAQE2OGas2v8LD1xFbunXcvtd6pxaMLH+XNFW9yNuNsiY4v50peZSQm\nzwIpwBPALfbXzyilLlJKnVFK1QLQWv8GvAEsBvbaH897JcdFUEb+hkVWVuPy22/w7rumdnfuXFN4\nAFBBCut8K52OdyIy0rxXrx6sXw/PPuvdPHtbWT1XCuIPMZEaCCFczUvzQBTkxAnTmmrrVmjSBH75\nxXSazi3TmsmArwfw2+7fstf1rtebBzs8mO98DUIURGv9AmY+B2cic6V9F3jX7ZkSwg2Sk2GkvYHe\niy9Cu3bn3lNKYQmzEFTBjE4XGQnTp5ubOm++CdddB5df7oVMC1FM0gdCCFfr3BmWL4dly8xrN9Fa\nE/9jPNF9orGE5V/bkZgIPXqYO11Nmpg2t5ULaO1048wbmb9zPsNbDuf+9vfTpEoTN+Re+Bp394Fw\nNbk2CF/z0kvwv/9B27amWVJhKqGfesoMm+2oHXbVvDtCuEp+1wYpQAjhah07mqvH8uXmtZucWXuG\ntW3XEnpJKB12dnA6+lJSkplFevVq01FvyRLTLrcgB5MOEh4UTsVyRZgIQvg9KUAIUXzHjpnhWs+e\nNX0bunUr3HZJSXDJJRAfb4bVHjjQrdkUosjyuzb4ThsLP+APbdI8TWLihIf6QBybdgyASv0rOS08\nZGaaavHVq6FOHTMwVPXqkGHN4OM1H/Pk70863W+tqFpuKTzIuZKXxMT/yd/QubIWl5dfNoWHAQPy\nLzw4i0lUlKm1ADO0tjX30AJlQFk7VwrDH2IiBQghXM1DM1GXq1+OsEZhVLulmtP3x441I4BUq2Yv\nPMRk8sW6L2j4fkPunX8v45ePZ1fCLrfmUQghSrv4ePj8c/P6lVecp8k4kcHGqzay8pKVed675x64\n+GL491/46Sc3ZlQIF5ImTEK4Wps28M8/sHatee1Gjv+N3DUQn35qLkrBwaY6fV/E1zy3+Dl2J+4G\noHHlxrwQ+wLXNblO5m4QgDRhEqK4HH0f+vWD+fOdp9E2jTXZCjYILJ93/Jr33oMxY0xfiKVL3Zxh\nIYpAmjAJ4SkenIlaKZWn8LBkCYwebV5/8onphrFk/xJ2J+6mQXQDpg+ezqZ7N3FD0xuk8CCEECWQ\nlgYffGBeP/JI/ulUgCIwMtBp4QHM6E3ly5uxN1avdkNGhXAx+fVQBP7QJs3TJCZOWK2mD4QXhnE9\neND0e8jKMk2YRoww65+PfZ5J10xi6+itDG0+FEuA5+eokHMlL4mJ/5O/oXNlJS4zZsDx49C6NXTv\nXnDagmISGQmjRpnX77zjuvz5g7JyrhSFP8REChBCuJqX5oGwWuGWW7OIj4c+feCNN869Vy2iGre3\nvp3AAJn6RQghXOWzz8zzgw+WfAjW0aPNZWPWLNOvQghfJn0ghHC1xo1Nb7itW81rF/v3jn+xRFio\n/URtQmJCADMB3OCHVjLvwyuoWDmTbZuDqOa8b7UQTkkfCCGKZssWaNbM1B4cOQLh4SXfZ79+ZqLP\nd94xfSKE8DbpAyGEp7i5D0TtJ2sTFB2Espj/58V7F9PomVuYN7EDAD3GTJXCgxBCuNkXX5jnoUMv\nXHhI2ZXC0silrGmzpsB0d955bt9SPha+TAoQReAPbdI8TWLihJv7QIQ1CKPOuDokhicy5Psh9Pj8\navZ8/iLYgrl62F6+f2akW45bUnKu5CUx8X/yN3SutMclPR2++sq8vuOOC6fXmZq1Z9diTSl4ooer\nr4YqVWDz5rLTmbq0nyvF4Q8xkQKEEK7moT4QQZYg/tjzB5Y/3oaTjWjcxMZ3n9Z16zGFEEKYZkYn\nT0Lz5tC2bSE2sF8WHDXH+QkOhttuM68dBRQhfJH0gRDC1erUgf37Yd8+MzuQG034YTVjbmiHxaL4\n+29o1cqthxOlmPSBEKLwhgyB776D11+Hxx+/cHpt1dm1D4GRBQ9msX69GdWpShU4dAiCglyRYyGK\nR/pACOEpbpqJOm1/Gtp67gdTVhZMeqk9WiseeUQKD0II4QlnzpybMfqmmwq3jbLY54G4QOEBoGVL\naNQITpyAP/4oQUaFcCMpQBSBP7RJ8zSJiRM2m0v7QPz4748M+mYQG67cwIqLVpD8bzJgZi7dsMFU\nePzvfy45lFvJuZKXxMT/yd/QudIclx9/hNRU6NwZatcu/HaFjYlScPPN5vXXXxc9f/6mNJ8rxeUP\nMZEChBCu5qJRmA6fOcz1313Ptd9ey5bFW0j9NxWdpSl3STn+++9coeHDDyEsrIR5FkIIUSiOH/WO\nH/nu4Nj37NmmsCKEr5E+EEK4WrVqZmrSY8egatUib661ZsamGYz+eTSn008THhTOB4Ef0OCtBlS9\nsSoN3m/Atdeau2DXXw8zZ7rhM4gyR/pACHFhp06Zvgk2m5n7oRhf8YV22WWwbh3MmQMDB7rvOEIU\nJL9rg0xLK4SrlbAPxLwd8xg2exgA/Rr0Y2L/idQuXxvbWBu2ZBu//WYKD5GRphmTEEIIz5g3z/Q/\n6969aIX90X8vAAAgAElEQVSH0ytPs7H3RqI6RtFyQctCbTNokClA/PijFCCE75EmTEXgD23SPE1i\n4kQJ+0D0b9ifgZcO5ItrvmDezfOoXd40sg0IDEBFBGaP+PHccxAT45Ice4ScK3lJTPyf/A2dK61x\nmT3bPA8aVLTtotpFkfFNBk1/aFroba691jzPm3fuvlRpVFrPlZLwh5hIDYQQrlbCPhABKoDZQ2aj\nVN7WJFOnwsaNpuPeAw+UJJNCCCGKIiXFzP8A537cF5ayKCzhlkKNwuTQtCnUqwd79sCKFdClS9GO\nKYQ7SR8IIVwtMhLOnoWkJPO6AKfSTlEhtEKhdpuaCg0amHHBp06FYcNckVkhDOkDIUTBZs+GwYOh\nfXtYtcozxxw7Ft55Bx59FMaP98wxhchJ5oEQwlMKMRN1elY6Tyx8gvoT6nMo6VC+6RIWJLDvpX2k\n7kvlvfdM4aFVKxg61NWZFkIIUZDiNl8qCUffhx9/BCkvC18iBYgi8Ic2aZ4mMXHC0QcinyZMG45u\noN1n7Xhj+RskpiWyaO+ifHcVUiuEzOOZHJx7ildfNevGj3f5HHUeIedKXhIT/yd/Q+dKW1ysVvj5\nZ/O6uB2aixOTzp0hOhp27oR//y3ecX1daTtXXMEfYuKHP0OE8HH5jMJktVl5bdlrtPusHZuOb6J+\ndH2W3b6MW1vemu+uwpuE0+D9Bny0twZJSdC3L/Tq5c7MC+FaSqmKSqnZSqmzSqm9Simno+crpYKV\nUh8rpY4qpeKVUj8qpWp4Or9COPP333DyJNSta2aJLqqTv55k41Ub2XLDliJtFxgIV19tXs+ZU/Tj\nCuEu0gdCCFcLDDSFiMxM89pu07FNtP6kNVZt5d629zK+93jCg8MvuLujR81FKy0N/vnHNGESwtXc\n1QdCKeWYS3ck0AaYD3TUWm/Lle5x4GagN5AEfA6Eaa2vz2e/cm0QHvPcc/Dyy3D//fD++0XfPn5u\nPJsHbqbS1ZVo/lPzIm07axZcdx106AArVxb92EKUhEf6QBThTtM4pVSGUipJKXXG/lzHlXkRwmvy\nGYWpebXmvN33bX695Vc+6v9RoQoPAG+9ZQoPAwdK4UH4F6VUGDAYeFZrnaq1/guYCzirdqsD/Ka1\njtdaZwDfAIUf81IIN5o/3zz371+87bXNXtgtxuB8ffpASIjpuH3kSPGOL4SruboJ00dAGlAFGAZM\nVEo1ziftN1rrKK11pP15n4vz4nL+0CbN0yQmuWgNWps+EE6GYX2ww4P0rd/3gruxZdnQWnPiBHz0\nkVn33HMuzanHybmSVxmISUMgS2u9O8e6DTgvGHwBdFFK1bAXPG4BfvZAHkukDPwNi6U0xeXwYVP7\nGxYGsbHF20elqythnW+lyfQmRd42IuJc09Wffire8X1ZaTpXXMUfYuKyAkQR7zQJUTrZax+2Vy7Z\nbg68cYC/m//NtPviSUmBfv3gsstckD8hPCsCOJ1r3WnA2fjGO4D/gEPAKaAR8JJbcydEITg6T/fs\nCaGhxdtHQGAAljALlvDizQ/k6Lg9d27xji+Eq7lyIrn87jR1zSf9AKVUPHAE+FBr/bEL8+IWscW9\n9VCKSUzOl5h8kvsHw4wWcNHOn+nXoF+R96G15ti0Y6RsS2HublOL4e+1DyDnijNlICZngahc66KA\nM07SfgyEABWBFOAJ4Ffg8vx2PmLECOrUqQNAhQoVaNWqVXZMHXfwZNk7y451vpKfkiyb5ktx1K8P\nULL9ORR1+4oVzfLixbGkp8OKFZ77/O5ejo2N9an8+MKyY503jh8XF8eUKVMAsr9fnXFZJ2qlVBfg\nO611TI51dwJDtdY9cqVthLnDdAxzcfgBeFhr/a2T/erhw4fLRUKWfX55we4F3PLWLcSnxBNWEz4Z\nMpVaCbWKvL+MYxmEjgwlDQtXpaTT+rIA1qzx/ueT5dK1HJfrIvHCCy+4vBO1vWY6AWjquLmklPoS\nOKS1fjpX2k3A01rrn+zL5YFEoLLWOsHJvqUTtXC79HSoVAmSk+G//+Cii7yXl5YtYeNG+P13Uxsi\nhCfkO8CG1tolD6AVcDbXurHAj4XY9glgZj7vaV+xePFib2fB50hMtE7OSNaj54/WPI/meXTHO9DT\nooNKtM+EQ5n6svDTGrRessRFGfUyOVfy8qWY2L9rXXZN0Oe+w2cA04EwoDOmUNDYSbpJwExMDUUQ\n8DRwoID9uj8oheBLf0NfUlrismCB6djWokXJ91XSmDz+uMnLo4+WPC++pLScK67kSzHJ79oQ4MJC\nyg4gUCl1SY51LYHCDHqsAZcPHyiEJ2RaM5m7fS5BAUG80mUcSyZDzZTitXN1mPJtIGuTo+jaFa64\nwkUZFcI7RmMKD8cxBYlRWuttSqkuSqmkHOkeBdKBnZja6SsBD875K0Rejv4PxR19yeHo1KNsvGoj\nO+7bUex9XHmlef7115LlRQhXcOk8EEqpGZjCwF1Aa2Ae0EnnHe/7GmCJ1vqUUqo9MAt4Ums9zck+\ntSvzKIQ7LD+wnLCgMFqF1YPy5SEqCk7n7jtaOFlZUL8+7N9vOswNGODizArhhLvmgXAXuTYIT2jY\n0MwCvWyZmRW6uGyZNmypNlSgwhJWvBtMGRlmVurkZDhwAGrVKn5+hCgsj8wDQeHvNN0E7LKvmwK8\n6qzwIIS/6HRRJ1pVb3VuDoiA4v9rzZ5tCg8NGpT8rpcQQoji2bXLFB6io+HyfLvyF05AUACBUYHF\nLjwABAef6/vw228ly48QJeXSAoTWOlFrPUhrHaG1rqPtnaK11su01lE50g3VWlfWZv6HJlrrD12Z\nD3dxdEAU55SlmFhtVj5b+xnpWekFJLICEGd/LtL+U60c/uQwH7+eCcCYMSUqh/icwp4rderUQSkl\nDzc8ChpRQxRPWfoOLIrSEJcFC8xz79555gUtFlfEpDQ2YyoN54qr+UNMXDmMqxCl1q6EXYyYM4K/\nDvzF7sTdvNbrNecJHTUQTiaRu5CsU1nsmplI/7XH+adiK4YPL0GG/dj+/fuRpinuoYpxXgpRVjkK\nEH36eDcfOTkKEAsXmuaugfIrTniJS/tAuIO0cxXeZNM2Jv49kcd/f5yUzBRqRNTg82s+z39+h2PH\noHp1qFYNjh4t8vFuvBG+n6l58inFK6+UMPN+yt7e0tvZKJUKiq30gRDinMxMM3zrmTPeH741t0aN\nYPv2kvfLEKIwPNUHQohSIzkjmT5T+3D/L/eTkpnC0OZD2Xzf5oInh3M0XSpG26N9++CHHyAwSHH/\n/cXLsxBCiJJbtcoUHho3dk3h4eAHB1kauZQ9T+0p8b5KYzMm4X+kAFEE/tAmzdNKc0zCgsKIDImk\nclhlvr/he6YPnk50ueiCN7I3YYrLzCzy8d5/32w+ZAjExFw4vb8pzeeKKLvkvHbO3+Pi6uZLtjQb\na8+uxZZhK/G+SlsBwt/PFXfwh5hI6zkh8qGU4tOrP8WmbVSLqFa4jYo5ClNqKkyaZF4/9FCRNhVC\nCOFiLu//4OgeZyl5K8Fu3SA0FNasgePHoWrVEu9SiCKTPhBCuNLevVCvHtSpY14XQtr+NOL6buPt\n7TU41a46q1e7N4u+TvpAuI/0gRDiwhISoEoVM/JSYiKEh5d8n7YMG7Y0GypIYSlX8iGd+vY1hZzp\n02Ho0JLnT4j8SB8IIfKxN3EvA74ewPb47SXfWTFqII7NOEbo9tN0IIF77y15FoT3xMfHc99991G3\nbl1CQ0OpXr06vXr14o8//gCgbt26vP32217OpRCiIIsWma/yLl1cU3gACAi2zwPhgsIDmAIEnKsp\nEcLTpABRBP7QJs3T/DkmVpuVd1a8Q7OJzZi3Yx5P/P5EyXfq6AORXsBcEbnsn3wcgJXhVRkypORZ\n8FX+fK4U1uDBg1mzZg2TJ09m586dzJ8/n379+nHy5ElvZ024SVk4r4vDn+PiruFbXRkTR94WLAB/\nr4jz53PFXfwhJlKAEGXSpmOb6DSpE2MXjCUlM4Wbm93MpwM+LfmOi1EDMbNdCyZQn+YjowkLK3kW\nhHecPn2aZcuW8dprrxEbG8tFF13EZZddxtixY7nxxhvp3r07+/fv57HHHiMgIABLjpmpli9fTmxs\nLOHh4dSqVYv77ruPM2fOZL/fvXt37r33XsaMGUN0dDTR0dE8/vjj3viYQpRqWvvm/A+5NW0KNWrA\nkSOwebO3cyPKIilAFEFsbKy3s+Bz/DEmZ9LPcMXkK1h9aDU1I2vy080/MeO6GVQNd0FPNPswrrER\nEYVKfvo0TJoTwmxqcc/o0v3v6I/nSlFEREQQERHB3LlzSXdSAzVr1ixq1arFuHHjOHr0KEeOHAFg\n06ZN9O3bl2uvvZZNmzYxe/ZsNmzYwMiRI8/bfsaMGWitWblyJZ9++imffvop7777rkc+m8hfaT+v\ni8tf47JzJ+zfD5UrQ6tWrt23K2Oi1Pm1EP7MX88Vd/KHmJTuXyxCOBEZEsm4buO4t+29bB29lasb\nXu26nRexBuKrryAlBXr2hEsvdV02Sj2l3PMoAYvFwpdffsm0adOoUKECnTp14rHHHmO1vVd8xYoV\nsVgsREREULVqVarah0558803uemmmxgzZgz16tWjXbt2fPjhh/zwww/Ex8dn779GjRq89957NGzY\nkOuvv57HHntM+lMI4WKOH+O9exdrOp987X1+L0sjl3LgrQMu22dpKUAI/yQFiCLwhzZpnuavMXm4\n48N81P8jokKiXLtjRx+I1NQLJtUaJk40r8tC52l/PVeKYtCgQRw+fJh58+bRr18/VqxYweWXX85r\nr72W7zZr165l2rRpREZGZj+6dOmCUordu3dnp7v88svP265jx44cOnSIs2fPuu3ziAsrC+d1cfhr\nXNzVfOnipy4m45sMYka7bpKfXr3M85IlZihwf+Wv54o7+UNMpAAhSq20rDS+XP+lZ4cEdcxEXYi7\n2csmnWHXNis1asA117g5X6WN1u55uEBwcDA9e/bk2WefZdmyZdxxxx08//zzZOYzuaDNZuPOO+9k\n48aNbNiwgQ0bNrBx40Z27txJK1e3oRBC5CsjAxYvNq9793btvgNCArCEW7CEumYUJjDzP7RuDWlp\nsGyZy3YrRKHIRHJF4A9t0jzNV2Py665feeCXB9iVsIuQwBBuanaTZw5sr4GILV/+gkn3vLif7znF\nuhvaExQU7O6ceZ2vnivu1rhxY7KyskhPTyc4OBiro5Bp16ZNG7Zs2ULdunUL3M+qVavOW16xYgUx\nMTFEFLK/jXCPsnpeX4g/xmXlSjh7Fpo0gZo1Xb9/d8Skb1/45x9Tc+LqQo+n+OO54m7+EBOpgRCl\nypbjW7hq+lVcNf0qdiXsonHlxtSMdMOVID+OPhCWgu8ynTkD98U3YwTtGfZA6S88lAUJCQn07NmT\n6dOns2nTJvbt28fMmTMZP348vXr1IiIigjp16rB06VIOHz6cPbTrE088werVq7n33ntZv349u3fv\nZt68eYwaNeq8/R8+fJiHH36YHTt28P333/Pmm28yduxYb3xUIUolfxh9KTfpByG8RQoQReAPbdI8\nzZdiErcvjhYft+DXXb8SFRLFG73eYP2o9Vxx8RWey4SjD8QF2qV/+63pPN28azD163siY97nS+eK\nO0RERNCxY0cmTJhAbGwszZo149lnn2XYsGF88803ALz44oscOHCASy65JLsTdfPmzVmyZAn79+8n\nNjaWVq1a8cwzz1C9evXz9n/LLbdgtVrp0KED99xzD3fddRdjxozx+OcU5yvt53Vx+WNc3F2AcEdM\nOnWCsDDYuNEM6eqP/PFccTd/iIk0YRKlRueLOtO4cmO6XdyN52Ofp0p4Fc9nopB9IL74wjzfcYeb\n8yM8Jjg4mJdffpmXX3453zQdOnTgn3/+ybO+TZs2/PzzzwXuPzAwkAkTJjBhwoQS51UIcb6TJ2HN\nGggOhq5dXb//XQ/vYuPHG2n4fkNi7nRdR+qQEIiNhZ9/hoUL4bbbXLZrIQokNRBF4A9t0jzNl2IS\nZAli7d1r+bD/h94pPMC5PhAVK+abZOtW09Y2MhKuu85TGfM+XzpXhHAVOa+d87e4/PGHGUehSxcI\nD3f9/q0pVlqktUBnuX5QD39vxuRv54on+ENMpAZC+BWrzcqMTTMItgQzpNmQPO+HBIZ4IVc5FKIP\nxLI79tKZCJrfWInwcCnDiwtTJZyjQghRsN9+M89u6/9gvzQoi+v/lx15XrjQXIJcOX+FEPmR06wI\n/KFNmqd5KiZWm5VvN39Ly49bctuc2xi7YCypmT448LW9CVPc6dNO3z6zK42GK/fzHNsYPtTmyZx5\nnfz/FN+iRYuk6ZKPkvPaOX+Ki9bnChB9+7rnGPXfr491npXqt1W/cOIiatQIatWC48dNXwh/40/n\niqf4Q0ykACF8mk3b+HL9lzT5qAk3/XATW05s4eLyF/NKj1cItvjg6EUXmIl66QvHAdgcVYkO3aUC\nUAghvG3bNjh0yMyr0KKFe45hCbVgCbcQEOL6n11KnSv4+GszJuF/lEcn2SoGpZT29TwK99Fac/kX\nl7P60GrqVqjLk12eZHjL4d5vqpSfhQtNfXLv3k6/yYf3TCFw0VE6jKrI3RPz7ydRlimlPDv5XxlS\nUGzt7/lNWym5NghXeecdGDsWhg2DqVO9nZvi+e47GDIEevaE33/3dm5EaZLftUFugQqfppTitZ6v\ncSDpADc3u5kgS5C3s1QwxyhMTmog4uNhxpIwtKUerzzv2WwJ4U1KqYrAJKA3cAJ4Wmv9dT5p2wDv\nAG2As8ArWuv3PZVXUfb44/wPufXsaWoili41Q4SHhXk7R6K0kyZMReAPbdI8zVUx2Xx8M99t+c7p\ne93rdue2lrf5fuEBzs0DkZiY563vvoOsLFM5Ua2apzPmffL/U6Z9BKQBVYBhwESlVOPciZRSlYBf\ngIlARaA+4NONMuS8ds5f4pKWBn/+aV67eyZnd8akUiVo2xYyMmDJErcdxi385VzxJH+IiRQghNdk\nWDOYuWUmvb7qRfOJzblz7p0kpSd5O1slU0AfCEfV+K23ejA/QniZUioMGAw8q7VO1Vr/BcwFnP0n\njAV+1Vp/o7XO0lona623ezK/omxZtgxSU6FlS6ju+v7N2bYO28rGqzZyYs4Jtx3DUYPi6BAuhDtJ\nHwjhFeMWj+PjtR9zPNl0Kg4LCuP2Vrczrts4783h4Apz58LAgXDNNfDjj9mrd/yTRaM2FsLCFceO\nuWec8dJC+kC4jzf6QCilWgF/aa3Dc6x7BOiqtR6YK+0fwCagHab2YSVwv9b6gJP9yrVBlNhjj8Gb\nb8Ljj8Prr7vvONY0KzpDExAaQECwe+7dLlkC3bpBkyawZYtbDiHKIOkDIXzK5hObOZ58nKZVmnLP\nZfdwa8tbqRBawdvZKrl8+kAsH3OArznK+ssvJTw82gsZE6XBCy+8wA8//MBG/xqrMQLIPa7xaSDS\nSdpaQGugF7AZGA98DXRxtuMRI0ZQp04dACpUqECrVq2yJ2ByNAGQZVkuaPm338xytWpxxMW573hL\nVy51++fJzISIiFi2boWZM+OoUsX78ZVl/1uOi4tjypQpANnfr05prX36YbLoGxYvXuztLPicgmKS\nZc3S8cnxTt9bf2S9XrZ/mbbZbG7KmZd8/73WoBdfcUX2KptN60su0bouZ/SCGWlezJx3Ffb/x5f+\n54tqxIgResCAAW7bf3Jysk5ISCj28QqKrf09d3yHtwLO5lo3FvjRSdr1wBc5lqMxU3BFOklb6M/t\nTnJdcM4f4nL4sNagdViY1mke+Gr2REwGDDCfadIktx/KZfzhXPE0X4pJftcG6QMhXMpqs7J0/1Ie\n+uUhLn73Yu766S6n6VpWb0nn2p1L3wy7TvpArFwJu3dDWo0Ietzoo8PPCr8QFhZGxYp+N/zvDiBQ\nKXVJjnUtAWeNLDYCudslaaCUfVEIX+AYfSk2FkJKyVezzAchPEUKEEXgqOoR5zhiEp8Sz+j5o6n1\nTi26TunKhNUTOHTmENvit5Fly/JuJj3J3oQpNkdvvGnTzPPQoWCxeCNTvqGs//8cOHCAQYMGERUV\nRVRUFNdddx2HDh06L82rr75K9erViYqKYsSIEbz44ovUrVs3+/0XXniB5s2bZ7/+8ssvmT9/PgEB\nAVgsFpb44PArWusUYBbwolIqTCnVGbgGcDbi/mRgkFKqhVIqCHgOWKa19tnRFcr6eZ0ff4iLp4dv\n9URMHJ9l4cJz97N8nT+cK57mDzGRAoRwiYjgCL7a+BVHzx6lboW6PNbpMVbesZKt920lMKAMdbXJ\nVQORkQHffGNWyehLZdvAgQM5ceIEcXFxxMXFcfjwYQYNGpT9/jfffMOLL77Iq6++yrp162jUqBFv\nv/12nlo6x/Kjjz7KjTfeSK9evTh27BhHjhyhU6dOHv1MRTAaCAOOA9OBUVrrbUqpLkqp7MKB1nox\n8DTwM3AUqAcM9UJ+RSlns50rQDju2rvTxqs3sjRyKYmL8w7x7Ur160OdOnDyJPzzj1sPJcq4MvTL\nruTi4uL8olToDmlZaaw4sILF+xbzQPsHskdKcsQkNDCUzwd8TsNKDWlVvVXpa5pUWI55IE6cIBZY\n+M5pYhIgpmkULVqU0ZjYufr/R73gPJ56XD6jDBUxvSstXLiQTZs2sWfPHi666CIAZsyYQf369Vm0\naBE9evRgwoQJjBw5kttvvx2AJ598ksWLF7Nz506n+wwPD6dcuXKkpKRQpYpvj1ymtU4EBjlZvwyI\nyrXuE+ATD2WtxMrydaEgvh6X9evN5J4XXQSXXur+49mSbaw9u5Zmuplbj6OUqYX49FMznOtll7n1\ncC7h6+eKN/hDTKQGQuRr3ZF1vPTnS3T/sjsVXqtAj6968NKSl/hl1y9O0w9pNoTWNVqX3cID5KmB\nSHxrL+/zD2OaH6csh6Ws+/fff4mJickuPADUrVuXmJgYtm7dmp2mXbt2523XoUMHj+ZTiLLCMVdC\n37545LtZ28yNCmVx/8EczZikH4RwJ6mBKAJfLw262hfrvuCjNR9lL7es1pLudbrTpEqT7HVlLSYX\n5OgDERPD8S1pxJw4RQaKXs9V8nLGvM/V50pRaw48UdOQ77G1zrdgnXN9mS58+yn5DnTO1+OSswDh\nCS1+bUHzzOYEhLn/vm2PHuYe1vLlcOYMRDobMNmH+Pq54g3+EBMpQJRBiamJbDy2kbVH1rLq0Co6\n1erEQ5c/lCfdoMaDUErRo24Pul3cjUph8iP4gnLUQPwyHzZSg5ox0KeJ/KuVZU2aNOHQoUP8999/\n1K5dG4A9e/Zw+PBhmjZtCkCjRo1YvXo1w4cPz95u1apVBe43ODgYq2PuESFEoZw5Y35cBwRAz56e\nOaalnAXKeeZYFStC+/ZmBMA//4Srr/bMcUXZIk2YisAx0Ya/WrB7AbXfqU30G9HEfhnLIwse4bst\n3zF/53yn6XvV68UH/T5gcOPB+RYe/D0mLufoA3HsGJN/DuVtLqXiyx5oYOsHysq5kpSUxIYNG857\n1K9fn5YtW3LLLbewbt061qxZw7Bhw2jbtm32naaHHnqIKVOmMHnyZHbt2sUbb7zB6tWrC6yVqFOn\nDps3b2bHjh2cPHmSrKwyNOKZjygr53VR+XJc4uIgM9P8yPbkqMiejIk/Defqy+eKt/hDTOS2aCmQ\nkJrA9vjtbD+5ne3x2ykXVI7/dftfnnTR5aI5kHSAcoHlaFa1Ga2qt6JDzQ50ushnR27xP/a7wcfS\novjzTwgNheuu83KehEctXbqUNm3anLfuuuuuY86cOTzwwAN0794dgN69ezNhwoTsNEOGDGHv3r08\n9dRTpKSkMHjwYEaNGsXcuXPzPdZdd93Fn3/+Sdu2bUlOTmbx4sV07drVPR9MiFLC0XzJU8O3ekOf\nPvDCC+c+qxCupswkc75LKaV9PY/ulpKZQlhQWJ71O0/upMPnHUhMO39YuIvLX8y+MfvypM+wZrAn\ncQ8NohtgCSjDExK400cfwejRvNphDk+vGsiQIeeGcRWFo5SirP/POwwePBir1cqPP/7okv0VFFv7\ne37TCUOuDaI4tIZLLoG9e00zpo4dvZ0j98jKgipV4NQp2LnTDO8qRHHkd21waQ2EUqoiMAnoDZwA\nntZaf51P2teBOzCzjE7SWj/hyrz4o+SMZF5a8hKHzxzmyNkjHD5zmENJhwgNDOXoo0fzpK8WUY3E\ntETCg8JpWKkhl1a+lEsrXUrjyo2d7j/YEkyjyo3c/THKNpsNGzB1R3tA5n4QhZeamsrEiRO58sor\nsVgs/PDDD8ydO5dZs2Z5O2tClBrbtpnCQ+XKpgmTp6zruI7kLcm0/qs1Ec0j3H68wEC48kpzA2v+\nfHgobzdHIUrE1U2YPgLSgCpAG2C+Umq91npbzkRKqXswM5E2t6/6XSm1W2v9qYvz41LOxuXVWpOS\nmUJ4cHie9Fm2LF768yXiU+KJT40nPiWekyknSUpPYveDu/O0bQ6yBPH6X6873U96VjohgSHnrY8K\nieL4o8epHFbZa6O3+MNYxR5ltbKGwfRLnEuzyGvo06eGt3PkM+RcKZhSil9++YVXX32V1NRUGjRo\nwLRp07jmmmu8nTVRADmvnfPVuMy3d/m76iqweLAivuUfLYlbHEd4k7y/Fdylf3//KED46rniTf4Q\nE5cVIJRSYcBgoInWOhX4Syk1F7gVM7NoTrcBb2mtj9i3fQu4E/B4AeKv//4iOTOZlMyU8x73tbsv\nzwzKWmu6Tu7KqbRTnE4/TVJ6EknpSdi0jfRn0wm2BJ+X3qIsvPbXa2RYM/IcNzkzmYjg8+9CBFuC\nebP3m0SXiyYmMoYakTWoEVGjwAKCY0I34SNsNr6hMz8Rwx1dAgkK8naGhL8IDQ1l4cKF3s6GEKWa\nowDh6ZGJLGEWLOEWj8wD4XDllWaOiz//9I/hXIV/cVkfCKVUK+AvrXV4jnWPAF211gNzpT0F9NZa\n/0wci8AAACAASURBVG1fvgxYpLUu72S/etRPo8iwZpBuTSfDmkGGNYMZ181w2i+g5cctSUhNMOmz\nzqVPeCKBqJCoPOmjXo3iTMaZPOtPPXGK8qF5skOF1ypwOv30eevCgsI48PABostF50k//q/xhAaG\nUjmsMpXCKlE5rDKVwypTK6oWAUoGwSptst54m1pPDOUY1Vm1yrNV5KWF9IFwH+kDIcqyxETTLwDM\nLNQVKng3P57QubPp6zFrFgzKMxe8EBfmiT4QEcDpXOtOA87KvLnTnravc6rnzWag5pH3jeRMmPmx\n7+hYvLrpalovbU1QtLnVeyjpECdTTzL5w8lEpEZkb5e7FsCxXbc63UjLSiMsKIywoDCGPDyEkOQQ\nNn+ymfab22fv1+HHyT8SOSeSitUrUj60PJHBkQRZgljddDVZiWYIxXab22Vv91jnx847Xu795bdd\n7nzKdv6x3aIdF3GM6jSseIJ27aR2SAghfMWCBWagvG7dykbhAUwzpuXLTc2LFCCEK7myAHEWyH2L\nPwrIe3s/b9oo+zqnPj/7OdWpTq9jvYioFkHDpg2zm/+sOrCKpKVJ9BrYC4D3G79PgAogJisG61kr\n61nPzI4zqVTOzGPgGFs38GggaHikxiPAuVn/3r/9faxnrLSiFehz6WNjY4mLi2PTkU003tGYNo3b\n5NlfVkIW61lPytKU7PzkPl7O/Tnyn9/xnH0+Xzveu+++S6tWrUrt5yvq8d6MOwvE0bnm3yj1WJ70\nZXk557jWF0ov3MfRtjYuLo4pU6YAZj4JUTz+0FbZG3wxLt5qvuTgjZj07w/PPAM//2ymKQrwwYYP\nvniueJs/xMSVTZjCgASgqdZ6t33dl8AhrfXTudL+hRl56Qv78kjgTq11ngkJlFI67VAaAMHVg1EB\n59eipB9JJ7ia8/XYP5qrtouLi6PjpR09djx/2M5xkvt6Pj2xXUZEMNUrZ5KSGcz0AaMYOvdjxDmF\n/UKUJkzuI02YXM8fLvTe4GtxsVqhWjU4eRK2boXGzgcrdJvVTVazct9K7v73bkJrh3rsuFpD7dpw\n8CCsWQOXXeaxQxear50rvsCXYpLftcGl80AopWZgfk7dBbQG5gGd8hmF6UHMcK8AC4D3tNafOdmn\nT1wkhLiQqVPhttugM8tY9uxv8NJL3s6SX5IChPtIAUKUVStWQKdOUK8e7NplOhd70vJay8k4lMHl\n/11O6EWeK0AAjBoFn3xiJpb7X945ZoUoUH7XBldXZo0GwoDjwHRglNZ6m1Kqi1IqyZFIa/0J8BOw\nCdgI/OSs8CCEP5k2zTzfylTfrCcWQogyytF8qX9/zxceALCZJ0+OwuTQv795dsRACFdw6a8crXWi\n1nqQ1jpCa11Ha/2tff0yrXVUrrRPaq0raa0ra62fcmU+3EXaZ+clMTGOHIHff4dgSxY3MJO4/fu9\nnSWfI+eKa9x8883ceOON3s6GsJPz2jlfi0vOAoQ3dNjZAes8K8HVgy+c2MV69ICQEPj7bzh2zOOH\nvyBfO1d8gT/ERG6TCuECX39tOqj1r7+DaBI9O0OR8LqAgAAsFgsBAQF5HhaLhZEjR3o7i0KUWQcO\nwPr1EBZmRmDyBku4fR6IAM/XQISHm0KE1qYztRCuIAWIIvCVDi2+RGJiOJovDWv6DwCxl1zixdz4\nptJ8rhw9epQjR45w9OhRPvvsM5RSHDt2LHv9e++95+0sCjcpzed1SfhSXObMMc9XXgmhnu1+cB5v\nxmSgfTau2bO9loV8+dK54iv8ISZSgBCihLZsgX/+MeOK96+71ayUPhBlStWqVbMfFewDzFepUiV7\nXaR9CthHHnmEhg0bEhYWRr169Xj22WfJysrK3s9TTz1Fu3btmDp1KvXq1aN8+fLccMMNnD6de4od\nePPNN4mJiaFy5crcfffdZGZmeubDCuFnZs0yz4MHezcf3jRwoOn7sWCBmZVaiJKSXzlF4A9t0jxN\nYnKu9uHGGyEkwPyIi9u714s58k1yrkCFChWYNm0a//77L++//z5Tpkxh/Pjx56XZsWMH/9/efYdH\nVaUPHP+eVEiBUEITQ6gioBQLCCiIoIIuxdXFgtgQCxbsKLsq1sXVRXFVLFhWLCD+VEBhxTIgIIhg\nQAFBejEJNQlJSAKZ8/vjzJAyk5BJZubOzbyf57lP5t45c+fknTv3zrmnzZs3j3nz5jF//nx+/PFH\nHn/88TJpvv76a3bs2IHD4WDGjBl8/PHHvPrqq0H8T4SbHNfehUpc9u2DxYshOtq6/g9uVsakWTMz\nClVhIcyfb1k2vAqVYyWU2CEmUoAQogacTvjgA/P42mtdG0BqIAJMqcAsgfaPf/yDs88+m5SUFC65\n5BIeeOABPvroI4907777Lp06daJ3797ceOONfPvtt2WeT05O5uWXX6ZDhw5cfPHFDB8+3CONEALm\nzjWn5QEDrJ19emnyUtYOWUtxfrFleXDXwIRiMyZhP/Irxwd2aJMWbOEek8WLTQe91FRzd4dic3Ho\n36GDpfkKReF+rAB89NFH9OnTh+bNm5OYmMiECRPYuXNnmTRt2rShbt26x9dbtGjB3r17y6Tp0qVL\nmXVvaURwyHHtXajEJVSaL/Xc1pPbM24noq51P7tGjDB/582DggLLsuEhVI6VUGKHmEgBQogaeP99\n83fUKFelg9RABIXWgVkCadGiRYwePZrhw4fz5ZdfkpaWxqOPPkpRUVGZdNHR0WXWlVI43ceVD2mE\nCHc5ObBwoalddHcitkpUQhRR9aJQlkxCYbRuDd26QW4uSIWlqCn5leMDO7RJC7Zwjkl+PnzyiXk8\napRro+tHnGPrVmsyFcLC+VgBWLp0Ke3ateOBBx6gR48etG3blm1h0ldGKdVAKfWZUipXKbVNKXXV\nCdJHK6V+V0rtrCxdKAj347oioRCXr76CoiLo2xeaNrU6N6ERE3dNjLtmJhSEQlxCjR1iIgUIIarp\n88/NaBY9e8Ipp7g2upowWTPVqQhlHTp0YNu2bXzyySds3bqVqVOn8n+hdBUPrFeBAiAZGAW8ppQ6\ntZL0DwIZwciYqL3cXy930x1RUoD44gsoNQCcED6TAoQP7NAmLdjCOSbvvmv+XnddqY2uGoj+HTsG\nPT+hLpyPFYDLL7+cO++8k3HjxtG9e3eWLVvmMbpSbaSUigMuA/6utT6itV4KzAGurSB9a+Bq4Nng\n5bL6wv24rojVcSkoKJk0LVQKEFbHBKBTJ+jQAQ4cgCVLrM6NEQpxCTV2iInSgW74W0NKKR3qeRTh\nZ/duSEkxQwOmp0PDhq4nxo6FN9+EN96Am2+2NI92pZRCvvOBUVlsXc/5vepMKdUNWKq1ji+17T7g\nPK21R8t0pdRc4E0gG3hfa51SwX7l2iAqNHcuDB0KPXrAqlXW5sVZ6GRJoyVE1Img7/6+1mYGmDAB\nJk+GO++EqVOtzo0IdRVdG6QGwgd2aJMWbOEakxkzTKfbYcNKFR6gpA/Epk3WZCyEheuxIkjAFAZK\nywYSyydUSo0AIrXWc4KRMX+Q49o7q+Mye7b5Gwq1D7pY48xzsirH4pKMi7sZ06eflrS6tZLVx0oo\nskNMoqzOgBB2ozW89555XKb5EpScjWUUJiHccoF65bbVA8rMh+tq6jQZGOzedKIdX3/99aSmpgJm\nkr5u3bodr/p3X4ADve4WrPezy3paWppl73/kCMyaZdZHjrTm/y+9rp2aNNLYzGbcrMzPWWdB06YO\n/vwTfvihP/37W3+8yHrZdSu/Pw6Hg3ddbbTd51dvpAmTED5asQJ69TKjeuzeDVGli+HXXQf//a/p\nIOFRuhBVIU2YAseiJkxxwEGgs9Z6i2vbe8AerfUjpdJ1BX4CDmAKDzFAfWAv0EtrvbPcfuXaILya\nNQtGjoSzzoKffrI6N6C1pjivGJwQVS807ttOnAjPPANjxphWt0JURJowCeEn7tqHa64pV3iAknkg\nIiODmichQpXWOh/4P+AJpVScUqoPMBR4v1zSX4GTgW5AV2AMZiSmrsCu4OVY2N2HH5q/V19tbT7c\nlFLH54EIFddcY/7Ong2FhdbmRdiTFCB84K7iESXCLSaFhfDxx+ax1woGVxMmx++/By9TNhFux4oo\nYxwQh6lN+AC4VWu9QSnVVymVA6C1dmqt97oXTK2FU2u9L5SrGuS49s6quBw8aEZfioiAK6+0JAsV\nCqVjpVMnM6lcVhbMn29tXkIpLqHCDjGRAoQQPpg7Fw4dgu7d4fTTvSSQmaiF8KC1PqS1HqG1TtBa\np2qtZ7q2L9Fal+8f4X7NoopGYBKiIp9+CkePwgUXQLNmVucmtLlraNw1NkL4QvpACOGDSy+FL7+E\nF1+Eu+/2kuBvfzPTU8+aBVdcEfT81QbSByJwrOgDEShybRDe9O8PixbBO+/A9ddbnZvQ5h6OPDYW\nMjOhnteivAh30gdCiBrKzIQFC0y/hwrb1koNhBBCWGLXLlN4qFOnZKjSUFC4p5DF8YtZ0XGF1Vkp\no2VLOO88M+mee9ZuIapKfuX4wA5t0oItnGLywQemi8OQIZCcXEEidx+I9euDlzGbCKdjRYQPOa69\nsyIu7v5pf/lLaN1Nj2keQ+/M3uT+O9fqrHhwd6a2shmTfIc82SEmUoAQogpKz/1QabW4uwZC2aYl\niBBC1AoffGD+hsroS24qwozCFBkXeqPz/fWvEB0N334LGRlW50bYiRQgfOCecEOUCJeYrFoFa9dC\no0ZwySWVJHQVIPp36xacjNlIOBwrN9xwAxEREURGRhIdHU2rVq24/fbbycrKqvI+Fi1aREREBAcP\nHqzwPYYOHerz60RghMNxXR3BjsuaNWZJSoLBg0+c3gqheKw0bGhq1Z1OmDHDmjyEYlysZoeYSAFC\niCpwT7QzejTExFSSUGaiDnuDBg0iIyODHTt2MH36dObOncu4ceN82oeqZg1WdV8nhN298Yb5e+21\nplOwqLqbbjJ/33jD1LYLURXyK8cHdmiTFmzhEJPc3JL2oTfffILErhoIx2+/BTZTNhQOxwpAbGws\nycnJtGjRgoEDBzJy5Ei+/vrr48/n5OQwduxYmjZtSr169Tj//PNZtWqVhTkWNREux7WvghmXvLyS\nu+cnPEdbKFSPlcGD4aST4I8/TCf0YAvVuFjJDjGRAoQQJ/Dxx6YQ0bcvnHrqCRLLKEyilK1bt7Jg\nwQKio6OPbxsyZAgZGRl89dVXpKWlcd5553HBBReQmZlpYU6FsK9ZsyAnB845B047zerceMpbn8fi\n+MVsvG2j1VnxKiqqbC2EEFUROvOq24Ad2qQFWzjExH1CHTu2ColdTZj6d+8euAzZlL+PFYdylOxb\ne+7boRwVbq/sdTU1f/58EhMTKS4upqCgAKUUU6ZMAeC7775j7dq17Nu3j1hXO4tJkyYxZ84c3n//\nfe6//36/50cEVjicA6sjmHHx6RxtAX1U48x30iO6h9VZqdBNN8FTT5mJ+Pbvh8aNg/fe8h3yZIeY\nyG1SISqRlgYrV5qOeZdfXoUXSA1E2OvXrx9r165l5cqV3HXXXQwZMoQ777wTgNWrV5OXl0fjxo1J\nTEw8vqxbt44tW7ZYnHMh7GftWli+HOrXN/N4hiLtNB0LVGTo9lFKSTFNmYqKSkYcFKIy8ivHB3Zo\nkxZstT0m7s7T114LdetW4QXuPhBr1wYuUzbl72Olv+5/fKno+eq8rqbi4uJo3bo1nTt35sUXXyQv\nL48nnngCAKfTSbNmzVi7di1r1qw5vvz+++88+eSTVdp/vXr1yM7O9tielZVFREQEiYmJfv1/ROVq\n+zmwuoIVl9Kdp+PigvKWPkvomkDfw33JfsbzextK3DU4we5MLd8hT3aIiRQghKhAfn7JuOJV7pgn\nNRCinMcee4zJkyeTkZFBjx49yMzMRClFmzZtyiyNq9hm4JRTTmH9+vUUFhaW2b5q1SpatWpVpr+F\nELVZfj68/755HKrNl6DUPBB1Q28eiNKGDIEWLWDTJms6Uwt7kV85PrBDm7Rgq80xmTULsrOhVy8f\nOua5+0CccUbgMmZTtflYqUy/fv3o3LkzTz31FAMHDqR3794MGzaMBQsWsH37dn788Ucef/xxli5d\nevw1Wmt+/fXXMrUUa9asAWDUqFFERUUxevRoVq9ezZYtW3jnnXeYOnUqDz74oFX/ZtgK1+P6RIIR\nF3fnaZ/O0RYK9WOldGfq118P3vuGelysYIeYSAFCCC+0hldeMY99urMlNRDCi3vvvZfp06eza9cu\n5s+fz4ABAxg7diwdO3bkyiuvZNOmTbRo0eJ4eqUUAwYMoEePHmWW/Px86tWrxw8//EBxcTHDhg2j\ne/fuvPzyy0yZMoWxoXwbVgg/0hqmTjWP5bD3nzFjzOVr9mzYtcvq3IhQpnSIzxqilNKhkkeHw2GL\nUmEw1daYLF9uhgRs1MicRKvU/wHg7LNh5Uocr71G/1tvDWge7aaqx4pSilD5ztc2lcXW9Vzo9vIs\nJ1SuDbX1HFhTgY7LokXQvz8kJ8POnVCnTsDeym/scqxceSXMnAn33QfPPx/497NLXIIplGJS0bVB\nbpMK4cXLL5u/Y8b4UHiAkpmoZUZgIYQIGNfIyNx2W+gXHrIWZbE4fjFbJthjpLUHHjB/33jDNOMV\nwhupgRCinIwMM6RdcTFs3QqtWvnw4u7dzdivq1ebx8JnUgMROFIDIWqDzZuhQweIjja1D02bWp2j\nyuliTfGRYpRSRMaHdkdqtwED4PvvYfJkkK5V4U1qIISootdfh6NHYdgwHwsPUNIHItIeFwkhhLCb\n5583fSCuuSb0Cw9g5n+ISoiyTeEBwD2n5UsvmbkhhChPChA+sMO4vMFW22JSVATTppnHrrm/fOOe\nB2LVKv9lqpaobceKECDHdUUCFZf0dHjnHdNK1G53xt/9/F1Wp69m4/6N7MnZQ05hTsjWtg4eDJ07\nw59/wkcfBfa95DvkyQ4xibI6A0KEktmzTROmLl1MBz2fSR8IIYQImClTzI2eyy6Djh2tzcvBIwdJ\ny0jj18xf2XJoC1sObWHroa3MGDGDM1p4DuX9wo8v8Nua38psqxtVl7lXzeWCNhcEK9tVopSphbjh\nBlPjM3q0XNZEWdIHQohSzjnHjMA0bRrccks1dtCxI2zcCBs2WH91synpAxE40gdC2NmBA5CaCrm5\nsHIlnHmmtfnp83Yflu1a5rH9y6u/ZEj7IR7bb513Kyv2rCCvKI/colyyC7PJP5rPqrGr6NG8h0f6\nK2dfSXZhNj1P6kmfk/vQN6UvdaN9GdWjZoqKoHVrUwsxdy5cemnQ3lqEkIquDVIDIYTLsmWm8JCU\nBKNGVXMn0gdCCCEC4vnnTeHhoouCU3jIyM1gweYFnNniTLo06eLxfJ+T+1DsLKZbs250aNSBtg3a\n0qZBG9o3al8m3f4v9rP+6vXcNfwuOn3QqcxzhwsPey0UOLWTBZsXkF2YzYLNCwCIjYylb0pf3hv+\nHifVO8mP/6l3MTFmKNf77oPHH4dLLpFaCFHCb30glFINlFKfKaVylVLblFJXVZL2MaVUkVIqRyl1\n2PU31V95CRQ7tEkLttoUk8mTzd9x4yA+vpo7cTVhcvz0k38yVYtU9Vhp1aoVSilZArC08nlUAHEi\ntekc6E/+jsvevSXDaz/xhF93XcbunN38+8d/0+utXjR/oTk3fHEDM9bO8Jr2uUHPsXzMcqZdOo17\nz7mXYR2HcVrT06gTVXZcWWeRE2e+kxW7V3jsIzE2kagIz3u5CsXa29Yy6/JZ3NPrHro3605hcSHL\ndy8nOT7ZP/9sFdx6KzRrBqtWwZw5gXkP+Q55skNM/FkD8SpQACQDPYAvlVJpWusNFaT/WGs92o/v\nL0S1rVtnTo516sBdd9VgRzITdY1t377d6iwETShNFhRISqkGwNvAIGAf8IjW2qNrplLqfuA6oJUr\n3Wta6yBMZSVC3XPPQV6eaUZz9tmBeY/317zP6M9LfpbUjarL+a3P54zmnv0ZfOK6LPhyy1YpRUr9\nFFLqp3BF5ysA2J+/n/X71hMTGeORPqcwh38u+Seju46mY2P/NZ+Ni4OHH4a774ZHH4W//EUub8Lw\nSx8IpVQccAjopLXe4tr2X2C31voRL+kfA9pWpQAh7VxFMFx/Pbz3npmU6NVXa7CjVq3MwOQ7dpjJ\nJISwiUD2gVBKuQsLN+K6wQScU/4Gk6sA8Q2wFmgHfA08qLWe5WWfcm0IEzt2wCmnQGGhuRPew7O7\ngF9sz9pOp1c6MaT9EEZ2HsklHS4hLjquxvt1HnPiLHCiIhSRcYFp3jp99XTGzB0DQK+Wvbi+6/WM\n7DKSpDpJNd53QQG0bw+7d8OMGWb4XBE+Kro2+KsA0Q1YqrWOL7XtPuA8rfUwL+kfA8YDxUA68IrW\neloF+5aLhAioXbugTRtTefDHH+ZxtbVsCXv2mJ22bOm3PAoRaIEqQPh6g6nca18C0Frf7eU5uTaE\niWuvNT9cr7oKPvywZvvaemgrn67/lPt734/y0qA//2i+XwoNwZaWkcZ/fvoPs9bN4nDRYcD0mZg8\ncDJ39/L4+vjsnXfgxhvNfbGNG0N/9m/hP4GeSC4BKD/heTaQWEH6mcCpmOZOY4FHlVIj/ZSXgLFD\nm7Rgqw0xmTIFjh2DK66oYeEBSuaBWL685hmrZWrDseJvYRKTDsAxd+HBZQ3QuQqvPRdYF5Bc+UmY\nfIY+81dcVq82hYeYGHj66ertQ2vN4h2LGfbxMNpNbceD3zzIkp1LvKYNZOEhkMdKt2bdeGvoW6Tf\nl877I95nQOsBFBYX0rlJVb5mJzZ6NJx2mqlgd/dF8Rf5DnmyQ0yq1AdCKfU90A/wdrtnKXAXUL/c\n9nrAYW/701r/Xmr1R9ddpssxBQsP119/PampqQAkJSXRrVu34+2G3UGWdWvW09LSQio/vq7PmePg\ntdcA+vPQQ37Yf0EBwPFGolb/f7Ie2utWfn8cDgfvvvsuwPHza4D4eoMJAKXUJEAB7wQoXyLEaW3a\n3gPccYcZUtRX8/+Yz2OOx1j550rA3JW/ovMVNKzb0I85DR3xMfGMOn0Uo04fxY6sHaTU996U9tCR\nQzSo26DK+42MNP1QBg+Gp54ytULNmvkr18KO/NkH4iDQuVQV9XvAnhNVUbvSPgicrbW+3MtzUk0t\nAubxx2HSJLjwQvjf//ywwyZNYN8+M2RIcvBGyhCipgLYhKkbsERrnVBq271AP29NXF3P3wHcA/TV\nWqdXkEZfd911cnOpFq8vXAjPPNOf5GR4+20HCQm+729t3bXcveBu6v1Zj+Edh/Pczc/RNKFpSPx/\nVq1n5GaQcncKvVr2YvKYyZxz8jlVer3W8Pzz/fnqK7joIgcTJoTG/yPrgb25NGnSpMD1gQBQSn2I\nqaG4GegOzAN6exuFSSk1FFistc5SSp0N/B8wQWvtMVaaFCBEoBw4YJos5eTAokVw3nl+2GmjRnDw\nIOzfbx4LYRMB7gNR5RtMSqkbgceBc7XWOyrZr1wbarHDh03H6fR0ePttMyNydeQW5fLB2g+4tuu1\nlvVtSJ+ezh93/UHzm5rTfmr7E78gwD7b8BkjZ4/kqPMoAL1P7s0DvR9g6ClDiVARlb5282bo3NlM\nMrd0KfTuHYwcCysFug8EwDggDtgLfADc6i48KKX6KqVySqW9Etjs2vYu8Ky3wkOocZfQRAk7x+Rf\n/zKFh0GD/FR4gJI+EMuW+WmHtYedj5VACYeYaK3zMTeJnlBKxSml+gBDgffLp1VKXQM8DQyqrPAQ\nSsLhM6yOmsblkUdM4aFnT7juusrTpmWkcdf8uzjmPObxXEJMAreceYulHaObjm5K78ze7Lxkp2V5\nKG3EqSPYMX4HD/d9mKQ6SSzbtYwRM0fw0MKHTvjadu3ggQfM41tuMQWJmpLvkCc7xMRvBQit9SGt\n9QitdYLWOlVrPbPUc0u01vVKrV+ttW6sta6nte6ktX7FX/kQoioyMmDqVPP4qaf8uGOZB0IIb7ze\nYPJyc+lJoCGwUpVMMlqTgZWFDS1bBq+8AlFR8Prr3k+nTu3kqz++4oL/XkD317vz8k8vM3v97OBn\ntgoioiOISogiMjYwQ7hWR/PE5jxzwTPsumcXL170IqlJqVzX7QQlNZeJE01B4rff4J//DHBGRcjy\nWxOmQJFqahEI48fDSy/B0KHwxRd+3HFCgpnt6PBh81gImwjkPBCBINeG2unIETjjDNiwwdRCeBt5\nad6meTy48EE27DctpBNiEhjTfQzje42nVZLMtl4dTu2ssPlSZm4mTROaltn2/fcwYABER5uRsrp0\nCUYuhRWC0YRJCFvYtQvXyEvw5JN+3rnUQAghRLU98ogpPJxyCvzjH97T5B/NZ8P+DZyUeBLPDXyO\nXffsYsrFU6TwUAMVFR427NtAyykt+dsnf2PF7hXHt59/PowdC0ePmhGZ/NGUSdiL/MrxgR3apAWb\nHWPy1FPmZDdyJJx+up937u4DscT7GOPhzI7HSqBJTOxPPkPvqhOXb76BF180TZdmzKh4srLLTr2M\nj//6MVvv3soDfR7wy2zLwWDHY2X57uUoFJ+s/4Re03tx7jvn8sXvX+DUTp5/3gxEkpYGjz5a/few\nY1wCzQ4xkQKECCvr18P06aaCYNKkALxBcbH5KzUQQghRZZmZ5k42wKOPagqaLOHqT6/mcKHndFJR\nEVGM7DKSmMiYIOeyenb9exeLExaT/rbXEYlD2g3db2Db3dt4qM9D1I+tz5KdSxg+czjPLX2OxERT\n0IuIMHNELFxodW5FMEkfCBE2tIaLLjInuVtvLWnG5FeRkaYW4tgx81gIm5A+EMIqxcVmLp7vvoNT\nz8wkYcwIVmb8CMCLF73I3b3utjiHNbPjmR1sm7iNlAkptHm2jdXZqbbDhYeZ/st0Xl35Ko7rHbRI\nbAGYpsCPPmqmPvrlFzjpJIszKvxK+kCIsDdnjik8JCUFoO+Dm/SBEEIIn0ycaAoPEYl72dCvx3bm\nvAAAH99JREFUGyszfqRh3Yb8/dy/M7LLSKuzV2Pa6Sro2vyeUmJsIuN7jWfjHRuPFx7AfH4XXmjm\nUB06ooDCQgszKYJGfuX4wA5t0oLNLjEpKIB77zWPn3gCGjcOwJu474YqhWPRogC8gb3Z5VgJJomJ\n/cln6F1V4zJjBkyeDBGRGudlI+nYOolpl0xj1z27eHLAkzRLaBbYjAZByoMp9D3cl23nbbM6K36h\nVNmb0RER5nNs0qKA1Svr0H7Qd3y/zUFVawjlO+TJDjGRAoQIC1OmwNatZgbN224L0JtI/wchhKiy\nJUtgzBjzeOpL4Hj8cdbdvs7yid/8LSLGNQ9EjM2rICqRnAw3PvcFROex64cBDLjpG0577TSm/TyN\n3KJcq7MnAkD6QIha788/oUMHMz3DN9/ABRcE6I2KiiA21gyMLWPaCZuRPhAikA7kH+C9Ne/xTto7\nLLhmAYd2nsS550JWFtx+u5k4TtjfOx9mceOo+qAVDL4Tev6H/wz+D+POHmd11kQ1VXRtiLIiM0IE\ni9Zwxx2m8DBiRAALD1DS/0E6TwshBFprlu1axrRV0/hk3ScUFpvG8c/P+5xZD4wjK8ucl6dOtTij\nwm9uuDqJ4ny4+WZg/st0bpnCtV2vtTpbIgCkrYUP7NAmLdhCPSazZsFnn0FiohlfPKBKNWEK9bhY\nQWLiSWJif/IZeudwOHj6h6fp+05fZqydQVFxERe3u5hpvb9m9kO38+ef0K8ffPhh+NxzCZdjZcwY\neOEF83jD9AdYOK+eR5qi4iIe+fYR1u9bHzZx8YUdYiIFCFFr7d0L41y1pi+8ACkpAX5DGYFJCCGO\nG9FxBE3im/Bw34fZctcWnu08n0nXDWL3bkWfPjBvXsWTxdUmWx/eyuKExez7dJ/VWQmae++Fxx4z\nl8Urr4R33y37/JyNc3h2ybN0frUzt395O2+tfsvrnB8idEkfCFFrXXEFzJ4NAwfC11+DCnTr7uxs\nM0Zs/fqmYa8QNiJ9IISvtNakZaSxcOtCHuzzoNc0x5zHiIqI4ptv4LLL4PBhU/Mwd66pGQ4HziIn\nziInEdERRMSGzw0mreHvf4dnnjHrTz5phnxVCtbtXcdLK17i498+5nCRKTjER8fz9ICnbT/vR21T\n0bVBChCiVvrkE/jb3yAhAX79FVJTg/Cmhw5Bw4bQoAEcPBiENxTCf6QAIapq04FNzF4/m49++4jf\n9v4GwK+3/UqXJl28pp8xA264wcyvOXIkvPeeGW9ChIdXXoE77zQFirFjzXqUqwduXlEen6z/hOm/\nTGfJziV8/NePa8XcH7WJTCTnB3ZokxZsoRiT3btLhmr917+CVHgA6QNxAhITTxIT+wu3z/CKT67g\nlP+cwsTvJvLb3t9oVLcRd5x1B/HR8WXSORwOCgth/Hi49lpTeLjvPtPnIVwLD+F2rLiNGweffmqa\nq73xhmkVsGePeS4+Jp7UrFR+uOEHNt+5maGnDPW6j/fXvM/SnUspdhYHMefWscOxIqMwiVrl6FFT\n83DggJkZc+zYIL659IEQQtRynZM7szB2IcM6DuPyUy/nonYXERMZ45Fuzx7o0wdWrTJ3m//9b3MX\nWoSnESPMMOqXXw6LFkG3bqYmasiQkjRtG7b1+trcolxumXcLR44dITkumb90+AtDTxnKoLaDatV8\nIXYjTZhErXLvvWbSuJYtYfVqM7lN0GRkQPPm0KwZpKcH8Y2FqDlpwiQOHTnEwq0Lmb95Pqc0OoUJ\nfSd4pMkpzKFOVB2vhQYwzVTeew/uusv0d0hNhY8/hp49A5x5YQuZmTB6tOmXCKaG6qmnID6+4tfs\ny9vHs0ue5YuNX7D10Nbj25vEN+HPe/8kMiJMhvGyiPSBELXep5+auxtRUbB4MZxzTpAzsGePKbm0\naFFSPyuETUgBIjzty9vHW6vfYv7m+SzbtYxibZqIdE7uzG+3/+bTvtavN81HFy8265dfDm++acaW\nCGebbttExvsZdHi1A81GN7M6O5ZzOk3z4okTTcvfk0+Gl1+GYcMqf53WmnX71vHF718wZ9McUpNS\nmXn5TI90eUV5OLWTxNgw6aUfYNIHwg/s0CYt2EIlJps2wY03msfPP29B4QHKNGEKlbiEEomJJ4mJ\n/dn9M8w7mscj3z3CDzt/QClFv1b9mDxwMh/99aMq7yMrCyZMgK5dTeEhORkmTHAwa5YUHgCKjxTj\nzHOydN1Sq7MSEiIi4KGHYPly6NEDdu1yMHw4DB0KGzdW/DqlFF2adGHieRNZMWYFH1z2gdd0s9bN\nouFzDTn3nXOZ5JjE99u+J68oL0D/TWDY4bwifSCE7WVmwuDBkJNj7njddZdFGZGZqIUQISSnMIeV\ne1ayYs8Kfsn4hY//+rFHc4/UpFTuP+d+erXsxcA2A6lfp37V958DL71k5tnJzjbDc95yCzz7LKxZ\nE4Shs+3CdWlQERKQ0s48E376Ce65x8wTMXcufPkljBoF//gHtGtX+eujIrz/hN2etR2tNUt2LmHJ\nziUARKpInr/wecb3Gu/n/yJ8SRMmYWu5uXD++fDzz3DGGeBwmKFbLbF1K7RtC61bm8dC2Ig0Yao9\nxi8Yz8KtC9mwbwOakhitGruKHs171Hj/e/bA66+b4TjdI1YPGGAKDmefXePd1zrOQtc8ELERRMRI\nww9v/vwTJk2Ct982I3ZFRsI115iO92ee6fv+sguycWx38N2271i6aym/ZPzCZyM/8zrK0zdbv0Fr\nzRktzqBh3YZ++G9qF+kDIWqdo0dNm8n586FNG1i2DJo2tTBDmzdD+/amELF5s4UZEcJ3UoCwj315\n+1iTuYauTbuSHO85UsQF/72A77Z9R0xkDN2adaPnST3peVJPhrQfQoO6Dar1nk6nGT3n1Vfhs89K\nRq0+91wzQVi/fjX5j4Qwtm41nar/+9+SY6xnTzMU7F//CnHVHHTpcOFhoiOjqRPlOfV5n7f7sGzX\nMsDUyJ3R/Ax6NO/BDd1uoHli8+r+K7WG9IHwAzu0SQs2q2LidJohWufPh8aNYcECiwsP7kyB9IGo\ngMTEk8TE/oLxGX7+++fcNu82Brw3gOYvNKfJ800Y9P4gvt32rdf0T57/JMtvWk7OhBxWjFnB1MFT\nueb0a3wuPGhtanfvuw9SUkwtw+zZ5rkrrjAFikWLvBce5Nj2JDHxrnRc2rQxtRAbN5pRFZOSYMUK\nM3JTkyamVmLePCgq8u09EmMTvRYeAPqe3JdzWp5D3ai6bM/azqcbPmXidxOPz5Bd3pKdS9iwbwMF\nxwp8y4QP7HCsSB8IYTvHjplZTWfMgLp1zcmkfXurc4X0gRBC+ORw4WF2ZO9gR9YOdmTv4IzmZ9Cz\nped4p99t+45pq6YdX0+ISeD0pqdX+IOo98m9q52nnBz49lv46itzY2b37pLnWrUyP+RuuQVOOqna\nbyHECbVta/rWPPmkmXzwrbdMQeLDD82SmGgmpBs8GC6+2IzkVF2TB00G4JjzGL/v/51Vf65iTeYa\n2jbwnJdCa82lH15KdmE2CkXLei1p17AdbRu0ZcrFU0iIsaoNdfBJEyZhK0VFcPXVZsjW+HhTeOjf\n3+pcuaxbB126QKdO5rEQNiJNmPxHa012YTZO7fTapvql5S8xadEkDhUcKrN94rkTeWrAUx7pF21f\nRFpGGqc0PoUOjTqQmpRKhKp5AwKnE7ZvN7UMS5fCkiWQllZyLwTMtDZXXAFXXQW9eknH6Cr78ksz\nk15lw1BlZZnAX3JJ8PJlY1u3wsyZZl6RtWvLPte6NfTta5aePeHUUyHG+1QlNZJblMtlMy9jy6Et\n7MjacXzY49jIWPIn5nt8L4udxVz60aW0SGhBy3otObn+ybSs15KU+il0Su7k/wwGgPSBELZ35Ii5\nkH35JdSvb5ovWTJca0V+/RVOP90UIn791ercCOGTQBYglFINgLeBQcA+4BGttddxQpVSk4GbAA28\nrbV+qIJ0Qbs2OLWTrIIstNY0imvk8fzCLQuZtmoa6YfTSc9NJyM3g4JjBYzvOZ4pF0/xSP/ayte4\n/avbqRNVh5T6KaQmpdKqfisu7XCp106eNaW1medy82bYsMGMkLRmjfkRdrhcK42oKFNQGDzYLF27\nmmE3hY+yssxEB08/zbqb93Bg/gE6z+xMo0saeTwvY936budOU0M2f76pMSt/HEdHm0JE165m6dLF\njOqUkmKe84ejxUfZkb2DzQc3sz9/P6NOH+WRJv1wOi3+3cJje4M6DTj40EGP7XlFefzj+3/QJL7J\n8aVxXGMaxzWmXcMTDEsVIFKA8AOHw0H/kLndHRqCFZPdu2HECHOnrFEjM4tlj5oPJuJfa9ZAt27Q\ntSuOF1+UY6Uc+f54CqWYBLgA4S4s3Aj0AL4EztFabyiX7hZgPDDAtekb4CWt9Rte9lnla0NuUS5/\nHv6TvKI8DhcdJqcwh5zCHFrWa8l5rc7zSP/lpi95fNHj5BTmcPDIQQ4eOYhTOxnTfQxvDn2zTFqH\nw8G2+tu4cc6NZbYnxiRyY/cbefHiFz32n12QTcGxAprEN0HV8Ja+1nDgAKSnmyUjo+Tx9u2wZYu5\nc5uf7/31TZua01afPiV3b6vbUbW0UDq2LeMqJBT/40kOFhbz4f99yOjrR9NAKSk8lFLTY6W42Nyz\nW7IEfvgBVq82x72300NkpGmK17at6W/RogU0b26WZs3M36ZNTUHaH/KP5uPY7mB3zm52Ze9i92Hz\nNyYyhq+u+coj/ZaDW2j3cjvYBrQu2Z5SP4Ud43d4pN+bt5cxc8bQoG4DGtRpQL3YeiTEJNAsoRmj\nu472SH/MeYysgiwSYhKIjYyt0vmnomuD9IEQIW/ZMrjsMjPfQ+vWptlSp1Cs+XMPGSG364Q4TikV\nB1wGdNJaHwGWKqXmANcCj5RLPhp4QWud7nrtC8AYwKMAAXDRjIvIP5p/fLmwzYW8NPglj3RzNs7h\nmv+7xmP7lV2u9FqAyC3K5ec/fy6zrX5sfY85FNz6p/Zn5uUzaZ7QnOaJzWme0Jz4mHivabWGWOrj\nPFqf9HTzw/7IkbKLe1tenvkNWnrJzi67vm+fGZHuRBo2NHdg27cvuSvbtWsIDD5RmyUlwdNPs+XS\nS/l8504idu/mzSkvMDwlhQ7z5knhwU8iI00huFs3uOMOsy031xQq3LVtGzaYQsXu3aZAXdlI60qZ\nwVkaNDBLUlLZx0lJpgl1XFzZxdu22Ng4Lmw9hMjIqjX/qxdbj38N+hc/L/uZOu3qsDdvL/vy99Es\nwfsM5pm5mczdNNdje8fGHb0WIDYf3Mypr5xq4qYiiYuOIy46ji5NuvDN6G9OnMHScQqVu/sVCaUa\nCBFcWsMbb5hxoI8eNSOAzJplaiBC0s8/w1lnmQkpfv75xOmFCCGBqoFQSnUDlmqt40ttuw84T2s9\nrFzaLGCQ1nqla/0M4DuttcfsZkopfUvTT9hWt4ivG+WAjqBVlmLgVkV9YkmiDlvIQ6PYk6D46aQI\n2uRHMyAnnggNeyKL+V0V0u6gQqNwEkEjomhMJGmRx8iNjiCyWBHlVLQojqAz0WhgL7AGxTGiOEo0\nx4iiMRE0B5aSwDGijj/XgmP04TBOIthKAv+jOccoaT/RhlzakcvXlP1x0IZcBpEJwDbivT5f+nX1\nyaI56XQlnXPII4E8Ikgnnh9pyxbasoUkssl1vbIZX5fZXy5tyGQQAPFs8/q8vK56r4NtfMrXPFjq\n+cnAzYDMOBB8BcSyjdZsoS3baE06zcmgGek0P77sIxkdgEFKoykimqNel5hyz0VSTATOEy7FEU4O\nxDk5Fqk5GumkONJsi3U6OW1f2bQKzcE68FUHzdFIjTPC/dta0+iI5or1YM6GZrv78X+4S2oghH2k\np8PNN5v+DgDjx8O//uW/asWAkFGYhPAmAcguty0bSKxC2mzXNq+2Z75FEW1gexMgiR10Yzr9OY99\ntGEe79Ea6A+5wEYHmiyGunb3HttIozVp9HftzcHpZPFX2rCguAsUO1zbzf568AEASQzkO7oADqAI\n6M157MPJPDa638+1v8ZkMZwk1/ut4Jjr+RgKiWYhzTjEIDqwmwMc4wdiKaQ5HTmNY7TkDwD60Ypz\nWcU+1pFALucQSz1asJYYbmImfyGXuhTgALI4nSReAhLZxv9owieccTw3kEUC7ehDM76m5L+DI7Rk\nLh0BGEgLr8/Pp0W5/87sL4krAdhWEm15v1Lv140WdC/3fA/gA+C0cvtD1oOwXsip/E4mv3MacKeX\n9EeJYg6J5JJAe1qSRRJLKSKXBJLpQjb1+YPdFFCHepxBPnHsYROF1CGKvuQRzyHSKKAOTi7gKNE4\n+YGjwFF//0fO/ub85uX5Nd7SF/SHtZ7pD+Bg2vF1B/Cu6/lUKiI1ED6QNp2eAhGTmTPh9tvNDKdJ\nSWbioquu8utbBMby5aZXd69eOJ59Vo6VcuT74ymUYhLgGoglWuuEUtvuBfpVUAMxUGv9s2u9B/B9\nRTUQHw3fwdEmdcnrnoxSHF9i9+YTk57HkTPM9ogIsz06I5+6q/ejFBQ3r0vR2WWfj0zPJ2p3HsXn\nlN1fZHo+Ucv3gwJOqos6N5noaHNDY+1aB2c2Oxu1I4/4i5KJiuL4c86d+eR/vZ/ISIjrUJcmlyUT\nGVnSyjF/Uz556/NIHl52Mrj8Tfns/2I/AHXb1/X6fKi/zuFwcHaLs0M+n4F+nbO5k1mPnMeDu3bh\nwPw8m3zyyYxds4YGDao3qV9tE0rnwUBxOk1LCm9LUZHntlWrHHTt2h+nE58Wrb1vd7ew1rpkKb1e\n0WOt4e67pQ+ECHEbNpiJYxYsMOsXXQTTp9tovHHpAyGEN5uAKKVUW631Fte2roC3sY7XuZ5ztwHs\nVkE6AK78LKWCZ+Jci7ftFb3mBK+72fvrjh6Fs/pX8LqWcdC74veL6xBHXAfP18V1iCPlAXldrXhd\nVhbDp6UwGYjcvZsVLVsyIiXFdKQWYSMiAmJjzVIVRUVwnmf3LEvcfbf37VIDISx38CA8/ripaSgu\nNkO0/vOfZrIiW51jf/jBfOP79jWPhbCRAI/C9CFmWNabge7APKB3BaMw3QXHG5B/jRmFqezQR8i1\nQdhAqaFaDzqdrF+/ns6dO8soTMJWKro2yK1SYZnMTJgwAVJT4eWXTVXZLbfAH3/ArbfarPAA0gdC\niIqNw9yi34tp/n2r1nqDUqqvUirHnUhr/TowF/gV01J3rrfCgxAhr9w8Dw0bNqRv376m2ZJrdCYm\nTjTphLAhKUD4wOFwWJ2FkFOdmPz+uxlZKTUVJk82E8AMGgS//ALTpkFy8gl3EZpKNWGSY8WTxMRT\nuMREa31Iaz1Ca52gtU7VWs90bV+ita5XLu0ErXUjrXVjrfXD1uS46sLlM/RV2Mdl6VKPGoYyMXEX\nIpYuDX7eQkzYHyte2CEm0gdCBMWRIzB7Nrz5ZtnWPcOGwcMPm8mLbM9dAyF9IIQQIrxdcsmJ0yQl\nVS2dECFI+kCIgMnPN9PMf/KJmfwtL89sj4+Hq682tRCnnWZtHv3q669Nz+8LL4T//c/q3Ajhk0D2\ngQgEuTYIIUTgyUzUIuC0hvXrze/ohQvB4TA1D25nnWXmdrjySkj0NgK83ckoTEIIIYQIA/JLxwd2\naJMWTIcOwXPPOXjiCVML26QJdOlihmKdP98UHnr2NBPAbd0KP/1kChC1svAAZZowybHiSWLiSWJi\nf/IZeidx8SQx8U7i4skOMfFLDYRSahxwPWZixQ+11jeeIP09wINAHeBT4Dat9VF/5EX436FDsHGj\nqV3YsKFk2brVM22zZqZD9IUXwsCBZj1sSB8IIYQQQoQBv/SBUEoNB5zARUDdygoQSqmLMHNknw+k\nA58DP2qtH6kgvbRzDZAjR2DfPti7t2TJzISdO2HHDrPs3Ak5Od5fHxsLPXqYWoaePaFXL2jVyobD\nr/rLF1/A8OGmZ/jnn1udGyF8In0ghBBClBfQPhBa689db3IWcKJ5g0cD07XWv7te8yRmXHCvBYhw\nprVpVl9YCAUFZin9uKJtubnmR39FS3a2KTjk5lYtH3Fx0KEDnHpq2aV9e4iJCWwMbEX6QAghhBAi\nDFjRibozptbBbQ3QRCnVQGt9yNsLrkhZAZhpTN2FIA1oXI+1wn0fSqNOnEarku1l9lk+Tdl9ZhX9\nRP2Ys6v8Pk4dQbGO4Jj7rzOS4ipsM48jcerA/hCNjjhGk9hsmtTJITn2ME3qZNMkNoeU+AOkxO2n\nVbxZGsbkltQqbHAtLo7MTPo3bRrQfNrGzp3mr6sPRP/+/S3NTqiRmHiSmNiffIbeSVw8SUy8k7h4\nskNMrChAJADZpdazAQUkAl4LELN3vQakutaSgG5Af9e6w/U3GOuZriVY7weKb4nmKPGcTR0K0DiI\n5iiNOJ06FHCE5cRQRAtOoQ4FHCKNOI5wKk2pRw4ZbCCefHpSl3rksIk9xJPPpeRSz5nDoiPAkern\nNi2o0bDJursmgpKOUO4TgazLeun1tLQ0y97f4XDw7rvvApCamooQQghRVSfsA6GU+h7oB3hLuFRr\nfV6ptE8CJ52gD0Qa8JTWerZrvSGwD2jsrQZCKaVn3vkDSpm3V5S0sVfKfd/fbHPf+1eqVBp0qfRV\nSVO1fVblfZSCqEgnkUoTFamJjDCLe5t5XPG2yAgdvv0J7ComxswFUWuHmhK1lfSBEEIIUV61+0Bo\nrc/3c17WAV2B2a71bkBmRc2XAP42ta+fsyCEEEIIIYSoDr80sldKRSql6gCRQJRSKlYpFVlB8v8C\nNymlTlVKNQAmAu/4Ix+BZodxeYNNYuKdxMWTxMSTxMT+5DP0TuLiSWLincTFkx1i4q9eun8H8oGH\ngGtcjycCKKVOVkrlKKVaAmit/wc8B3wPbHMtj/spH0IIIYQQQogA8ss8EIEk7VyFECLwpA+EEEKI\n8iq6NsiA9UIIIYQQQogqkwKED+zQJi3YJCbeSVw8SUw8SUzsTz5D7yQuniQm3klcPNkhJlKAEEII\nIYQQQlSZ9IEQQgghfSCEEEJ4kD4QQgghhBBCiBqTAoQP7NAmLdgkJt5JXDxJTDxJTOxPPkPvJC6e\nJCbeSVw82SEmUoAQQgghhBBCVJn0gRBCCCF9IIQQQniQPhBCCCGEEEKIGpMChA/s0CYt2CQm3klc\nPElMPNX2mCilGiilPlNK5Sqltimlrqok7f1KqV+VUjlKqS1KqfuDmdfqqu2fYXVJXDxJTLyTuHiy\nQ0ykAOGDtLQ0q7MQciQm3klcPElMPIVBTF4FCoBkYBTwmlLq1ErSXwskAYOBO5RSfwt8FmsmDD7D\napG4eJKYeCdx8WSHmEgBwgdZWVlWZyHkSEy8k7h4kph4qs0xUUrFAZcBf9daH9FaLwXmYAoJHrTW\nz2ut07TWTq31JuALoE/wclw9tfkzrAmJiyeJiXcSF092iIkUIIQQQgRCB+CY1npLqW1rgM5VfP25\nwDq/50oIIUSNSQHCB9u3b7c6CyFHYuKdxMWTxMRTLY9JApBdbls2kHiiFyqlJgEKeCcA+fKrWv4Z\nVpvExZPExDuJiyc7xMQWw7hanQchhAgHvgzjqpT6HugHeDtHLwXuApZqreNLveZeoJ/Welgl+70D\nuAfoq7VOrySdXBuEECIIvF0boqzIiC/sNC65EEKEC631+ZU97+oDEamUaluqGVNXKmmWpJS6EXgQ\nOLeywoPr/eXaIIQQFgn5GgghhBD2pJT6EFNDcTPQHZgH9NZab/CS9hrgeaC/1npjUDMqhBDCJ9IH\nQgghRKCMA+KAvcAHwK3uwoNSqq9SKqdU2ieBhsBKpdRh13wQrwY9x0IIIU5IaiCEEEIIIYQQVSY1\nEEIIIYQQQogqkwJENSml2iuljiil/mt1XqymlIpRSr2llNqulMpWSq1SSl1sdb6soJRqoJT6TCmV\nq5TappS6yuo8WUmOjcrJeaT2kc/UkO9+CbkueJLjo3J2OI9IAaL6/gP8ZHUmQkQUsBMzckp94FFg\nllIqxdpsWeJVoABIBkYBrymlTrU2S5aSY6Nych6pfeQzNeS7X0KuC57k+KhcyJ9HpABRDUqpK4FD\nwLdW5yUUaK3ztdZPaK13uda/BLYBZ1ibs+ByDVt5GfB3rfURrfVSYA5wrbU5s44cGxWT80jtI59p\nCfnuG3Jd8E6Oj4rZ5TwiBQgfKaXqAZOA+zAzpYpylFJNgfZUMt57LdUBOFZqzHuANUBni/ITcsL4\n2ChDziO1j3ymlQvj775cF6ogjI+PMux0HpEChO+eAN7UWu+xOiOhSCkVBcwA3tVab7I6P0GWAGSX\n25YNJFqQl5AT5sdGeXIeqX3kM61AmH/35bpwAmF+fJRnm/OIFCBKUUp9r5RyKqWKvSyLlVJdgYHA\ni1bnNZhOFJdS6RTmJFAI3GlZhq2TC9Qrt60ecNiCvIQUOTZKKKW6EYbnETuTa4MnuS5UmVwXKiHH\nRwm7XRuirM5AKNFan1/Z80qpu4FWwE7XQZ8ARCqlOmmtzwxGHq1woriUMh1oDAzRWhcHMEuhahMQ\npZRqW6q6uithXiXrEu7HRmn9CMPziJ3JtcGTXBeqTK4LlQv346M0W10bZCI5Hyil6lD2TsIDmA/7\nVq31QWtyFRqUUtOA04GBWut8q/NjFaXUh4AGbga6A/OA3u7Zd8ORHBtlyXmk9pHP1Dv57htyXfBO\njo+y7HYekRoIH2itCzBDsQGglMoFCkLxgw0m17BrYzGxyTQFZzRwi9b6IyvzZoFxwNvAXmA/5osf\nthcJOTY8yXmk9pHP1JN898uQ60I5cnx4stt5RGoghBBCCCGEEFUmnaiFEEIIIYQQVSYFCCGEEEII\nIUSVSQFCCCGEEEIIUWVSgBBCCCGEEEJUmRQghBBCCCGEEFUmBQghhBBCCCFElUkBQgghhBBCCFFl\nUoAQQgghhBBCVNn/A/DkBetHy0ZRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6bff5ea438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.plot(z, np.sign(z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(z, logit(z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, np.tanh(z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, relu(z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Activation functions\", fontsize=14)\n",
    "plt.axis([-5, 5, -1.2, 1.2])\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.plot(z, derivative(np.sign, z), \"r-\", linewidth=2, label=\"Step\")\n",
    "plt.plot(0, 0, \"ro\", markersize=5)\n",
    "plt.plot(0, 0, \"rx\", markersize=10)\n",
    "plt.plot(z, derivative(logit, z), \"g--\", linewidth=2, label=\"Logit\")\n",
    "plt.plot(z, derivative(np.tanh, z), \"b-\", linewidth=2, label=\"Tanh\")\n",
    "plt.plot(z, derivative(relu, z), \"m-.\", linewidth=2, label=\"ReLU\")\n",
    "plt.grid(True)\n",
    "#plt.legend(loc=\"center right\", fontsize=14)\n",
    "plt.title(\"Derivatives\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"activation_functions_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def heaviside(z):\n",
    "    return (z >= 0).astype(z.dtype)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def mlp_xor(x1, x2, activation=heaviside):\n",
    "    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAEOCAYAAAAjY9sgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXu4HGWV7/9ZkstMDAlJjPwAkYCIXCRcDMgYhMBRGRwH\nhOFnCBdFRgdm0GeOkGdwApENGeKg5MxRznAZuSgM7B8j4oA8w8icI1skZ8Q4mOxwU4RsQC4hhmSH\nEMgm8P7+qKqkdqW6u6r6rXrfql6f5+ln7+p+u2r1292rvr3qW6vEGIOiKIqiKIrinne4DkBRFEVR\nFEUJUGGmKIqiKIriCSrMFEVRFEVRPEGFmaIoiqIoiieoMFMURVEURfEEFWaKoiiKoiieoMKsJERk\nlYicX8F2LhGRwQq2IyJynYj8XkTeEpGjyt5mh3huEpG7HWx3DxF5W0QOrXrbKbFkngMROTp836a2\nGXOKiLxtL0KliWhuKz0eJ7mtEz7FlSUWEfmRiNxYVUw2Ee1jBiJyMPBL4D+NMR/N+dxLgFOMMQcm\n7p8GvGaMecNSjHsAq4BZxpiHY/dPAMYbY9bZ2E6b7f8J8APg6DCOV4wxW8rcZrjdo4H7gXcZY16J\n3b8jwed3Q9kxJOLZA3gaOCz+PrggzxyIyBhgqjHm5TZj/gz4F2PMDhbDVByiuS3T9jW3ZcCnuLLE\nIiI/AtYYY86uLjI7jHEdgCd8EfhH4LMi8gFjzK9zPn87dWuMWWslsm1Ii+1sAjZZ3lYa7wdeNMY8\nVMG24kSvW+J3GmNerTiOONJ5SPnkmYNwR9NSlCmNRXNbZzS3ZcCnuHyKpRSMMT19A/4AWAd8EPgO\n8M2UMbsAtwK/B14DHib4dfU54G3grdjfz4bPWQWcH/5/G3BHYp0CPAv8dbh8HPAA8AqwFvh3YN/Y\n+Ph23gZ+Et7fB6xMrHdhuO43gEHghNjje4TPPxm4L3w9jwIfazNHNyW2/3R4/wDw7ZSxd8eW7yfY\nMVwOrAFWJ+cYGAssBobCmH8LfCkWa3x+bwyf893EdsYB/xN4CXgd+E9gduzxo8N1HAv8PHzdy4BD\ncn5eMs0fsD9wD7AhfM23ATvHHp8F/Dick2HgZ8ARscezfGaSc31U+LpfBdaH/++feP1TY+M/G875\nRuBu4K+AtxLb/FOCisvrwFPA3wFjXX9v9Zbps6q5TXNb3s/M12Kxvgh8t83rnwDcTJBvXgS+Cvwo\neh2xz8rC8LkbwvfuM8BkoD987m+AjyfiOCp8La+Hr/t/EMs7KbH8YThvUSx/m4ylTjfnAbi+AWcC\nvwr/Pzr8cu2Q+PA9SbDjnA3sCXw6HDse+CbwGDAdeDdB6T36QEbJ65MEv/wmxdZ7DDACTA+XTwZO\nAvYiSKT/X7jdMeHjs8Iv38fC7ewU3n8JMBhb71cIdspzgb2BS4EtwMzw8SghPBbG9b7wA70GmNBi\njnYkSJLPhK9zWnj//WRLXuvC5+8NnAK8CcyNjekPv7CfBmaEc3sGQSI+iSBpfSB83Tu22M63gOeB\nPw7H/lP4Jd059t6+TfBlPwrYh2AH8Wgi/reBr7X5vHScP+D/CZcXh9v5IHAX8FDi/T89fHwf4NsE\nO66pOT4zW+cA2CF8/hXhHO4DnAp8IPb634qt/8Ph8lfD9+WLBDvnt2LbO45ANH429r48DnzD9fdW\nb5rb0NxmO7f9GcH3/Y+B9wCHAn/V5vVfG34WjgX2C1/rerYXZr8Hzgnfj28SiK17wnnYC7ieQHyN\nC5+zK8GPxX8MX+8nCcTWN9vEcjXwXPgZ2h/4l/C1qDCr443gl9FXYstPAyfFlr8YvsFTWjx/VPJI\nfCCj5LUDQVL8fOzx7wD3tonrnQRJ5yPhcpR0Dm23feB3wEWJMfcDNyfW84XY47uG932kTTwXEP6a\nTKw3S/JamhhzH/BP4f/vD7f98RbbHSUo0rZDsIPZDJwee/wdBL9OL4ut521iv56Bj4Tr3jV232PA\nX7aZh47zR7DD+I/E86aEY2a1WK8ALwCnZf3MJOZgSvhaPpplHgmqJD9OjPkOo4XZT1M+SycCr5b5\nndSbnRua27b7braIR3Ob2Sp8Hycm3tvE9c4wrv839vgEgh+HSWF2a+K9fxv4h9h9o95/ggrkbxLb\n/hyBoPuDFrG8AZya2M46airMevqsTBHZm+CXYn/s7tsIElbEwQTJobAB1RjzFnA7QYUEERlH8Ovk\nllgse4nIbSLyWxEZJvgFIcB7s24nNETuCvzfxEMPEvyKiLMyFt8L4b/vzrqtnCTPrHohtq1DCBLI\nQBfrfx+BX3Lr6zbGvE14KC82zhB73WEcEosFY8z+xphrMmyz3fx9CDhaRF6NbgS/mk0YKyIyPTwT\n7Ncisp6gzD+d8P3O8pmJE34+vwfcJyL3iMhXROQ9beLfj2B+4iSXPwRclHgdtwF/KCI7t1m34hjN\nbVvj09y27Xmdctv3CQ4JDonI9eFZ2uM6xLUstv5NwCMpYwdjY14jqLDGx60O/0ax7sv2uehBgkO6\ne7eIZSxBxTC+nZUpY2tBr5v/v0Dw6+M5kdF+bhHZzRjzPPaM3v8MLBWRXYA/Ivgg/Wvs8XsISrF/\nQVC23kLw66XVF6MdJsN9b6aMySvU32b7+RmbMi65LVNgW+2IYsj7uqPHisTSbv7eQfB+XsD28xMl\noZsJhNhfExxG2Qz8hNHvd6fPzCiMMWeLyD8QHIo4AbhcRE40xvxHyvAsn+t3EFT/vp/y2JoMz1fc\nobltNJrbOmCM+Z2I7AP8N4JDglcCl4jI4caY13PElSRtjtrFmnoySIf7G0XPVsxEZAcC78xXgYMS\nt0Hg8+HQh4GZbfo/jRCU89tijPkFgXn6tPD2r+EvDMJ17wssNsb8xARnTk1mtHAeCf+23JYJzlR5\nATgy8dCRBGVs26whMA/HOSjnOh4m+Bwe0+Lxjq+boKz/JrHXLSLvINhJPJozHhs8DBwAPGuMeTpx\ney0cMxu4yhjz78aYxwkMu6Pmst1nphXGmJXGmG8aY44h+KX+uRZDHwOOSNz3RymvY9+U1/B0+Ktd\n8RDNbVboydxmjBkxxtxrjLkAOJwgj81uEdeWcEwU1wQCD2G3PMb2ueijBD9en2oTy9Z8JiLvtBSL\nE3pWmAGfAqYB1xtjHovfCErzfx6Ou42gzcC/isiRIjJDRP407EEDwRkse4jIISIyrU3pN1rXFwjM\njP8cu38dgUHyiyLyvnDd1zD6V8XLBMfYjxORd4vIpBbb+CYwX0ROFZH3i8hlBF/qK7NMSk5+Ahwf\nzsc+IrIE2D3PCowxvyWoyFwvIieH83ukiJwRDnmG4FfSn4jIu8IvXHIdmwjm6+9F5HgR2ZfAmPru\n8P6Ijr+sROQJEfmrPK8hhX8k2Pn8i4gcLiJ7isjHwkOXUfy/Ac4Qkf1E5DCCQ06bU9bV6jOTjHuG\niHxdRP5IRN4rIscAMxmdvOOv/9vAx0TkqyKyt4h8kcCgHOcy4DQRuVREDhCRD4jIn4nIFdmnQnGA\n5rbu6bncJiKfE5E/F5EPisgM4GwC8fhkSlyvATcC3xCRY0VkfwJvYauqVh6uBnYVkWtEZF8J+sx9\nneCH7Ha988JYbgCuCPPsAeFybfVNbQO3wNkEp2Wn+Su+D7xXRD4WfjGOJijB301wbLyPbR++HwD/\nBvwfggRzanh/2ofznwnOMlkPbD28ZAK34mcIdqQrgauAi4ntqEMvx5cJkt/ztD6k9W2CBHZFuK4T\ngZONMfHj7VnK4lm4MbzdQOABeBW4s8B6zyRI7N8iOMRxEzAJtnpELiEwhL5EMDdpXEhwJs6NwK8I\nfi0dZ4xZHRuT5XW/n2Cn1o626zHGvEjwK/Mt4F6Cz8xVBAbV6D39PDCRoBXFbQRzOJSy3tTPTAqb\nCM7G+hfg1wRzeAvwjRYxPkSwgz4XWEEgyi4Z9YKMuQ/4E2AO8FB4u5Bgh6L4i+a20WhuC+iU29YT\n5IQHCOb3JIKTRVp93+eHY+8i+IysAP6LIM/liWvUfeG8HE/ggfwVwVmbtwIXtYl9PsHJGHeGsawM\nY6sl2vlfURRFUZSuCCuqzxC00/kH1/HUmV43/yuKoiiKkhMJLve1H/ALgirghQRHAW53GVcTUGGm\nKIqiKEoRziewUGwBlhP0UXyh/VOUTuihTEVRFEVRFE/oZfO/oiiKoiiKV3h/KFNEtKSnKD2IMab2\njSM1fylKb9JN/vJemEFwDq4vXAP8pesgEtiOaeYFMH7BMCPThjyJqFs0nvb4Fg/k7+XpLw+6DiDB\nDWxrZFY2s+d1HtO3EvoOLD+WrPRNgj7HX4e7D/rE1v/7+37LvL60KxHZ41rOyTX+yb7beX/f3Nzb\nufeBk3M/JxM39cHn+4L/+8rZxCjuf6jDgGTv7nzoocycPO86gBRsxzS4BDYvnsy4tTMKrsG3WdJ4\n2uNbPEqZvFjhtpb2B7d2DG2sJpasDP3YdQRwwor7tv6/eih5NST7nMt1ucZvGnq50HaOP+pOjj8q\n2Q7OAi8Obfu/z/7qt+OYD5e6ehVmSiqROFto9MxnRVG6o5M4844rwptD4uKsCs7lutwCrSiliLM4\nfeWuHihVnKkwy8mJrgNIoayYBpfAKbK4gDjzbZY0nvb4Fo9SJp90tN1W4uysvaqNoxOj4vFAnP23\ns3ardJtZxNl7zmp1+c/sWBVnx5+1/X19lC/QShJn3rfLEBHjk8esV7nDLGDRaZdD/6DrUJSe4KDG\nmP9985i5JovvzDsudB3AaN9ZFeT1nRWlNN9ZnL6S17+d5+yIrvKXVsxyssx1AClUEdMpspjhW8bD\nvJkZRvs2SxpPe3yLRymThx1vP1k5G1idPs4VqfE4rJwNhF9PF4c2W7F24BFr27FSOfvVQPvH+7rf\nRFssV85UmCmZGRozkkOcKYqipFM7zxk4P6wJfokzm5TuOYNaiTM9lKnkZsaWcUwfXtNFOw1F6YQe\nyuwF9LBmMfSwZhf0lbz++x9CD2UqlTM0ZqTLdhqKoijZ2ml4h1bOSkMrZwEqzHLioxvHRUzt22n4\nNksaT3t8i0cpE9cesyQP45c4y+R5q1CcDbT4erpsp2HTY5akUK+zTh6zJH35hleNCjOlMINL4LL5\nRdppKIqijMYncZaJHux1Bg2qnvXhrUBTj5liBW2nodhFPWa9ivrOiqG+sy7os7y++0U9Zop78rXT\nUBRFSad2lTNwXjkD9Z11RV/5m8iDCrOc+OjG8SWmre00jn3FdSgJfJmhCI1HcYePHrMkLsVZ4b5q\nJYmzVh6zNKoSZysHghzvjTjL6zFLo6/7VdhChZlilaExI9yz4ON6xqaiKF2hlbNiaOWsC/rK30QW\n1GOmlMLMC2D8gmHtdaYURD1myjZq5ztTz1lp1MJz1qXHTIWZUhoqzpTiqDBTRqPiLD9VizNokEDr\n6+K5av6vFh/dOL7FFMXTvtdZlfg6Q77gWzxKmdTBY5ZGVYc2rV2701I7jTwesyRlHdaMPGZpODm0\nacNjlqQPZ4c2VZgppTK4JDhjc6G5Xc/YVBSlK9R3lp8TVtynvrNu6Ct/E0n0UKZSGdrrTMmOHspU\nWlO7w5rQk4c2G3NYE/IJNPWYVYcBbpwIZ2+E2u8xHDFjyzgmn7lZxVkV7HQpTH7G3vqG94D1l9hb\nX1tUmJWBAW6dCKc3IIepOCtGncTZyguv4bXVL2Qau2799M6DJu0DX/inwvFkFmddCrMxRZ/Yizw4\nHlbsD0tXwJGbXUezjWXAYa6DiNEunqExIwxvGc/0/7WmwpMC6jRDFpn8DHz+vzqPWwXsmWF9NwHr\nu4xJccpD4+Gx/eGhFXCEJznsYeDQAs9b2l+OOBtYDXN2tr9eIDismVOcDSyDORbTxQkr7utanK0c\neIUD50zNNPZcrisszl5b/QLr9nys88BVwCEZVvirQmFso49KDm2qxywjBvjZXnDiqfDAXsGyUoyh\nMSNsXjxZe50pSoUY4KG94NOnBn+bkMPUc1aMpnrOKqGv/E2oMMvIg+PhQ6fCIYfAoXNh6XjXEW3D\np1oQZIsnOmOzGnFWxxmqkCzVMqX2PDQeZoU5bNbcYNkHilTL4izttyvQSquWxckhzmxWy+J0I86y\nVsvilCrOqs5hfeWuXoVZBqJq2ayPBsuHHaVVMxv4005DUZpNVC07LJbDmlI1i6hd9cxSO41ucFE5\na0z1rK+8Vaswy0BULROB5cuDvz5VzXzrQpUnnlHtNEqjzjNUAatcB6CUTVQti+cwX6pmNvuq2RBn\n1vqYZaWDOOumj1kWioizdn3MsmBdnLnKYX2UItCsCjMROU9ElonIGyJyY4exXxGRF0VknYhcLyJj\nbcZii2S1LEKrZnbRXmeKa5qYv2D7allEE6tmUMPKGXhROVPfWRf02V2d7YrZ88Ai4IZ2g0TkOOBv\ngGOAGcD7gEstx2KFeLUM4OCDg78+Vc08cywVjucUWczwLeNLEGdNmaGSUI9ZROPyF4yulsHoHOZD\n1axbj1ka3YizSjxmabQQZ2V5zNLIKs6KeMzSsCbOfMhhffZWZVWYGWP+1RhzN9CpzvlZ4AZjzBPG\nmGGCZPh5m7HYoFW1LEKrZvYZGjNSkjhTlPY0LX9B62pZRFOrZqCVs6Jo5awL+uysxpXH7AAg3jd2\nBfBuEZniKJ5UktUyCPwZEb5UzTxzLHUdTyTO7J2x2bQZsox6zPJSi/wF21fLYPsc5rpqVua1O4uI\ns8o9ZkkS4qxsj1kancRZtx6zJF2Ls4w5bMpOa7rbThb6ul+FqwazE4Hh2PIwQSPqHYF1ycEXA7vF\nnrgv2w4GRZ9Z28uzCKplH94xSGRR+f+3vw3+RstjJ8EdO8PsZ4MXUFY87ZafqHh7VcTDmBE2XzCZ\nsXfdw5YTX/IgIt9mKONylLCiUn/a8ksdHo8vlxYvwC8JjiZ6T6789XfALrEnvp9th+8iUVLGsgF+\ntDOcuOO2WJYvD3JYlL+WLw9y2H/uBR9+fFv/zSrii5afLHn9D/cHy7PnbRNd0eHKtOXl69o/Xsly\n2Ih2YBks//W2w5mRSKti+YQV93H5ulnAtkOXSUEWLScfL7J8Ltdx+cBsAKbN+SAAb770WrChTvmJ\nDo+Hy2++9BqHv30Zv3jH14I7ooufHzKnu2WA5QPw4hA2KOWSTCKyCNjNGHN2i8eXA39njLkjXJ4K\nrAHeZYxZlxjr5JJMPxsPmxYEpf5O/OKnMPHrfl0NoCnMvAC+duUCFslc16HUjz3Oztb5Pys3fQie\naeuJt4i7SzLZzl+uLsn08/Hw1gI4PGMO2+Hr/lwNoCxqdxmnHryEE2y7jNPPz1qYrfN/Rqas2p8j\nvrto63Jp19g8urtLMrk6lPkocFBs+WBgdTKpuaKTtyyJes3KY3AJXDa/7HYaipILr/MXdPaWJWmy\n1yxO7XxnPdjrDKrznR1/1J2VbCcvtttl7CAifwDsAIwRkfEiskPK0JuBPxeR/UJfxkUEV+LzgjRv\nWUTcnxHh2mvmmWPJejyjep0VOimg6TPUJeoxA5qTvyDdWxbRKoe58pqV6TFLo5M4c+4xSzCwGi/E\nWVyg2faYpZFLnHWRw3wUZ7YrZhcDmwgKsKeH/18kIruLyKsi8h4AY8yPgW8A9xNM6SoquQJVZ/JW\nyyK0alY+5bXTUBSgAfkL8lfLInqlagY1rJyBc3EG1VfPduWFSrbjmzgrxWNmk6o9Znm8ZUnUa1YN\nM7aMY/KZm6F/0HUofqMeM+e48Jjl8ZYl6RWvWUTtPGfQU76zBWf9gkf3XG9tfUmPWRJrnrOaesy8\npGi1LEKrZtVgv52GojSDotWyiF6qmoFWzoriwndWBb5UzlSYxWjnLYtI82dEuPKaeeZYqiSeoTEj\nbF48OaM468UZyoF6zBpDO29ZRKccVrXXrGqPWZKl/aMFmpcesySOxdnAMs/EmcUc5oM4U2EW0m21\nLEKrZtUxuIQc4kxRmk231bKIXquaRdSueqaVs9I4/qg7nQo0FWYhWaplsK0xYytcVM08u/JipfFE\n4qx9O41enqEM+HCdOaVrslTLIFsOq7JqVsa1MouytN/htTJb0DYeR+004tfv9EKclZTDXIkzNf+H\n3DgVXt09aN/dLQbY8Tk4u/wzipUYd5gFLDrtcj0pIGKnS2HyM/bWN7wHrL/E3vraoub/vNw6FTZZ\nzGETnoPTezSH6UkBxbB9UsD/uvBRXlj9WsvHX2DXXOt75867cuAVf5k7jtwnBXRp/ldhlpNleFfv\n8C4ml/GkizOdofb4Fg+oMCuPh/GrSuVrPL6Is4HVOap4FYmzgWWjq2Zxqr5SwLWcw9qBR7Zexqks\ncokzPStTUbahvc4URbFB7Txn0JO+syZeJUArZkojmbFlHNOH1zAybch1KEohtGKm+IEvlbNcNPCw\nZiei62uWTabKmVbMFGV78rXTUBRFSUcrZ8XQyllxVJjlxLMOVIB/MfkST3TG5pi7nnMdSgJfZijC\nt3iUMnHdNyxJHeJJ9jqrksJ91UoUZwMZU0ZV4iy6dmeV4qxMgabCTGk0g0vgP279VId2GoqiKJ2p\nXfXMUTuNOC4qZ3WvnqnHTOkZ7jALWCRzXYehZEI9Zoq/qO8sP1V7zsCh70w9ZoqSjVNkcVA50zM2\nFUXpgtpVzsCLypn6zrKhwiwnPrpxfIvJ53j8aKfh8wwpTacOni6XZI2nKnFm9dqdlsRZVo9ZGmWI\ns8hjlkYdxZkKM6XnGBoz4oE4UxSl7mjlrBhaOWuPesyUnkV7nfmMesyU+qCes2I0tteZ/Jl6zBSl\nCNrrTFEUG7hsp1EYrZx5iwqznPjoxvEtpjrFE/U6q7adRp1mSGkadfV0VUU38ZQhzqx6zJIUFGfd\neMyS2BBn7TxmSapsp1EUFWZKzzO4BC6bv1h7nSmlUMvDXEphalk58+CMzarxWZypx0xRYtxhFrDo\ntMuhf9B1KD1OczxmJhRmtdthK11RS0GuvjMrqMdMUSziRzsNpYnUcketFKaWQlx9Z15gVZiJyBQR\n+aGIbBSRVSKSmopEZJyIXCsiL4nI70XkLhHZxWYsZeGjG8e3mOoeT/ntNOo+Q82kivzlgzhrkqer\nDGzGY0OcleoxSyODOLPpMUsjrzjL4zFLwzdxZrtidjXwBjAdOAO4RkT2Sxn334EPAx8EdgWGgass\nx6IohYnEmZ6x2VNUkr9mz/NDoCnVoJWzYvRy5cyax0xEJgDrgP2NMU+F990M/M4YsyAx9mpggzHm\nq+HyJ4ElxpjtkqB6zBSXzLwAxi8Y1l5nlVOtx6zM/GXaiLBa7rSVwtROkKvnrBA+ecz2AbZESS1k\nBXBAytgbgCNFZJcwIZ4O/JvFWBTFClE7Da2cNR4n+at2O2qlK2onxHu0cua6emZTmE0kKOnHGQZ2\nTBn7G+BZ4HlgPbAvsMhiLKXhoxvHt5iaFo/9XmdNm6FG4Cx/VS3OmuzpskHZ8eQVZ5V7zJKktNMo\n22OWpJM469ZjloZLcTbG4ro2ApMS900CXk0Zey0wHpgCbCIomP47cETaii8Gdgv/n0iQBQ8Ll6PP\nR1XLT1S8vSzLT2g8pcfDEjhlyWKG7n+GWxZ9AX4y1XFEvs1Q1zMM/JJA6zihtPx11n/CjInB/zuN\nhYOnwJydg+VopztnXrDDjkTBoeHfMpafLHn9Gk/nZfoDQb71/U9+HmLLy9e1f7yy5Stg4FhGEQm0\nOYeVv3zCivsYWAZL957FgXOC/JsUZNFy8vGiy7MHvs7d/CnT5nwQgLUDjwBstwzwysCjbBp6GRvY\n9pi9AhwQ82h8D3g+xaOxElhgjPlRuDyZwN/xLmPMK4mx6jFTvEJ7nVWBE49ZKfmrnccsSe0OdSld\nUctD2eo764g3HjNjzCbgTuAyEZkgIrOBE4BbUoYvAz4rIpNEZCxwHkECtF+PVBTLaK+z5uFL/qrl\njlopTC2FeI/6zqrEdruM84AJwMvArcC5xpjHReRIEdkQGzcf2ExQNV4N/DFwkuVYSsFHN45vMfVC\nPN210+iFGaolXuSvsttp9JqnKy9Vx9NJnDn3mKUwcL7rCEaLszI8ZkmqFGdWhZkxZp0x5iRjzERj\nzAxjAre0MeZBY8yk2LhXjDFnGGN2NsZMNcYcZYz5pc1YFKVshsaM6BmbDcK3/KXVs95BK2fFaGrl\nTK+VqShdor3OyqB518osSi132kphaifI1XO2Hd54zBSlV7HfTkNRtlG7HbXSFbUT4intNKqmab3O\nVJjlxEc3jm8x9WI8g0uCkwKyibNenCGlG2yKs173dHXCh3ji4sxLj1laTA7F2cCy6sUZlHdoU4WZ\nolhkqzjTMzYVy2jlrLeoXeUMvKicNcF3ph4zRSmBGVvGMfnMzdrrrDDqMWtFLXfYSmFqKch73Hem\nHjNF8ZConYZWzhTblN1OQ/GLWgpxPWOzK1SY5cRHN45vMWk8Aa17nekMKd1TVJz54KGKo/G052H8\nE2eZfG8VirNW1+6sqzhTYaYoJaK9zpQy0cpZ77C03z+B1hGtnBVCPWaKUgEzL4CvXbmARTLXdSg1\nQT1meajdDlvpitoJ8h7znJ0o96nHTFF8Z3AJXDY/azsNRclH7XbUSlfUToj3YK+zblBhlhMf3Ti+\nxaTxpBP1Ojvz/m94dlKALzOkdENWceajh8on6hKPS3FWuLdaSeKslccsiYt2GkVQYaYoFXP0Mbfq\nGZtKKWjlrLeoXeUMnFfOwP/qmXrMFMUR2uusHeox64Za7rCVwtRSkDfYd9YTHrMZW8a5DkFRrNO6\nnYaidIf2OustainEtXLWkloIs8lnbuYOs8B1GICfbhzfYtJ42hOPx492Gr7NkGKLNHFWFw+VK+oa\nT5XtNKxdv9OSOMvqMUvDR3FWC2FG/yCLZC53mAXMvMB1MIpil8EleCDOlKailbPeonbVM62cbUct\nPGawzWW20NzOZfMXM7jEYVCKUgLa6yyOesxsU7sdttIVtRTkjn1ntjxn3XrMaifMAMatncHmxZNV\nnCmN5A6zgEWnXd7jJwWoMCsDFWe9hYqzYnQr0HrC/J9kZNoQ4xcMOzkpwEc3jm8xaTzt6RTPKbKY\nhbddVGEDLosjAAAgAElEQVQ7Dd9mSCmL2fPq66GqiibFU5YQt+YxS6PAoc1uPGZpuD60WUthBoE4\nm3zmZj1jU2kkp8hi7XWmlMKBx7qOQKmSWlZJe9x3VstDmaOYN5PhW8YzNGakuqAUpSJmbBnH9OE1\njEwbch1KxeihzCqo5U5bKYQe1ixGkcOaPXkocxT9g16101AUm/jRTkNpKrXcWSuFqKUI79HKmVVh\nJiJTROSHIrJRRFaJSMuvvYgcKiI/FZFXReRFEfly4Q3H2mmUjY9uHN9i0njakzee8ttp+DZDbnCW\nvyom6Q9yLc6a5OkqA5vx2Op1VqrHLEkGcWbbY5akanFmu2J2NfAGMB04A7hGRPZLDhKRacC9wDXA\nFGBvoOtXrr3OlKYSibOF5nbXoTQZp/nLJa7FmVIttaueXYHz6lmV4syax0xEJgDrgP2NMU+F990M\n/M6Y0aUsEbkceI8x5nMZ1tveY5aCttNQmswdphd6nVXrMSszf/nsMUtSux220hW1FOQ16HXmk8ds\nH2BLlNRCVgAHpIw9AlgnIktFZLWI3CUiu9sKJGqnoZUzpYmcIouDypmesWkTb/KXS2q5o1YKU0sh\n7kHlrOzqmU1hNhEYTtw3DOyYMvY9wGeBLwO7A0OA1Y9IWb3OfHTj+BaTxtMeG/HYbafh2ww5obz8\n5cGZZXE6+YOqFmdN9nTZoOx4ioizSj1maSTEWdkeszTKFGdjLK5rIzApcd8k4NWUsa8DPzTGPAwg\nIpcCvxeRHY0xKeMvBnYL/58I7AscFi5H78j2yyPThph87I+5539/nN0/tqXD6GzLT3T5/DKWn9B4\nejKeoTEj3PO/x/Kp1f8BP5nqQUTdLAP8EngeR5SWv85aCDN2BZbCTmPh4CkwZ+fgsWgHV+Xy8nUZ\nxofi7Kpwp31o+FoikWBz+cmS16/xdF6mPxDkWT9PES4+v1uXr4CBRE++SKDNOaya5UnX38fSvWcB\n8MjAK6weeh0b2PaYvQIcEPNofA94PsWjcTMwYoz5Qrg8FVgDTDHGbEiMze0x2455M1l420WcIou7\nW4+ieEgze5058ZiVkr/M8tgdHpz+n5daHu5SClHLQ9keVKSTvjNvPGbGmE3AncBlIjJBRGYDJwC3\npAy/CThJRGaKyFhgIfBgMqlZo8J2GopSNdrrrHsqy18e7ETyUsudtVIIW+00KsWDHzu2D2vabpdx\nHjABeBm4FTjXGPO4iBwpIluTljHmfmAB8G/AS8BewGmWY9kOG+00fHTj+BaTxtOeMuLprp2GbzPk\njGryl2NxVsQfVKY46zVPV15cxNNJnDn3mCUYON91BHbFmVVhZoxZZ4w5yRgz0Rgzw5hgL2GMedAY\nMykx9jpjzHuMMdOMMScaYyoxlyySuXrGptJIBpfAZfMXa6+zglSav7RypnhOLStnHpyxaYP6Xyuz\nINrrTGkyd5gFLDrtcugfdB1KQRp0rczlbQZ4cBgmL7XbYStdUUtB7viHjxxMV/mrZ4UZBOJszeTp\negF0pZHM2DKOyWdurqk46xFhFlEzgabirLdQcZaPboVZ/S9i3gUj04aYfObmXL3OfHTj+BaTxtOe\nquIZGjOSsdeZbzPUg1S4E7HhD7K5o1ZPV3t8iCcpxL3zmKXFU7MfO3F6WpgB0D+YW5wpSl2IxJme\nsVkDauY7mz2vppUUpRC1rJLWVJz19KHMUWivM6XBzLwAxi8YrlGvsx47lBmnhjuTWu60lcLUTpBX\n/KNHD2XaItbrTM/YVJpG1E5DK2c1oGaVM6jhjlrpitoJ8Zr92FFhlqBTOw0f3Ti+xaTxtMdVPK17\nnfk2Q0qZ4qwsf1BRceaDhyqOxtOZh/FLnGX6THvQTiMrKsxSiC6ArpUzpWkMLgkugL7Q3G7pAuhK\naWjlTPEcn8RZZmogztRj1gZtp6E0Gb97nfWwxyyNGuxM4tRyh60UppaCvMQfPuoxK5Ei7TQUpS6c\nIoszttNQnFOz6lktd9RKYWopxD3+saPCrBOJdho+unF8i0njaY9P8QyNGeGePx+rJwXUAUvirKoe\nVFnbafjmodJ4OpMWk0txVvgz7ak4U2GWhVCc3WEWuI5EUazz0se26BmbdaFmlTPQ6lkvoZUzO6jH\nLCcLze3a60xpJH71OlOPWVs83Jl0opY7baUwtRPkFn/0qMesYrTXmdJUWrfTULxDK2eK59ROiHvU\nTkOFWW6Wdex1VjU+eZZA4+mEz/GMaqeh+E1BcebyOodp4sw3D5XG05msMVUlzqx+pj0QZyrMCqK9\nzpQmo73OaoJWzhTPqV3lDJyLM/WYdYn2OlOazIwt45h85mYHvc7UY5YbD37p56GWO2ylMLUU5AV/\n+KjHzDHa60xpMkNjRrTXWV2oWfUsazsNpRnUUog7+rGjwiw3KQ4hx+00fPYs+YDG055O8UTiTNtp\n1IAM4sylxyyNN491HcFofPN0+RYPFI+pLHFW6mfagThTYWaL/sGtZ2wqStMYGjOivc7qQs0qZ6CV\ns15iaX8Nq2cVizP1mJXAQnM7l81fzOAS15Eoil1mXgBfu3IBi2RuyVtSj1nX1MxzBjXcYStdUTtB\nnvFHT7ceMxVmJTFu7Qw2L56s4kxpHNWIMxVmVlBxpnhO7cQZdBRoXpn/RWSKiPxQRDaKyCoRaTvl\nIjJWRJ4QkWdtxlEu2RxCVbbTqJtnqWo0nvbkjWdUr7MGnRTQyPyVsgPxzWOWjMf1jto3T5dv8YDd\nmGwI8co/0yX/4LHtMbsaeAOYDpwBXCMi+7UZ/zfAS5Zj8IZInOkZm0oTOUUWN+2MzVLy190HfcJO\ndEW5kNr5zlyLM6VaalklLVGcWTuUKSITgHXA/saYp8L7bgZ+Z8z2jngR2RO4Bzgf+I4x5r0t1lvL\nQ5mjmDeT4VvGa68zpZGU0+us2kOZZeavu0wgzE5YcV9J0eegZoc2a7nDVgpTS0Ge8qPHp0OZ+wBb\noqQWsgI4oMX4bwN/S/ALtdmE7TS0cqY0kYa00yg9fzmvnEEtK2e13FkrhailEC/hx45NYTYRGE7c\nNwzsmBwoIicBOxhj7ra4/Yoo6BAqsddZ3T1LZaPxtMdGPA1op1FJ/vJBnA141jcsiz+oSnHmm6fL\nt3ig3JiKtNNw7pu0LM7GWFzXRmBS4r5JwKvxO8JDBlcAx0d3dV71xcBu4f8TgX2Bw8LlaLdS1fIT\nxZ/fP8ii/r0Yuv90br7nVgaX2InuCYuvTuPReIqub3AJ/Pi5yXz8rnvYcuLuOdcA8EvgeRxRWv76\nn2etZOcZfwjAhJ3GsPLgWVw05ZcADIQvf044HVUsL/81zLkweBXRDm3OzuHjDpaXr8s2fvY8uCrc\nYR8a/NkqEGwuP1ny+useT5wyt7e0f1vz4U6fjwhXn2eAgTNgaE+sYNtj9gpwQMyj8T3g+bhHQ0QO\nAn4BrCVIauOAycDLwBHGmGcT662/xywFbaehNBU77TSceMxKyV+RxyyJes6KUcvDXUph6ngoW/o9\n6mMmIrcBBvgicAiBOfYjxpjHY2PeAbwr9rTZwFXh+N+bREBNFWag4kxpNneYBSw67fKCJwVU38es\nrPzVSphFOBdoKs4Uz6mbOOtWmNlul3EeMIHg1+OtwLnGmMdF5EgR2QBgjHnbGPNydCP4lfq2MWZN\nMqn5iT2HkK12Gk30LNlE42lPWfGcIotZeNtFdWqn4SR/Ve07G0i+4Y7baRTxB5W5o/bN0+VbPFB9\nTJ2EuHOPmWWsCjNjzDpjzEnGmInGmBnGmNvD+x80xiT9G9FzftrqVPNeYGTakJ6xqTSWOvU6c5m/\nfDgpoI5nbCq9Qy9VSfWSTL6gvc6UBjNjyzimD69hZNpQxmc055JMnQ5lxnF+WBP00KbiNXUQ5L4d\nylSKUmI7DUVxTQPaaVSCVs6KUYedtWKHXhDhKsxyU6JDqH+QRTI3tzjrFc9SUTSe9lQVz+ASVJxl\noGxxtp3HLI0KxZktf5Atceabp8u3eMB9TMleZ+oxU0onEmdVXABdUaokEmcLA/uW0gKtnBVDK2e9\nRVOrZ+ox8xhtp6E0mTtMu15nvekxS8O570w9Z4rn+CbI1WPWYKJ2Glo5U5rIKbI4qJzV4IxNlziv\nnmnlTPGcpglxFWa5qdYhlKXXWa96lrKi8bTHZTx1aqfhEpviLJPHLEmJvc7K8gcVFWeu/VNJfIsH\n/IvpYZolzlSY1QDtdaY0maExIyrOMuC8cga1q57NnqfVs16iKeJMPWZ1Yt5MFt52EafIYteRKIp1\nRvc6U49ZK5x7zkB9Z4rXuBbj6jHrJQq201CUOqC9zrKhlbNiuN5ZK9WRbKdRN1SY5ca9QyjZTsN9\nRKPReNqj8bQmaqehtKcbcVbIY5aGJXFWZQ+qLOLMR/+Ub/gWU6t46irOVJjVlEUyV8/YVBqJtofJ\nxt0HfcJ99UwrZ4rn1FGcqces5mivM6WJHER3Hg1fKMNjloZz35l6zhTPqVKQq8esx8nSTkNRlGbj\nReWsZtUzrZz1FnUS4irMcuOTIydgZNr3vWqn4dsMaTzt8S0epRhZxZk1j1kaBcSZy+scpomzuvin\nXOJbTFnjqYs4U2HWFPoHvRJniqJUj/PKGdSycqbVs96hDuJMPWZNQ3udKQ2gSR6z480POJfrKt2u\nc88ZqO9M8Z6yBLl6zJTRxHqd6RmbiuIH13JOpdvTylkxtHLWW/gqxFWY5cZHR872MS2SuXztSjfi\nzLcZ0nja41s8TcUXcVaqxyxJBnHm0mOWxpvHuo5gNL75ucC/mLqJx0dxpsKswWivM0XxCxfizHn1\nTCtniuf4Js7UY9YDjFs7gzWTpzM0ZsR1KIqSiaZ5zJJU7TkD9Z0VwbcdtlIutgS5esyUjoxMG9Iz\nNhXFI67lHG8ObVZKzapnWjnrLXwR4laFmYhMEZEfishGEVklIqkfaxGZLyIrRWSDiDwlIvNtxlEu\nPjpyMsRUYTsN32ZI42mPb/G4wkX+ciHOKvWYpZEQZ755zJLxuG6n4ZufC/yLyWY8Pogz2xWzq4E3\ngOnAGcA1IrJfi7FnAjsBxwNfEpHPWI5FSRKKszvMAteRKIqPOMlfVYuzpXvPqnR7qdSscgZaPesl\nXIszax4zEZkArAP2N8Y8Fd53M/A7Y9orARH5FoAx5q9THlOPWQksNLdrrzPFW6r2mJWZv9I8Zmlo\nr7N64HqnrVRLEUHuk8dsH2BLlNRCVgAHZHjuR4FHLcaidEB7nSnKKJznL/Wc1QOtnPUWLoS4TWE2\nERhO3DcM7NjuSSJyKSDATRZjKRHXBo00isVUVjsN32ZI42mPb/E4wov8VYU4Wznwytb/fWinMeBZ\n37AsnrcqxZlvfi7wL6ay46lanI2xuK6NwKTEfZOAV1s9QUS+RODlONIY82brVV8M7Bb+PxHYFzgs\nXI52K1UtP1Hx9rIsP1H4+SPTvs/Yu+7hTT7F4BLX0fg2OxpPVcsAvwSexxml5a8VZ13FhBnvBmDM\nThOYdPCeTJvzQQDWDjwCMGr5cmZz0ZylwDYRdeCcqdaWn16+YbvHmfMJTlhx39YTA+aEb1AVy8t/\nDXMuBK7YJorm7Bw+7mB5+bps42fPg6vCHfahwZ+tAsHm8pMlr7/IMh0eb2I8S/u3NR9Ofh4ABl6G\noY1YwbbH7BXggJhH43vA82keDRE5G+gDPmqMeabNetVjVgHa60zxCUces1LyV1aPWRrqO/Mf9Zz1\nFlmqpd54zIwxm4A7gctEZIKIzAZOAG5JjhWR04HLgY+3S2pKdWivM6WX8TV/qe/Mf1y301CqpQoh\nbrtdxnnABOBl4FbgXGPM4yJypIhsiI1bBEwFlonIq2E/oKstx1ISPjpyLMVkqZ2GbzOk8bTHt3gc\n4mX+si3O4h6zNKoWZ6l91RyKs6J91coSZ775ucC/mKqOp2xxZlWYGWPWGWNOMsZMNMbMMMbcHt7/\noDFmUmzcXsaY8caYScaYHcO/f2UzFqUg/YNbz9hUlF7C5/yllbN6oJWz3mFpf3kCTa+VqbRkobmd\ny+YvZnCJ60iUXqPp18osinrO6oH6znqLpCD3xmOmNI+y2mkoilIMF5Uz59UzrZwpnmNbiKswy42P\njpzyYhqZNpRbnPk2QxpPe3yLR2lPt+Ksk8csjTLFWaZrd1Yozmxdu9OWOPPNzwX+xeRDPDbFmQoz\npSORONMzNhXFD6qunIEHvrMLqV31TCtnvYUtcaYeMyU782YyfMt47XWmlI56zLKjvjP/Uc9Zb3Ek\n6jFTqiJsp6GVM0XxBz1j03+015mSBxVmufHRkVNhTBl6nfk2QxpPe3yLR8lPHnFWxGOWxKY4y+Qx\nS6MkcWbLY5ZGEXHmg38qiW8x+RZPt6gwU/IT63WmZ2wqSnvufeDkSrajlbN6oJUzpRPqMVO6Ytza\nGWxePFl7nSlWaZLHjJ8GOfb4o+6sZJvqOasH6jtrLuoxU5xSpJ2GovQiTa6cOa+eaeVMaRAqzHLj\noyPHbUzJdhq+zZDG0x7f4mkyPogzGx6zNIqKs8IesySW2mmU6TFLkkWc+eif8i0m3+LpFhVmihVG\npg3pGZuKkoF7Hzi5EoF2Leeo76wGaOVMSaIeM8Uu2utMsUATPWZpqO+sRNR3pjhCPWaKX2Rop6Eo\nSoAPhzbLQCtnxdDqmQIqzArgoyPHs5j6b9jaTsMHPJsdjUcZRdXirCyPWZKs4syaxyyNAuKsSo9Z\nGklx5qN/yreYfIunW1SYKaWhvc4UJRtaOSsRrZwpNUM9ZkrpaK8zJS+94jFL0lTPGXjgO1PPmVIR\n3XrMVJhF7HQpTH7G3vqG94D1l9hbX81RceYnBrhxIpy9EXxSQb0qzKC4OFt54TW8tvqFzON3pf3Y\nXXd+J1+64oBCsbRCxVl+VJy1xwC3ToTTPcph3QqzMRZjqTeTn4HP/1fncauAPTOs7yZgfZcxZWYZ\ncFhVG8vA9vGMTBti/Nph1lwxvfIzNv2fHXc8OB5W7A9LV8CRm11Ho8C2w5p5Bdprq19g3Z6PdR4Y\n5rB1WcZZ5u6DPrGdOBtYBnOq+kJEhzXbCLSB1TBn50qiycSbx8LYn7iOYjQPA4e6DiLkofHw2P7w\n0Ao4oiE5TD1mSmVorzO/MMDP9oITT4UH9gqWFX+oyndWNeo7y8/seeo7S8MAD+0Fnz41+NuUHKbC\nLC9ZqmWV40v9JaJNPA7aadRodirlwfHwoVPhkEPg0LmwdLzriJQkpYgzD3JYXJxVVi1L0kKc+VQt\ng9Hx+CLOfKqWzQpz2Ky5wXITUGGmVE//oFftNHqRqFo266PB8mFHadXMV7RyViI1q5yBP+LMNVG1\n7LBYDmtK1cyqMBORKSLyQxHZKCKrRKTlR0hErhCR34vIGhGpjyWzBN9F9/jWiSpbPFW106jn7JRL\nVC0TgeXLg7+9XjXzOX9ZFWce5bC7D/pEuX3MspAQZ677mCVJi8e1OPOhb1hULYvnsKZUzWxXzK4G\n3gCmA2cA14jIfslBInIOcAJwIDAT+JSI/IXlWJQasEjmMn7BsPY6q5BktSxCq2Z+56+mVs6W7j3L\nffVMK2e1Ilkti2hK1cyaMBORCcDJwMXGmNeNMUuBu4EzU4Z/FlhijHnRGPMisAQ4y1YspeKBP2N7\nfHEtReSLZ2TaUKnirN6zY594tQzg4IODv71cNatL/rIizjzLYQfOmQp4cGgzFGc+e8ySuBJnrj1m\n8WoZjM5hTaia2ayY7QNsMcY8FbtvBZDWCOcARjcnazVO6REicaZnbJZLq2pZRA9XzWqTv6qqnL3A\nrpVsJ44X4qxm1bNeq5y1qpZFNKFqZlOYTQSGE/cNAztmGDsc3uc/HvkztuHapJGkWDxltdNoxuzY\nIVktg8CfEdHDVbNa5a97Hzi5uEDLkcOu5ZzSL+OUvHana3E2sAyvxFkWz1vV4sylxyxZLYPtc1jd\nq2Y2G8xuBCYl7psEvJph7KTwvhZcDOwW/j8R2JdtB4Si3Vy3yyFR0tqzxfJLHR7fLunZiq/d8hMl\nr7/CePpvYPLqHzO85TiGxoy4jsa32elq2QA/2Bn+NCY1li+H3/5226GA5cth7KSgajb7cfhlRfFB\nsK3ncUZ5+WvxWbDLjOD/d+4E7z8YDpkTLP9qIPhbcPneb03l8IMeYNqcDwb3d8pfqwhyWKf8FS6v\nHXgEgGvnnMO5XLdVREWHH20sP718w3aPMydoRBudGBC11Khiefmvw+ULYeD88PHwcGIkkqpcXr4u\n2/jZ84LllT/ZdqgxElC2l+nweJnLP9oZLgmrZXFBFl8+7Ci45nYY93g18QH8CngRO1i7JFPo0XgF\nOCA6HCAi3wOeN2Z0XwQRWQrcaIy5IVw+G/iCMeYjKeut5pJMe5ydrfN/Vm76EDxzo7319RrzZrLw\ntos4RRa7jqQx/Gw8bFoQJK1O/OKnMPHr7q4GUPUlmUrNXzkvyVSE44+6k5+ftTBb5/+MTFm1P0d8\nd9Go+6q+xqbzSziBXsbJI34+Ht5aAIdnzGE7fN3N1QC6vSSTtUOZxphNwJ3AZSIyQURmE5y5dEvK\n8JuB80VkVxHZFTif4CJGihIQ63WmZ2x2TydvWZJe85qVmr/67MebpCrfWdmHNZO4PqwJeHVYMytN\n9J118pYlqbPXzHa7jPOACcDLwK3AucaYx0XkSBHZEA0yxlwH/AhYCQwCPzLGfMdyLOWgHrMM2Itn\nkczla1d2J86aOzvZSfOWRSQPB0DPes3Ky199ZYW8jXXrp2cb2GUOsy3Okh6zJFWLs9S+ag7FWdG+\namWKMxceszRvWUSrHFZXr5lVYWaMWWeMOckYM9EYM8MYc3t4/4PGmEmJsV81xkwzxrzLGPO3NuNQ\nmoX2OuuOvNWyiB6smpWbv/rsx+wKF5Uz59UzrZw5I2+1LKKuVTO9JFNePOsBFOC6M1YS+/F0006j\n+bPTnnbVMthm/E/So1WzcunDvUCzlMNsibPI+J+FKsRZ22t3Omin0W1ftTLEWdV9zNpVy6B9Dqtj\n1UyFmVIbymqn0WSKVssieq1qVhl9rgOwQxXtNJI4r5xB7apnda6cFa2WRdSxaqbCLC/qMctAifH0\nD+YWZz00O9vRqVoG6f6MCK2alUifo+2WkMO6EWedPGZplCnOMl+7syJxZuvanbPn2RNoVXrMOlXL\noHMOq1vVTIWZUj9CcXbH6C4GSoJuq2URWjUrkT7XAdhDK2f1oE7Vs26rZRF1q5qpMMuLeswyUEE8\nsXYaHkSTi6riyVItg9b+jAitmpVMX8XbKzGHFRFneTxmScoQZ209ZmmULM7KuHZnt+KsKo9ZlmoZ\nZMthdaqaWWswWxaVNZjd6VKY/Iy99Q3vAesvsbc+pSULze1cNn8xg0tcR+IXN06FV3cHG11aDbDj\nc3B2/qNOhai6wWxZiIjhmIw5tq/LjV3/F7DhN22HTNlpTebVvXPnXTnwir8sFIo2oq0HvjeivXUq\nbLKYwyY8B6dXkMO6bTCrwiw3y/CvBuNbTNXHM27tDDYvnpwqznR22uNbPNCjwgwqqZ4d/vZl2y7h\nVCJZxdnKgVe6qprFsSHQBpYVqJpFlCDOBlaXUzWLKCLOHqb6MzPb4Vs83nT+VxSXRO00tNeZUmv6\nKF2c/WJFhuvZWKBqzxl44DtTz5liAa2YKY1i3NoZrJk8naExI65DUbqgZytmcfqshpLK8UfdWf5G\n0EObdcD3w5p1QitmihJDe50pjaGv/E3oNTZLpGbVM5vtNJTuUGGWG9+6YoF/MTmOJ9FOQ2enPb7F\no8ToK2GdvxoYtehanBXpY5aFouIscx+zLFgQZ7b6mGUlizhzca3MdvgWT7eoMFOaSY52GoriNX3l\nb8K1OCsLrZwVQytnblGPmdJ4tJ1G/VCPWQp9dlbTDvWclUjNPGegvrOiaLsMRclAu3Yain+oMGtB\nn71VtaKp4gw8EGgqznoCNf9Xjo+OHN9i8i8en9pp+Dc7Sm3os7COhMcsSdWHNcvymKWR5dCmVY9Z\nkgKHNav2mCVJO6zpm6fLt3i6RYWZ0jNE4kzP2FRqTR+lV86a6jkDD3xnF1I735l6zqpFD2Uqvce8\nmQzfMl57nXmMHsrMSF95q45o6qFN54c1oXaHNvWwZjb0UKai5CVsp6GVM6X29JW/iaZWz5xXzqCW\nlTOtnpWPCrPc+OjI8S2mGsST6HXmOBqn+BZP47j/oXLX35dzfAePWRplirO1A49s/d8HcVaqxyyN\nDuLMtccsjTePdR3BaNRjpihNIdbrzIeTApQG45s4K4BWzkqkZpUz0MpZmajHTFHQXme+0SiPGT/f\ndscxHy53g33lrh7Uc1YqNfOcgfrO0tA+ZopiCe115g+NFWag4iwH2uusHqg4G40X5n8RmSIiPxSR\njSKySkRaFjlFZL6IrBSRDSLylIjMtxFDdfjoyPEtpnrGU1U7jXrOTrOpNIdVcVizr83jBTxmSWwe\n1ox7zJK4aKdx+bpZlW9zFIl2Gj56zJIxuT6sqR6zdK4G3gCmA2cA14jIfm3GnwnsBBwPfElEPmMp\njgp4wnUAKfgWU33jGZk2VPoZm/WdnUZTbQ4rW5xBa3H25HIrq7/3gZOtCLQNy1e1ffxazqlUoD29\nfINXvrPl69yGkUZaTC7F2ZPuNl0KXQszEZkAnAxcbIx53RizFLibIHFthzHmSmPMcmPM28aY3wB3\nAbO7jaM6NroOIAXfYqp5PCW306j57DQOZzns/ofcnBTw2nqrm+hWnG1ZvynTuKrE2ab1WwB/TgpY\n/6brILanVUyu2mk0LYfZqJjtA2wxxjwVu28FcEDG538UeNRCHIpiD4ftNJTKcZvD9IzNzPTkGZs1\nKltEuD60WXdsCLOJwHDivmFgx05PFJFLAQFushBHRTzvOoAUfIupIfHE2ml4EE1p+BaPA9znsCrF\n2YtDpWyiqDjbNPRyrvFli7PVQ6+PWnYtzoZewLt2GkMZSlRVirMXq9tUNRhj2t6A+4G3gbdSbg8A\nB91a12AAAAV4SURBVAOvJZ5zPnBXh/V+CXgK2KXDOKM3vemt926dclPWGw5zmOs51Jve9Obm1k3O\nGkMHjDHHtHs89GfsICLvix0KOIg2pX0RORv4G+Cjxpi2YrcJp8wriuIOlzlM85eiKHmx0sdMRG4j\nUIlfBA4B7gE+Yox5PGXs6cCVwBxjzK+73riiKEqXaA5TFMUXbLXLOA+YALwM3AqcGyU0ETlSRDbE\nxi4CpgLLROTVsBfQ1ZbiUBRFKYLmMEVRvMD7zv+KoiiKoii9gncXMffhKgI5Y7hCRH4vImtEpJSL\naWSNp6qrKuSZn3D8WBF5QkSedR2PiBwqIj8NKx0visiXXcUjIuNE5FoReSn8DN0lIruUEM95IrJM\nRN4QkRs7jP1KOC/rROR6ERlrO54mo/mreDxV5a88McXGaw5LH6c5rAxsnflk8Qyq/vD2hwQdXNYD\n+7UYO5/gjKp3EPQiGgI+U1UMwDnA48Au4e1R4C9czUlZ89HNexSOvwgYAJ51+ZkBpgGrgVOBMcA7\ngQ84jOdvgF8B7wLGATcDd5QQz6eBE4B/BG5sM+44gjPP9wUmE5zNuLiM96ypN81fXcVTSf7K+z6F\n4zWHaQ6r7OY8gMSkTgA2A++L3Xdz1okFvgV8q6oYgKXAF2LLZwP/15c5sTEf3cYD7Bkm/OPKSGo5\n36/Lge/ZjqGLeK4G/j62/Eng8RJjW9Qhqd0K/F1s+VjgxTLnq0k3zV9256SM/FUkJs1hmsOqvvl2\nKNOHqwjkieGA8LFO46qKJ0kZV1XIG8+3gb8luA5hGeSJ5whgnYgsFZHVYdl9d4fx3AAcKSK7SNCy\n4XTg3yzHk4e0z/O7RWSKo3jqhuav7uJJUtZVYTSH2YtHc1gJ+CbM3HfgzhdDcuxweJ9NCs2Jxfko\nHI+InATsYIy523IMheIB3gN8FvgysDvBoZJ+h/H8BniWoPn+eoLy+yLL8eQh7fMsZPj+KYDmr27j\n2UqJ+StXTJrDOsajOawEKhVmInK/iLwtIm+l3B4guBbp5MTTJgGvdljvl4AzgE8aY7q95OvGcJtZ\nYkiOnYT966nmiQewPh+F4gl/PV1BkEAg+HKUQZ75eR34oTHmYWPMCHAp8BERsfmlzRPPtcB4YAqB\nV+SHwL9bjCUvaZ9nQ4fvX6+g+av0eIDS81fmmDSHZYpHc1gJVCrMjDHHGGPeYYzZIeV2FIH63kFE\n3hd7WtYO3MeaDlcRyMhvgDEZY3g0fCzi4HaxVhBPGfNRNJ73A3sAPxORF4EfALuKyAsi8l4H8QAM\nEnxJ4xjsJtw88cwEvmuMGQ53QFcBh4vIVIvx5CHt87zaGLPOUTxeofmr9HiqyF95YtIc1jkezWFl\n4NrklrwBtxEY+CYQnA2yjtZnNZ1OcAaG1bNSssZAcFbTo8Cu4e0R4Iuu5qSs+SgSD4Hof3fsdhLw\nO2A6Yf88B/NzDLCWIJmMBf4B+KnD9+tG4PsEv+rGAguA50qIZwfgD4DFBCbe8QSHZ5LjjgNeAPYj\n+AX8f4DLy/wsNe2m+aureCrJX1lj0hymOczVzXkAKRM7haAcupHg+Pnc2GNHAhtiy08TnD2ygaBU\nuQG4uqwYktsP7/v78Ivye+DrVc5JVfPRzfzEnnM05Z1qnuf9OidMrmuBu4DdHL5fU4F/Jjj9/RWC\nC2rPKiGeS9j+It5fI/CovAq8Jzb2vwMvEfhFrgfGlvGeNfWm+at4PFXlr7xzFHuO5jDNYZXctPO/\noiiKoiiKJ/h2VqaiKIqiKErPosJMURRFURTFE1SYKYqiKIqieIIKM0VRFEVRFE9QYaYoiqIoiuIJ\nKswURVEURVE8QYWZoiiKoiiKJ6gwUxRFURRF8QQVZoqiKIqiKJ7w/wOiAZBmyw4BkAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6bff5acbe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1s = np.linspace(-0.2, 1.2, 100)\n",
    "x2s = np.linspace(-0.2, 1.2, 100)\n",
    "x1, x2 = np.meshgrid(x1s, x2s)\n",
    "\n",
    "z1 = mlp_xor(x1, x2, activation=heaviside)\n",
    "z2 = mlp_xor(x1, x2, activation=sigmoid)\n",
    "\n",
    "plt.figure(figsize=(10,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plt.contourf(x1, x2, z1)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: heaviside\", fontsize=14)\n",
    "plt.grid(True)\n",
    "\n",
    "plt.subplot(122)\n",
    "plt.contourf(x1, x2, z2)\n",
    "plt.plot([0, 1], [0, 1], \"gs\", markersize=20)\n",
    "plt.plot([0, 1], [1, 0], \"y^\", markersize=20)\n",
    "plt.title(\"Activation function: sigmoid\", fontsize=14)\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# FNN for MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## using tf.learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train = mnist.train.images\n",
    "X_test = mnist.test.images\n",
    "y_train = mnist.train.labels.astype(\"int\")\n",
    "y_test = mnist.test.labels.astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_task_id': 0, '_evaluation_master': '', '_keep_checkpoint_max': 5, '_save_checkpoints_secs': 600, '_model_dir': None, '_environment': 'local', '_keep_checkpoint_every_n_hours': 10000, '_is_chief': True, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_task_type': None, '_master': '', '_num_ps_replicas': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f6bf60816d8>, '_num_worker_replicas': 0, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_tf_random_seed': None}\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpyryb4i5c\n",
      "WARNING:tensorflow:From /home/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:615: scalar_summary (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2016-11-30.\n",
      "Instructions for updating:\n",
      "Please switch to tf.summary.scalar. Note that tf.summary.scalar uses the node name instead of the tag. This means that TensorFlow will automatically de-duplicate summary names based on the scope they are created in. Also, passing a tensor or list of tags to a scalar summary op is no longer supported.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpyryb4i5c/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.27088, step = 1\n",
      "INFO:tensorflow:global_step/sec: 109.115\n",
      "INFO:tensorflow:loss = 0.305753, step = 101 (0.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.355\n",
      "INFO:tensorflow:loss = 0.293447, step = 201 (0.882 sec)\n",
      "INFO:tensorflow:global_step/sec: 124.783\n",
      "INFO:tensorflow:loss = 0.33924, step = 301 (0.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 127.93\n",
      "INFO:tensorflow:loss = 0.204983, step = 401 (0.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.194\n",
      "INFO:tensorflow:loss = 0.253081, step = 501 (0.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.999\n",
      "INFO:tensorflow:loss = 0.0850319, step = 601 (0.885 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.907\n",
      "INFO:tensorflow:loss = 0.118361, step = 701 (0.991 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.884\n",
      "INFO:tensorflow:loss = 0.18682, step = 801 (0.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.23\n",
      "INFO:tensorflow:loss = 0.122104, step = 901 (0.825 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.9354\n",
      "INFO:tensorflow:loss = 0.212864, step = 1001 (1.022 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.1647\n",
      "INFO:tensorflow:loss = 0.195504, step = 1101 (1.247 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.0469\n",
      "INFO:tensorflow:loss = 0.160396, step = 1201 (1.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 74.1753\n",
      "INFO:tensorflow:loss = 0.203332, step = 1301 (1.353 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.7899\n",
      "INFO:tensorflow:loss = 0.0632323, step = 1401 (1.132 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.471\n",
      "INFO:tensorflow:loss = 0.0949302, step = 1501 (1.003 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.9203\n",
      "INFO:tensorflow:loss = 0.134847, step = 1601 (1.171 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.3459\n",
      "INFO:tensorflow:loss = 0.0247064, step = 1701 (1.053 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.597\n",
      "INFO:tensorflow:loss = 0.161126, step = 1801 (0.988 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.489\n",
      "INFO:tensorflow:loss = 0.0946856, step = 1901 (0.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.693\n",
      "INFO:tensorflow:loss = 0.107431, step = 2001 (0.884 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.313\n",
      "INFO:tensorflow:loss = 0.0303565, step = 2101 (0.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.156\n",
      "INFO:tensorflow:loss = 0.0464571, step = 2201 (0.978 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.68\n",
      "INFO:tensorflow:loss = 0.0653001, step = 2301 (0.983 sec)\n",
      "INFO:tensorflow:global_step/sec: 117\n",
      "INFO:tensorflow:loss = 0.0668861, step = 2401 (0.855 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.93\n",
      "INFO:tensorflow:loss = 0.0786721, step = 2501 (0.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.134\n",
      "INFO:tensorflow:loss = 0.05038, step = 2601 (0.832 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.995\n",
      "INFO:tensorflow:loss = 0.0112002, step = 2701 (0.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.599\n",
      "INFO:tensorflow:loss = 0.0525919, step = 2801 (0.823 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.089\n",
      "INFO:tensorflow:loss = 0.125073, step = 2901 (0.862 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.509\n",
      "INFO:tensorflow:loss = 0.015612, step = 3001 (0.828 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.597\n",
      "INFO:tensorflow:loss = 0.0453644, step = 3101 (0.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.225\n",
      "INFO:tensorflow:loss = 0.0104845, step = 3201 (0.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.3738\n",
      "INFO:tensorflow:loss = 0.0249421, step = 3301 (1.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.014\n",
      "INFO:tensorflow:loss = 0.181013, step = 3401 (0.859 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.933\n",
      "INFO:tensorflow:loss = 0.0940675, step = 3501 (0.968 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.1773\n",
      "INFO:tensorflow:loss = 0.11485, step = 3601 (1.061 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.536\n",
      "INFO:tensorflow:loss = 0.0358894, step = 3701 (0.870 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.2266\n",
      "INFO:tensorflow:loss = 0.00391026, step = 3801 (1.045 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.4727\n",
      "INFO:tensorflow:loss = 0.0868192, step = 3901 (1.025 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.628\n",
      "INFO:tensorflow:loss = 0.14505, step = 4001 (0.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.769\n",
      "INFO:tensorflow:loss = 0.0356335, step = 4101 (0.946 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.795\n",
      "INFO:tensorflow:loss = 0.0647914, step = 4201 (0.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.96\n",
      "INFO:tensorflow:loss = 0.148931, step = 4301 (0.841 sec)\n",
      "INFO:tensorflow:global_step/sec: 125.808\n",
      "INFO:tensorflow:loss = 0.101883, step = 4401 (0.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.38\n",
      "INFO:tensorflow:loss = 0.0244017, step = 4501 (0.883 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.526\n",
      "INFO:tensorflow:loss = 0.0472389, step = 4601 (1.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.0294\n",
      "INFO:tensorflow:loss = 0.0110032, step = 4701 (1.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 75.3269\n",
      "INFO:tensorflow:loss = 0.0420206, step = 4801 (1.326 sec)\n",
      "INFO:tensorflow:global_step/sec: 74.8312\n",
      "INFO:tensorflow:loss = 0.0807173, step = 4901 (1.338 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.0986\n",
      "INFO:tensorflow:loss = 0.0610474, step = 5001 (1.146 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.9139\n",
      "INFO:tensorflow:loss = 0.0131842, step = 5101 (1.010 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.3601\n",
      "INFO:tensorflow:loss = 0.0243407, step = 5201 (1.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.2004\n",
      "INFO:tensorflow:loss = 0.0568055, step = 5301 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.813\n",
      "INFO:tensorflow:loss = 0.144256, step = 5401 (0.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.575\n",
      "INFO:tensorflow:loss = 0.0246808, step = 5501 (0.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.855\n",
      "INFO:tensorflow:loss = 0.0428523, step = 5601 (0.895 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.804\n",
      "INFO:tensorflow:loss = 0.00910601, step = 5701 (0.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.24\n",
      "INFO:tensorflow:loss = 0.00944913, step = 5801 (0.832 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.097\n",
      "INFO:tensorflow:loss = 0.0560337, step = 5901 (0.870 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.732\n",
      "INFO:tensorflow:loss = 0.133482, step = 6001 (0.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.766\n",
      "INFO:tensorflow:loss = 0.0218963, step = 6101 (0.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.711\n",
      "INFO:tensorflow:loss = 0.0188566, step = 6201 (0.986 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.017\n",
      "INFO:tensorflow:loss = 0.0803808, step = 6301 (0.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.141\n",
      "INFO:tensorflow:loss = 0.0587137, step = 6401 (1.148 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.4439\n",
      "INFO:tensorflow:loss = 0.0172104, step = 6501 (1.013 sec)\n",
      "INFO:tensorflow:global_step/sec: 93.575\n",
      "INFO:tensorflow:loss = 0.00879865, step = 6601 (1.076 sec)\n",
      "INFO:tensorflow:global_step/sec: 79.9843\n",
      "INFO:tensorflow:loss = 0.0151028, step = 6701 (1.245 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.564\n",
      "INFO:tensorflow:loss = 0.00937015, step = 6801 (0.993 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.705\n",
      "INFO:tensorflow:loss = 0.0161164, step = 6901 (1.115 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.6281\n",
      "INFO:tensorflow:loss = 0.0260159, step = 7001 (1.036 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.785\n",
      "INFO:tensorflow:loss = 0.00337572, step = 7101 (0.971 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.787\n",
      "INFO:tensorflow:loss = 0.0451997, step = 7201 (0.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.595\n",
      "INFO:tensorflow:loss = 0.00944478, step = 7301 (0.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.022\n",
      "INFO:tensorflow:loss = 0.0251339, step = 7401 (0.840 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.78\n",
      "INFO:tensorflow:loss = 0.00393062, step = 7501 (0.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.398\n",
      "INFO:tensorflow:loss = 0.0184842, step = 7601 (0.820 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.859\n",
      "INFO:tensorflow:loss = 0.0151264, step = 7701 (0.841 sec)\n",
      "INFO:tensorflow:global_step/sec: 106.295\n",
      "INFO:tensorflow:loss = 0.00786291, step = 7801 (0.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.121\n",
      "INFO:tensorflow:loss = 0.00539545, step = 7901 (0.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.986\n",
      "INFO:tensorflow:loss = 0.0028784, step = 8001 (0.880 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.423\n",
      "INFO:tensorflow:loss = 0.0555786, step = 8101 (0.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.639\n",
      "INFO:tensorflow:loss = 0.0265214, step = 8201 (0.975 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.967\n",
      "INFO:tensorflow:loss = 0.0574934, step = 8301 (0.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.872\n",
      "INFO:tensorflow:loss = 0.014721, step = 8401 (0.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.2295\n",
      "INFO:tensorflow:loss = 0.00530998, step = 8501 (1.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5901\n",
      "INFO:tensorflow:loss = 0.00565723, step = 8601 (1.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.7359\n",
      "INFO:tensorflow:loss = 0.00487031, step = 8701 (1.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.72\n",
      "INFO:tensorflow:loss = 0.00784375, step = 8801 (1.001 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.2774\n",
      "INFO:tensorflow:loss = 0.00449987, step = 8901 (1.279 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.341\n",
      "INFO:tensorflow:loss = 0.0146847, step = 9001 (0.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.419\n",
      "INFO:tensorflow:loss = 0.00584301, step = 9101 (0.968 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.667\n",
      "INFO:tensorflow:loss = 0.00937499, step = 9201 (0.874 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.851\n",
      "INFO:tensorflow:loss = 0.00642365, step = 9301 (0.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.05\n",
      "INFO:tensorflow:loss = 0.0288566, step = 9401 (0.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.865\n",
      "INFO:tensorflow:loss = 0.00707008, step = 9501 (0.825 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.07\n",
      "INFO:tensorflow:loss = 0.00872278, step = 9601 (0.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.746\n",
      "INFO:tensorflow:loss = 0.0160108, step = 9701 (0.980 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.2761\n",
      "INFO:tensorflow:loss = 0.00632974, step = 9801 (1.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.4566\n",
      "INFO:tensorflow:loss = 0.0255936, step = 9901 (1.005 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.588\n",
      "INFO:tensorflow:loss = 0.00659009, step = 10001 (0.837 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.063\n",
      "INFO:tensorflow:loss = 0.00583273, step = 10101 (0.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.628\n",
      "INFO:tensorflow:loss = 0.0126203, step = 10201 (0.816 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.033\n",
      "INFO:tensorflow:loss = 0.00469632, step = 10301 (0.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.708\n",
      "INFO:tensorflow:loss = 0.0126431, step = 10401 (0.815 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.091\n",
      "INFO:tensorflow:loss = 0.00549423, step = 10501 (0.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 124.642\n",
      "INFO:tensorflow:loss = 0.00750569, step = 10601 (0.802 sec)\n",
      "INFO:tensorflow:global_step/sec: 139.646\n",
      "INFO:tensorflow:loss = 0.00899679, step = 10701 (0.716 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.672\n",
      "INFO:tensorflow:loss = 0.00896117, step = 10801 (0.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.5484\n",
      "INFO:tensorflow:loss = 0.00319331, step = 10901 (1.015 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.792\n",
      "INFO:tensorflow:loss = 0.031507, step = 11001 (0.982 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.258\n",
      "INFO:tensorflow:loss = 0.00451178, step = 11101 (0.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.4383\n",
      "INFO:tensorflow:loss = 0.00169156, step = 11201 (1.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.734\n",
      "INFO:tensorflow:loss = 0.00650894, step = 11301 (0.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 125.25\n",
      "INFO:tensorflow:loss = 0.00702863, step = 11401 (0.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.3759\n",
      "INFO:tensorflow:loss = 0.0150975, step = 11501 (1.061 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.8412\n",
      "INFO:tensorflow:loss = 0.00229354, step = 11601 (1.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.0636\n",
      "INFO:tensorflow:loss = 0.00269045, step = 11701 (1.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.9645\n",
      "INFO:tensorflow:loss = 0.000612422, step = 11801 (1.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.1337\n",
      "INFO:tensorflow:loss = 0.00613072, step = 11901 (1.204 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.1211\n",
      "INFO:tensorflow:loss = 0.000354799, step = 12001 (1.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.2718\n",
      "INFO:tensorflow:loss = 0.00256022, step = 12101 (1.082 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.137\n",
      "INFO:tensorflow:loss = 0.00543334, step = 12201 (1.135 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.225\n",
      "INFO:tensorflow:loss = 0.00604155, step = 12301 (0.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.383\n",
      "INFO:tensorflow:loss = 0.000230141, step = 12401 (0.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.852\n",
      "INFO:tensorflow:loss = 0.00199115, step = 12501 (0.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.304\n",
      "INFO:tensorflow:loss = 0.00312397, step = 12601 (0.969 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.7516\n",
      "INFO:tensorflow:loss = 0.00403427, step = 12701 (1.004 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.963\n",
      "INFO:tensorflow:loss = 0.0174728, step = 12801 (0.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.665\n",
      "INFO:tensorflow:loss = 0.00341833, step = 12901 (0.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.6715\n",
      "INFO:tensorflow:loss = 0.00650311, step = 13001 (1.034 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.7804\n",
      "INFO:tensorflow:loss = 0.00313541, step = 13101 (1.028 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.018\n",
      "INFO:tensorflow:loss = 0.00168857, step = 13201 (0.842 sec)\n",
      "INFO:tensorflow:global_step/sec: 139.227\n",
      "INFO:tensorflow:loss = 0.00215157, step = 13301 (0.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.519\n",
      "INFO:tensorflow:loss = 0.00395612, step = 13401 (0.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 133.786\n",
      "INFO:tensorflow:loss = 0.00559643, step = 13501 (0.750 sec)\n",
      "INFO:tensorflow:global_step/sec: 130.016\n",
      "INFO:tensorflow:loss = 0.00559, step = 13601 (0.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.042\n",
      "INFO:tensorflow:loss = 0.00143656, step = 13701 (0.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.3995\n",
      "INFO:tensorflow:loss = 0.00611374, step = 13801 (1.027 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.289\n",
      "INFO:tensorflow:loss = 0.00306185, step = 13901 (0.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.13\n",
      "INFO:tensorflow:loss = 0.00160339, step = 14001 (0.846 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.045\n",
      "INFO:tensorflow:loss = 0.0093842, step = 14101 (0.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.5968\n",
      "INFO:tensorflow:loss = 0.00438923, step = 14201 (1.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.566\n",
      "INFO:tensorflow:loss = 0.00141394, step = 14301 (0.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.866\n",
      "INFO:tensorflow:loss = 0.000691021, step = 14401 (0.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.7392\n",
      "INFO:tensorflow:loss = 0.000639933, step = 14501 (1.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.1093\n",
      "INFO:tensorflow:loss = 0.00603079, step = 14601 (1.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 79.5456\n",
      "INFO:tensorflow:loss = 0.00173162, step = 14701 (1.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.1903\n",
      "INFO:tensorflow:loss = 0.00100964, step = 14801 (1.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.2147\n",
      "INFO:tensorflow:loss = 0.00184101, step = 14901 (1.029 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.1951\n",
      "INFO:tensorflow:loss = 0.000815807, step = 15001 (1.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.7052\n",
      "INFO:tensorflow:loss = 0.00143294, step = 15101 (1.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.951\n",
      "INFO:tensorflow:loss = 0.0016952, step = 15201 (0.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.7644\n",
      "INFO:tensorflow:loss = 0.0015011, step = 15301 (1.002 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.5\n",
      "INFO:tensorflow:loss = 0.00127045, step = 15401 (0.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 143.955\n",
      "INFO:tensorflow:loss = 0.00512107, step = 15501 (0.695 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.062\n",
      "INFO:tensorflow:loss = 0.00312368, step = 15601 (0.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.069\n",
      "INFO:tensorflow:loss = 0.00885074, step = 15701 (0.869 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.421\n",
      "INFO:tensorflow:loss = 0.000690355, step = 15801 (0.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.457\n",
      "INFO:tensorflow:loss = 0.000349773, step = 15901 (0.838 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.848\n",
      "INFO:tensorflow:loss = 0.00575736, step = 16001 (0.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.9471\n",
      "INFO:tensorflow:loss = 0.00284249, step = 16101 (1.042 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.4505\n",
      "INFO:tensorflow:loss = 0.00019728, step = 16201 (1.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.2701\n",
      "INFO:tensorflow:loss = 0.00486241, step = 16301 (1.014 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.9906\n",
      "INFO:tensorflow:loss = 0.00137645, step = 16401 (1.053 sec)\n",
      "INFO:tensorflow:global_step/sec: 106.566\n",
      "INFO:tensorflow:loss = 0.00266368, step = 16501 (0.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.382\n",
      "INFO:tensorflow:loss = 0.00344792, step = 16601 (0.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.676\n",
      "INFO:tensorflow:loss = 0.00221835, step = 16701 (0.858 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.2272\n",
      "INFO:tensorflow:loss = 0.00195335, step = 16801 (1.017 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.184\n",
      "INFO:tensorflow:loss = 0.00500357, step = 16901 (0.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.861\n",
      "INFO:tensorflow:loss = 0.00441929, step = 17001 (0.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.02\n",
      "INFO:tensorflow:loss = 0.0014685, step = 17101 (0.870 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.268\n",
      "INFO:tensorflow:loss = 0.00173975, step = 17201 (0.988 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.0974\n",
      "INFO:tensorflow:loss = 0.0018612, step = 17301 (1.123 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.431\n",
      "INFO:tensorflow:loss = 0.00102956, step = 17401 (0.985 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.051\n",
      "INFO:tensorflow:loss = 0.00177292, step = 17501 (0.993 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.435\n",
      "INFO:tensorflow:loss = 0.000449727, step = 17601 (0.957 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.081\n",
      "INFO:tensorflow:loss = 0.000921113, step = 17701 (0.859 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.4\n",
      "INFO:tensorflow:loss = 0.000382117, step = 17801 (0.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.518\n",
      "INFO:tensorflow:loss = 0.00230232, step = 17901 (0.810 sec)\n",
      "INFO:tensorflow:global_step/sec: 128.021\n",
      "INFO:tensorflow:loss = 0.000351347, step = 18001 (0.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 128.801\n",
      "INFO:tensorflow:loss = 0.00100194, step = 18101 (0.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.418\n",
      "INFO:tensorflow:loss = 0.0044797, step = 18201 (0.789 sec)\n",
      "INFO:tensorflow:global_step/sec: 138.35\n",
      "INFO:tensorflow:loss = 0.0106527, step = 18301 (0.723 sec)\n",
      "INFO:tensorflow:global_step/sec: 135.564\n",
      "INFO:tensorflow:loss = 0.00310102, step = 18401 (0.740 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.42\n",
      "INFO:tensorflow:loss = 0.00272409, step = 18501 (0.821 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.973\n",
      "INFO:tensorflow:loss = 0.00387095, step = 18601 (0.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.256\n",
      "INFO:tensorflow:loss = 0.000694669, step = 18701 (0.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.782\n",
      "INFO:tensorflow:loss = 0.0021766, step = 18801 (0.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.89\n",
      "INFO:tensorflow:loss = 0.00357013, step = 18901 (0.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.995\n",
      "INFO:tensorflow:loss = 0.00181011, step = 19001 (0.952 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.2971\n",
      "INFO:tensorflow:loss = 0.00108108, step = 19101 (1.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 71.3352\n",
      "INFO:tensorflow:loss = 0.000303804, step = 19201 (1.402 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.3104\n",
      "INFO:tensorflow:loss = 0.00780343, step = 19301 (1.187 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.8629\n",
      "INFO:tensorflow:loss = 0.000820897, step = 19401 (1.087 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.7437\n",
      "INFO:tensorflow:loss = 0.00375399, step = 19501 (1.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.1409\n",
      "INFO:tensorflow:loss = 0.000378957, step = 19601 (1.019 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.2608\n",
      "INFO:tensorflow:loss = 0.000118754, step = 19701 (1.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.618\n",
      "INFO:tensorflow:loss = 0.000441555, step = 19801 (0.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.624\n",
      "INFO:tensorflow:loss = 0.00179196, step = 19901 (0.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.141\n",
      "INFO:tensorflow:loss = 0.00241299, step = 20001 (0.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 129.392\n",
      "INFO:tensorflow:loss = 0.00136867, step = 20101 (0.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.418\n",
      "INFO:tensorflow:loss = 0.00191704, step = 20201 (0.980 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.1255\n",
      "INFO:tensorflow:loss = 0.00154219, step = 20301 (1.040 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.817\n",
      "INFO:tensorflow:loss = 0.000264576, step = 20401 (0.884 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.4706\n",
      "INFO:tensorflow:loss = 0.000999024, step = 20501 (1.081 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.486\n",
      "INFO:tensorflow:loss = 0.00102787, step = 20601 (0.861 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.035\n",
      "INFO:tensorflow:loss = 0.000945482, step = 20701 (0.961 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.258\n",
      "INFO:tensorflow:loss = 0.000609766, step = 20801 (0.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.216\n",
      "INFO:tensorflow:loss = 0.00108098, step = 20901 (0.873 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.542\n",
      "INFO:tensorflow:loss = 0.00125468, step = 21001 (0.838 sec)\n",
      "INFO:tensorflow:global_step/sec: 138.856\n",
      "INFO:tensorflow:loss = 0.00109906, step = 21101 (0.718 sec)\n",
      "INFO:tensorflow:global_step/sec: 133.736\n",
      "INFO:tensorflow:loss = 0.00594512, step = 21201 (0.750 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.283\n",
      "INFO:tensorflow:loss = 0.000867914, step = 21301 (0.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.82\n",
      "INFO:tensorflow:loss = 0.0016451, step = 21401 (0.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.405\n",
      "INFO:tensorflow:loss = 0.000256016, step = 21501 (0.978 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.596\n",
      "INFO:tensorflow:loss = 0.00123037, step = 21601 (0.966 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.9015\n",
      "INFO:tensorflow:loss = 0.000802957, step = 21701 (1.030 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.0692\n",
      "INFO:tensorflow:loss = 0.000565691, step = 21801 (1.040 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.5598\n",
      "INFO:tensorflow:loss = 0.00122137, step = 21901 (1.142 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.7924\n",
      "INFO:tensorflow:loss = 0.00010733, step = 22001 (1.078 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.3586\n",
      "INFO:tensorflow:loss = 5.66901e-05, step = 22101 (1.311 sec)\n",
      "INFO:tensorflow:global_step/sec: 66.4275\n",
      "INFO:tensorflow:loss = 0.00166592, step = 22201 (1.505 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.6193\n",
      "INFO:tensorflow:loss = 0.00302231, step = 22301 (1.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.9407\n",
      "INFO:tensorflow:loss = 0.00190215, step = 22401 (1.053 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.3677\n",
      "INFO:tensorflow:loss = 0.00112757, step = 22501 (1.038 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.6634\n",
      "INFO:tensorflow:loss = 0.00386172, step = 22601 (1.117 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.3118\n",
      "INFO:tensorflow:loss = 0.00128377, step = 22701 (1.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 93.6613\n",
      "INFO:tensorflow:loss = 0.000604829, step = 22801 (1.067 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.3135\n",
      "INFO:tensorflow:loss = 0.0032004, step = 22901 (1.061 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.5427\n",
      "INFO:tensorflow:loss = 0.000466001, step = 23001 (1.116 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.6531\n",
      "INFO:tensorflow:loss = 0.00394725, step = 23101 (1.128 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.5787\n",
      "INFO:tensorflow:loss = 0.00209325, step = 23201 (1.036 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.608\n",
      "INFO:tensorflow:loss = 0.00228685, step = 23301 (0.995 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.904\n",
      "INFO:tensorflow:loss = 0.000577641, step = 23401 (0.964 sec)\n",
      "INFO:tensorflow:global_step/sec: 90.3455\n",
      "INFO:tensorflow:loss = 0.00126437, step = 23501 (1.104 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.018\n",
      "INFO:tensorflow:loss = 0.000700838, step = 23601 (0.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.874\n",
      "INFO:tensorflow:loss = 0.000308366, step = 23701 (0.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.9102\n",
      "INFO:tensorflow:loss = 0.00118046, step = 23801 (1.010 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.0903\n",
      "INFO:tensorflow:loss = 0.0015566, step = 23901 (1.041 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.5583\n",
      "INFO:tensorflow:loss = 0.00120477, step = 24001 (1.059 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.917\n",
      "INFO:tensorflow:loss = 0.000551837, step = 24101 (1.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.331\n",
      "INFO:tensorflow:loss = 0.00221812, step = 24201 (1.028 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.464\n",
      "INFO:tensorflow:loss = 5.3221e-05, step = 24301 (0.839 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.105\n",
      "INFO:tensorflow:loss = 0.00164571, step = 24401 (0.996 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.526\n",
      "INFO:tensorflow:loss = 0.000686215, step = 24501 (0.995 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.818\n",
      "INFO:tensorflow:loss = 0.000221111, step = 24601 (0.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.9004\n",
      "INFO:tensorflow:loss = 0.000705277, step = 24701 (1.011 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.0023\n",
      "INFO:tensorflow:loss = 0.00145277, step = 24801 (1.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.1294\n",
      "INFO:tensorflow:loss = 0.00108229, step = 24901 (1.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.9619\n",
      "INFO:tensorflow:loss = 0.000333501, step = 25001 (1.126 sec)\n",
      "INFO:tensorflow:global_step/sec: 93.4009\n",
      "INFO:tensorflow:loss = 0.00045063, step = 25101 (1.069 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.706\n",
      "INFO:tensorflow:loss = 0.00196114, step = 25201 (0.995 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.0636\n",
      "INFO:tensorflow:loss = 0.000162464, step = 25301 (1.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.344\n",
      "INFO:tensorflow:loss = 0.000652182, step = 25401 (0.996 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.818\n",
      "INFO:tensorflow:loss = 0.00064856, step = 25501 (0.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.007\n",
      "INFO:tensorflow:loss = 0.000920504, step = 25601 (0.961 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.263\n",
      "INFO:tensorflow:loss = 0.000529175, step = 25701 (0.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.55\n",
      "INFO:tensorflow:loss = 0.000959268, step = 25801 (0.865 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.48\n",
      "INFO:tensorflow:loss = 0.00296025, step = 25901 (0.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.088\n",
      "INFO:tensorflow:loss = 5.3733e-05, step = 26001 (0.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.011\n",
      "INFO:tensorflow:loss = 0.00110358, step = 26101 (0.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.663\n",
      "INFO:tensorflow:loss = 0.00085139, step = 26201 (0.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.057\n",
      "INFO:tensorflow:loss = 0.00140477, step = 26301 (0.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.19\n",
      "INFO:tensorflow:loss = 0.00183658, step = 26401 (0.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.259\n",
      "INFO:tensorflow:loss = 0.000769894, step = 26501 (0.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.877\n",
      "INFO:tensorflow:loss = 0.000604787, step = 26601 (0.970 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.384\n",
      "INFO:tensorflow:loss = 3.70872e-05, step = 26701 (0.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.594\n",
      "INFO:tensorflow:loss = 0.000340864, step = 26801 (0.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.891\n",
      "INFO:tensorflow:loss = 0.000705364, step = 26901 (0.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.28\n",
      "INFO:tensorflow:loss = 0.000437724, step = 27001 (0.830 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.0098\n",
      "INFO:tensorflow:loss = 0.000623345, step = 27101 (1.239 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.3715\n",
      "INFO:tensorflow:loss = 0.000227634, step = 27201 (1.226 sec)\n",
      "INFO:tensorflow:global_step/sec: 79.5147\n",
      "INFO:tensorflow:loss = 0.00122763, step = 27301 (1.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.8829\n",
      "INFO:tensorflow:loss = 0.000121265, step = 27401 (1.091 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.1224\n",
      "INFO:tensorflow:loss = 0.00155231, step = 27501 (1.062 sec)\n",
      "INFO:tensorflow:global_step/sec: 106.359\n",
      "INFO:tensorflow:loss = 0.00122636, step = 27601 (0.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.9368\n",
      "INFO:tensorflow:loss = 0.000300534, step = 27701 (1.032 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.556\n",
      "INFO:tensorflow:loss = 0.000200766, step = 27801 (0.998 sec)\n",
      "INFO:tensorflow:global_step/sec: 124.818\n",
      "INFO:tensorflow:loss = 0.000295802, step = 27901 (0.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.628\n",
      "INFO:tensorflow:loss = 0.00169272, step = 28001 (0.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.3719\n",
      "INFO:tensorflow:loss = 0.000427281, step = 28101 (1.007 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.494\n",
      "INFO:tensorflow:loss = 0.000965312, step = 28201 (0.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.6301\n",
      "INFO:tensorflow:loss = 0.000911033, step = 28301 (1.014 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.6771\n",
      "INFO:tensorflow:loss = 0.000945343, step = 28401 (1.056 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.332\n",
      "INFO:tensorflow:loss = 0.000168487, step = 28501 (0.968 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.317\n",
      "INFO:tensorflow:loss = 0.00029019, step = 28601 (0.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.09\n",
      "INFO:tensorflow:loss = 0.000911714, step = 28701 (0.826 sec)\n",
      "INFO:tensorflow:global_step/sec: 134.063\n",
      "INFO:tensorflow:loss = 0.000982961, step = 28801 (0.746 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.698\n",
      "INFO:tensorflow:loss = 0.000492107, step = 28901 (0.872 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.555\n",
      "INFO:tensorflow:loss = 0.00125565, step = 29001 (0.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 130.894\n",
      "INFO:tensorflow:loss = 0.00107044, step = 29101 (0.764 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.697\n",
      "INFO:tensorflow:loss = 0.00154578, step = 29201 (0.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 129.257\n",
      "INFO:tensorflow:loss = 0.00117781, step = 29301 (0.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.669\n",
      "INFO:tensorflow:loss = 0.0013569, step = 29401 (0.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 106.871\n",
      "INFO:tensorflow:loss = 0.00138969, step = 29501 (0.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.008\n",
      "INFO:tensorflow:loss = 0.000270474, step = 29601 (0.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.29\n",
      "INFO:tensorflow:loss = 0.00026167, step = 29701 (0.859 sec)\n",
      "INFO:tensorflow:global_step/sec: 110.033\n",
      "INFO:tensorflow:loss = 0.000721617, step = 29801 (0.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.964\n",
      "INFO:tensorflow:loss = 0.00131128, step = 29901 (0.855 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.3764\n",
      "INFO:tensorflow:loss = 6.80431e-05, step = 30001 (1.049 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.4003\n",
      "INFO:tensorflow:loss = 0.00067758, step = 30101 (1.144 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.9139\n",
      "INFO:tensorflow:loss = 0.000518952, step = 30201 (1.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.2825\n",
      "INFO:tensorflow:loss = 0.00110232, step = 30301 (1.198 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.8098\n",
      "INFO:tensorflow:loss = 0.00131401, step = 30401 (1.035 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.6675\n",
      "INFO:tensorflow:loss = 0.000786718, step = 30501 (1.042 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.8625\n",
      "INFO:tensorflow:loss = 0.00124654, step = 30601 (1.011 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.132\n",
      "INFO:tensorflow:loss = 0.000689732, step = 30701 (1.000 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.916\n",
      "INFO:tensorflow:loss = 0.000969612, step = 30801 (0.990 sec)\n",
      "INFO:tensorflow:global_step/sec: 125.769\n",
      "INFO:tensorflow:loss = 0.00115421, step = 30901 (0.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.103\n",
      "INFO:tensorflow:loss = 0.000887637, step = 31001 (0.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.5209\n",
      "INFO:tensorflow:loss = 0.000893447, step = 31101 (1.015 sec)\n",
      "INFO:tensorflow:global_step/sec: 105.938\n",
      "INFO:tensorflow:loss = 0.000304312, step = 31201 (0.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.9939\n",
      "INFO:tensorflow:loss = 0.000731763, step = 31301 (0.997 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.9964\n",
      "INFO:tensorflow:loss = 0.000799198, step = 31401 (1.011 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.904\n",
      "INFO:tensorflow:loss = 0.000367327, step = 31501 (0.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 130.221\n",
      "INFO:tensorflow:loss = 0.000213264, step = 31601 (0.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.275\n",
      "INFO:tensorflow:loss = 0.000626006, step = 31701 (0.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 127.357\n",
      "INFO:tensorflow:loss = 0.000218763, step = 31801 (0.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.749\n",
      "INFO:tensorflow:loss = 0.000783426, step = 31901 (0.818 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.534\n",
      "INFO:tensorflow:loss = 0.00054644, step = 32001 (0.856 sec)\n",
      "INFO:tensorflow:global_step/sec: 100.033\n",
      "INFO:tensorflow:loss = 0.000255154, step = 32101 (1.000 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.276\n",
      "INFO:tensorflow:loss = 0.00112902, step = 32201 (0.855 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.555\n",
      "INFO:tensorflow:loss = 0.000730119, step = 32301 (0.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.96\n",
      "INFO:tensorflow:loss = 0.000282129, step = 32401 (0.968 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.911\n",
      "INFO:tensorflow:loss = 0.00050279, step = 32501 (0.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.5472\n",
      "INFO:tensorflow:loss = 0.000221824, step = 32601 (1.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.9696\n",
      "INFO:tensorflow:loss = 0.00109462, step = 32701 (1.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.5268\n",
      "INFO:tensorflow:loss = 0.0010723, step = 32801 (1.045 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.2558\n",
      "INFO:tensorflow:loss = 0.000435507, step = 32901 (1.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.0945\n",
      "INFO:tensorflow:loss = 0.000496574, step = 33001 (1.009 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.1781\n",
      "INFO:tensorflow:loss = 0.00156574, step = 33101 (1.063 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.0464\n",
      "INFO:tensorflow:loss = 0.00112433, step = 33201 (1.099 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.3421\n",
      "INFO:tensorflow:loss = 0.000244621, step = 33301 (1.213 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.2555\n",
      "INFO:tensorflow:loss = 0.00138953, step = 33401 (1.296 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.6432\n",
      "INFO:tensorflow:loss = 0.000336887, step = 33501 (1.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.3778\n",
      "INFO:tensorflow:loss = 0.000558819, step = 33601 (1.131 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.3235\n",
      "INFO:tensorflow:loss = 0.0010824, step = 33701 (1.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.6102\n",
      "INFO:tensorflow:loss = 0.00129974, step = 33801 (1.014 sec)\n",
      "INFO:tensorflow:global_step/sec: 117.063\n",
      "INFO:tensorflow:loss = 0.00101776, step = 33901 (0.853 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.375\n",
      "INFO:tensorflow:loss = 0.000843827, step = 34001 (0.834 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.368\n",
      "INFO:tensorflow:loss = 0.000515851, step = 34101 (0.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.1925\n",
      "INFO:tensorflow:loss = 0.00135123, step = 34201 (1.050 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.9958\n",
      "INFO:tensorflow:loss = 0.000299793, step = 34301 (1.020 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.012\n",
      "INFO:tensorflow:loss = 0.000365835, step = 34401 (0.968 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.8587\n",
      "INFO:tensorflow:loss = 0.000354905, step = 34501 (1.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.206\n",
      "INFO:tensorflow:loss = 0.000515715, step = 34601 (0.960 sec)\n",
      "INFO:tensorflow:global_step/sec: 125.749\n",
      "INFO:tensorflow:loss = 0.00174276, step = 34701 (0.796 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.527\n",
      "INFO:tensorflow:loss = 0.000698155, step = 34801 (0.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.609\n",
      "INFO:tensorflow:loss = 0.00051754, step = 34901 (0.858 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.4577\n",
      "INFO:tensorflow:loss = 0.000410336, step = 35001 (1.051 sec)\n",
      "INFO:tensorflow:global_step/sec: 93.8467\n",
      "INFO:tensorflow:loss = 0.000878718, step = 35101 (1.063 sec)\n",
      "INFO:tensorflow:global_step/sec: 90.193\n",
      "INFO:tensorflow:loss = 0.00045169, step = 35201 (1.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.4713\n",
      "INFO:tensorflow:loss = 0.000251107, step = 35301 (1.063 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.523\n",
      "INFO:tensorflow:loss = 0.000830846, step = 35401 (0.876 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.404\n",
      "INFO:tensorflow:loss = 0.00018106, step = 35501 (0.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.44\n",
      "INFO:tensorflow:loss = 0.000217935, step = 35601 (0.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.476\n",
      "INFO:tensorflow:loss = 0.000496036, step = 35701 (1.080 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.2667\n",
      "INFO:tensorflow:loss = 0.000707686, step = 35801 (1.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.9594\n",
      "INFO:tensorflow:loss = 0.000399849, step = 35901 (1.112 sec)\n",
      "INFO:tensorflow:global_step/sec: 93.6673\n",
      "INFO:tensorflow:loss = 0.00040634, step = 36001 (1.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 98.8626\n",
      "INFO:tensorflow:loss = 0.00113488, step = 36101 (1.014 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.911\n",
      "INFO:tensorflow:loss = 0.000625635, step = 36201 (0.878 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.647\n",
      "INFO:tensorflow:loss = 0.000176652, step = 36301 (0.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 112.766\n",
      "INFO:tensorflow:loss = 0.00089847, step = 36401 (0.887 sec)\n",
      "INFO:tensorflow:global_step/sec: 128.016\n",
      "INFO:tensorflow:loss = 0.000351239, step = 36501 (0.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 116.746\n",
      "INFO:tensorflow:loss = 0.000152989, step = 36601 (0.857 sec)\n",
      "INFO:tensorflow:global_step/sec: 127.752\n",
      "INFO:tensorflow:loss = 0.000382323, step = 36701 (0.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 128.569\n",
      "INFO:tensorflow:loss = 0.00074105, step = 36801 (0.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 124.718\n",
      "INFO:tensorflow:loss = 0.000547528, step = 36901 (0.802 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.043\n",
      "INFO:tensorflow:loss = 0.000548707, step = 37001 (0.881 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.158\n",
      "INFO:tensorflow:loss = 0.000135922, step = 37101 (0.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.9495\n",
      "INFO:tensorflow:loss = 0.000254125, step = 37201 (1.032 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.997\n",
      "INFO:tensorflow:loss = 0.000619946, step = 37301 (0.962 sec)\n",
      "INFO:tensorflow:global_step/sec: 118.081\n",
      "INFO:tensorflow:loss = 0.000265094, step = 37401 (0.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.126\n",
      "INFO:tensorflow:loss = 0.000346456, step = 37501 (0.967 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.999\n",
      "INFO:tensorflow:loss = 0.00023934, step = 37601 (0.980 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.7649\n",
      "INFO:tensorflow:loss = 0.000121965, step = 37701 (1.024 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.4734\n",
      "INFO:tensorflow:loss = 0.000472929, step = 37801 (1.184 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.2006\n",
      "INFO:tensorflow:loss = 0.00148828, step = 37901 (1.134 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.6449\n",
      "INFO:tensorflow:loss = 0.000185361, step = 38001 (1.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 87.7131\n",
      "INFO:tensorflow:loss = 0.000981002, step = 38101 (1.140 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.2982\n",
      "INFO:tensorflow:loss = 0.000777568, step = 38201 (1.231 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.625\n",
      "INFO:tensorflow:loss = 0.000145526, step = 38301 (1.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 102.612\n",
      "INFO:tensorflow:loss = 0.000453466, step = 38401 (0.974 sec)\n",
      "INFO:tensorflow:global_step/sec: 107.341\n",
      "INFO:tensorflow:loss = 0.0004499, step = 38501 (0.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.896\n",
      "INFO:tensorflow:loss = 0.000903513, step = 38601 (0.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 111.881\n",
      "INFO:tensorflow:loss = 0.00064379, step = 38701 (0.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.362\n",
      "INFO:tensorflow:loss = 0.000193101, step = 38801 (0.878 sec)\n",
      "INFO:tensorflow:global_step/sec: 103.402\n",
      "INFO:tensorflow:loss = 0.00123127, step = 38901 (0.965 sec)\n",
      "INFO:tensorflow:global_step/sec: 101.862\n",
      "INFO:tensorflow:loss = 0.000242546, step = 39001 (0.981 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.3244\n",
      "INFO:tensorflow:loss = 0.000970162, step = 39101 (1.060 sec)\n",
      "INFO:tensorflow:global_step/sec: 127.328\n",
      "INFO:tensorflow:loss = 0.000624692, step = 39201 (0.784 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.941\n",
      "INFO:tensorflow:loss = 0.000223198, step = 39301 (0.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 108.371\n",
      "INFO:tensorflow:loss = 0.000452549, step = 39401 (0.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.748\n",
      "INFO:tensorflow:loss = 0.00012364, step = 39501 (0.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 104.548\n",
      "INFO:tensorflow:loss = 0.000784426, step = 39601 (0.957 sec)\n",
      "INFO:tensorflow:global_step/sec: 99.2459\n",
      "INFO:tensorflow:loss = 0.000232702, step = 39701 (1.008 sec)\n",
      "INFO:tensorflow:global_step/sec: 97.5055\n",
      "INFO:tensorflow:loss = 0.00124041, step = 39801 (1.025 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.5848\n",
      "INFO:tensorflow:loss = 0.00105927, step = 39901 (1.080 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmpyryb4i5c/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.000499539.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SKCompat()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)\n",
    "dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,\n",
    "                                         feature_columns=feature_cols)\n",
    "dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1\n",
    "dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/tmpyryb4i5c/model.ckpt-40000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9819"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = dnn_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred['classes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.075499019998784889"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "\n",
    "y_pred_proba = y_pred['probabilities']\n",
    "log_loss(y_test, y_pred_proba)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Using plain TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def neuron_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        stddev = 2 / np.sqrt(n_inputs)\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)\n",
    "        W = tf.Variable(init, name=\"kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name=\"bias\")\n",
    "        Z = tf.matmul(X, W) + b\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = neuron_layer(X, n_hidden1, name=\"hidden1\",\n",
    "                           activation=tf.nn.relu)\n",
    "    hidden2 = neuron_layer(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                           activation=tf.nn.relu)\n",
    "    logits = neuron_layer(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,\n",
    "                                                              logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_epochs = 40\n",
    "batch_size = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.86 Test accuracy: 0.9134\n",
      "1 Train accuracy: 0.94 Test accuracy: 0.9299\n",
      "2 Train accuracy: 0.94 Test accuracy: 0.9369\n",
      "3 Train accuracy: 0.96 Test accuracy: 0.9432\n",
      "4 Train accuracy: 0.94 Test accuracy: 0.9487\n",
      "5 Train accuracy: 0.94 Test accuracy: 0.9527\n",
      "6 Train accuracy: 0.94 Test accuracy: 0.9536\n",
      "7 Train accuracy: 0.94 Test accuracy: 0.9574\n",
      "8 Train accuracy: 0.98 Test accuracy: 0.9596\n",
      "9 Train accuracy: 0.98 Test accuracy: 0.9605\n",
      "10 Train accuracy: 0.98 Test accuracy: 0.9634\n",
      "11 Train accuracy: 0.94 Test accuracy: 0.9646\n",
      "12 Train accuracy: 1.0 Test accuracy: 0.9658\n",
      "13 Train accuracy: 0.96 Test accuracy: 0.9674\n",
      "14 Train accuracy: 1.0 Test accuracy: 0.9665\n",
      "15 Train accuracy: 1.0 Test accuracy: 0.969\n",
      "16 Train accuracy: 1.0 Test accuracy: 0.9702\n",
      "17 Train accuracy: 0.98 Test accuracy: 0.969\n",
      "18 Train accuracy: 1.0 Test accuracy: 0.9718\n",
      "19 Train accuracy: 1.0 Test accuracy: 0.9721\n",
      "20 Train accuracy: 1.0 Test accuracy: 0.9715\n",
      "21 Train accuracy: 1.0 Test accuracy: 0.9726\n",
      "22 Train accuracy: 1.0 Test accuracy: 0.9716\n",
      "23 Train accuracy: 0.98 Test accuracy: 0.9736\n",
      "24 Train accuracy: 1.0 Test accuracy: 0.9739\n",
      "25 Train accuracy: 1.0 Test accuracy: 0.9745\n",
      "26 Train accuracy: 0.98 Test accuracy: 0.9739\n",
      "27 Train accuracy: 1.0 Test accuracy: 0.9734\n",
      "28 Train accuracy: 0.98 Test accuracy: 0.9746\n",
      "29 Train accuracy: 1.0 Test accuracy: 0.9752\n",
      "30 Train accuracy: 1.0 Test accuracy: 0.9744\n",
      "31 Train accuracy: 1.0 Test accuracy: 0.9746\n",
      "32 Train accuracy: 0.98 Test accuracy: 0.9746\n",
      "33 Train accuracy: 0.98 Test accuracy: 0.9757\n",
      "34 Train accuracy: 0.98 Test accuracy: 0.976\n",
      "35 Train accuracy: 1.0 Test accuracy: 0.9753\n",
      "36 Train accuracy: 0.98 Test accuracy: 0.9757\n",
      "37 Train accuracy: 0.98 Test accuracy: 0.977\n",
      "38 Train accuracy: 1.0 Test accuracy: 0.9762\n",
      "39 Train accuracy: 1.0 Test accuracy: 0.9776\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images,\n",
    "                                            y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_final.ckpt\") # or better, use save_path\n",
    "    X_new_scaled = mnist.test.images[:20]\n",
    "    Z = logits.eval(feed_dict={X: X_new_scaled})\n",
    "    y_pred = np.argmax(Z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted classes: [7 2 1 0 4 1 4 9 6 9 0 6 9 0 1 5 9 7 3 4]\n",
      "Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]\n"
     ]
    }
   ],
   "source": [
    "print(\"Predicted classes:\", y_pred)\n",
    "print(\"Actual classes:   \", mnist.test.labels[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2851015593374667&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0714285746216774\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;dnn/hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.1154700517654419\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mul&quot;\\n  input: &quot;dnn/hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;dnn/hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.20000000298023224\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;dnn/outputs/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mul&quot;\\n  input: &quot;dnn/outputs/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;dnn/outputs/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;dnn/outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 100\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 300\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_dnn/outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/add&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^dnn/hidden1/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden1/bias/Assign&quot;\\n  input: &quot;^dnn/hidden2/kernel/Assign&quot;\\n  input: &quot;^dnn/hidden2/bias/Assign&quot;\\n  input: &quot;^dnn/outputs/kernel/Assign&quot;\\n  input: &quot;^dnn/outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n        string_val: &quot;dnn/outputs/bias&quot;\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;dnn/outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@dnn/outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2851015593374667&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Using `dense()` instead of `neuron_layer()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function, except for a few minor differences:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* a few more differences are presented in chapter 11."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.94 Test accuracy: 0.9012\n",
      "1 Train accuracy: 0.88 Test accuracy: 0.9219\n",
      "2 Train accuracy: 0.96 Test accuracy: 0.9308\n",
      "3 Train accuracy: 0.96 Test accuracy: 0.9359\n",
      "4 Train accuracy: 0.9 Test accuracy: 0.9402\n",
      "5 Train accuracy: 0.94 Test accuracy: 0.9452\n",
      "6 Train accuracy: 0.88 Test accuracy: 0.9503\n",
      "7 Train accuracy: 0.9 Test accuracy: 0.952\n",
      "8 Train accuracy: 0.94 Test accuracy: 0.9555\n",
      "9 Train accuracy: 0.96 Test accuracy: 0.9555\n",
      "10 Train accuracy: 0.92 Test accuracy: 0.9584\n",
      "11 Train accuracy: 0.98 Test accuracy: 0.9605\n",
      "12 Train accuracy: 0.92 Test accuracy: 0.9624\n",
      "13 Train accuracy: 0.96 Test accuracy: 0.9633\n",
      "14 Train accuracy: 0.98 Test accuracy: 0.964\n",
      "15 Train accuracy: 1.0 Test accuracy: 0.9656\n",
      "16 Train accuracy: 1.0 Test accuracy: 0.9673\n",
      "17 Train accuracy: 0.98 Test accuracy: 0.9677\n",
      "18 Train accuracy: 1.0 Test accuracy: 0.9693\n",
      "19 Train accuracy: 0.96 Test accuracy: 0.9691\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "n_batches = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.06275436026702463&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;message&quot;\\n    value {\\n      s: &quot;Currently there is no way to take the second derivative of sparse_softmax_cross_entropy_with_logits due to the fused implementation\\\\\\'s interaction with tf.gradients()&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;train/GradientDescent/learning_rate&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden1/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_hidden2/bias/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/kernel/ApplyGradientDescent&quot;\\n  input: &quot;^train/GradientDescent/update_outputs/bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 6\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.06275436026702463&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## 1. to 8."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "See appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on)._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a `tf.summary.scalar()` to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\",\n",
    "                              activation=tf.nn.relu)\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, name=\"hidden2\",\n",
    "                              activation=tf.nn.relu)\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    loss_summary = tf.summary.scalar('log_loss', loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar('accuracy', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to define the directory to write the TensorBoard logs to:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logdir = log_dir(\"mnist_dnn\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the `FileWriter` that we will use to write the TensorBoard logs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hey! Why don't we implement early stopping? For this, we are going to need a validation set. Luckily, the dataset returned by TensorFlow's `input_data()` function (see above) is already split into a training set (60,000 instances, already shuffled for us), a validation set (5,000 instances) and a test set (5,000 instances). So we can easily define `X_valid` and `y_valid`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_valid = mnist.validation.images\n",
    "y_valid = mnist.validation.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m, n = X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 \tValidation accuracy: 90.620% \tLoss: 0.34764\n",
      "Epoch: 5 \tValidation accuracy: 95.120% \tLoss: 0.18129\n",
      "Epoch: 10 \tValidation accuracy: 96.320% \tLoss: 0.13729\n",
      "Epoch: 15 \tValidation accuracy: 96.940% \tLoss: 0.11001\n",
      "Epoch: 20 \tValidation accuracy: 97.340% \tLoss: 0.09324\n",
      "Epoch: 25 \tValidation accuracy: 97.580% \tLoss: 0.08503\n",
      "Epoch: 30 \tValidation accuracy: 97.700% \tLoss: 0.07684\n",
      "Epoch: 35 \tValidation accuracy: 97.800% \tLoss: 0.07433\n",
      "Epoch: 40 \tValidation accuracy: 97.960% \tLoss: 0.07129\n",
      "Epoch: 45 \tValidation accuracy: 98.060% \tLoss: 0.07054\n",
      "Epoch: 50 \tValidation accuracy: 98.120% \tLoss: 0.06801\n",
      "Epoch: 55 \tValidation accuracy: 98.140% \tLoss: 0.06820\n",
      "Epoch: 60 \tValidation accuracy: 98.180% \tLoss: 0.06887\n",
      "Epoch: 65 \tValidation accuracy: 98.180% \tLoss: 0.06861\n",
      "Epoch: 70 \tValidation accuracy: 98.240% \tLoss: 0.06794\n",
      "Epoch: 75 \tValidation accuracy: 98.280% \tLoss: 0.06794\n",
      "Epoch: 80 \tValidation accuracy: 98.240% \tLoss: 0.06927\n",
      "Epoch: 85 \tValidation accuracy: 98.180% \tLoss: 0.07003\n",
      "Epoch: 90 \tValidation accuracy: 98.260% \tLoss: 0.06998\n",
      "Epoch: 95 \tValidation accuracy: 98.240% \tLoss: 0.07077\n",
      "Epoch: 100 \tValidation accuracy: 98.260% \tLoss: 0.07116\n",
      "Epoch: 105 \tValidation accuracy: 98.280% \tLoss: 0.07136\n",
      "Epoch: 110 \tValidation accuracy: 98.280% \tLoss: 0.07139\n",
      "Epoch: 115 \tValidation accuracy: 98.280% \tLoss: 0.07208\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_deep_mnist_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_deep_mnist_model\"\n",
    "\n",
    "best_loss = np.infty\n",
    "epochs_without_progress = 0\n",
    "max_epochs_without_progress = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # if the checkpoint file exists, restore the model and load the epoch number\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"Training was interrupted. Continuing at epoch\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})\n",
    "        file_writer.add_summary(accuracy_summary_str, epoch)\n",
    "        file_writer.add_summary(loss_summary_str, epoch)\n",
    "        if epoch % 5 == 0:\n",
    "            print(\"Epoch:\", epoch,\n",
    "                  \"\\tValidation accuracy: {:.3f}%\".format(accuracy_val * 100),\n",
    "                  \"\\tLoss: {:.5f}\".format(loss_val))\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "            if loss_val < best_loss:\n",
    "                saver.save(sess, final_model_path)\n",
    "                best_loss = loss_val\n",
    "            else:\n",
    "                epochs_without_progress += 5\n",
    "                if epochs_without_progress > max_epochs_without_progress:\n",
    "                    print(\"Early stopping\")\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, final_model_path)\n",
    "    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9799"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "nav_menu": {
   "height": "264px",
   "width": "369px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}