15_autoencoders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 15 – Autoencoders**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 15._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"autoencoders\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "A couple utility functions to plot grayscale 28x28 image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
    "    images = images - images.min()  # make the minimum == 0, so the padding looks white\n",
    "    w,h = images.shape[1:]\n",
    "    image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# PCA with a linear Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Build 3D dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "rnd.seed(4)\n",
    "m = 100\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "X_train = np.empty((m, 3))\n",
    "X_train[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 2] = X_train[:, 0] * w1 + X_train[:, 1] * w2 + noise * rnd.randn(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Normalize the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Going to need TensorFlow..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's build the Autoencoder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2  # codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden = fully_connected(X, n_hidden, activation_fn=None)\n",
    "outputs = fully_connected(hidden, n_outputs, activation_fn=None)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_iterations = 10000\n",
    "codings = hidden\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run(feed_dict={X: X_train})\n",
    "    codings_val = codings.eval(feed_dict={X: X_train})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure linear_autoencoder_pca_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGkxJREFUeJztnX2wXHV5xz9f7r0SgptKVZgGSaJBScy1UWgTJ4qAFOMr\nVGYctS1a8V1aB7QVR6KAdFSsMzKdppWBgEWg4lDFiKSUdAJUbgqOtqk3L2AxCZpQrGNIbsRAbnz6\nx+9s78lmX87Z3fOyZ5/PzM7uOfvbc57d3PPN8zy/5/ccmRmO4zhZcFTRBjiOU11cYBzHyQwXGMdx\nMsMFxnGczHCBcRwnM1xgHMfJDBcYx3Eyo1CBkfQsSddL2iFpr6QfSHp9m/GXSHpc0p7oc2N52us4\nTjqK9mBGgceA083st4DPAN+QNK9xoKSVwCeAs4AFwELgyvxMdRwnLSpbJa+kTcAVZvathv23ANvN\nbFW0/VrgFjP7nQLMdBwnAUV7MIch6QTgxcDmJm8vATbFtjcBx0s6Lg/bHMdJT2kERtIocDPwVTN7\npMmQZwN7Y9t7AQG1HMxzHKcLRos2AECSCOLyNPDnLYbtB+bEtucABkw1OV654j7HqSBmpk5jyuLB\nrAGeB5xvZodajNkMLI1tvxx4wsz2NBtsZqV5XH755YXb4PYMtk1lsycphQuMpK8Ai4BzzeyZNkNv\nAt4raXGUd7kMuDEPGx3H6Y6i62DmAR8g8kYkTUnaJ+mdkk6KXr8AwMzuBr4IbAC2R48rCjLdcZwE\nFJqDMbPHaC9y8ZwLZnYNcE2mRmXAmWeeWbQJh+H2dKZsNpXNnqSUrg6mH0iyKn4vxykLkrABSvI6\njlNBXGAcx8kMFxjHcTLDBcZxnMxwgXEcJzNcYBzHyQwXGMdxMsMFxnGczChcYCRdJOn7kg5IuqHN\nuHdLmo6WD9SXFLwmT1sdx0lHGdo17AKuAlYCx3QYO2FmLioZMDUFk5MwPg4177Dj9InCPRgzu8PM\n1gK/LNqWsjA1BRs3hue8znf66fCa14TnvM7rVJ/CBSYlr5D0c0nbJK2SNFD2JxGOThd7FuIzOQmb\nN8P0NGzZEl47Tj8oQ4iUlPuAcTPbKWkJ8A3gIHB1s8FXXHHF/78+88wzC1+NWheOzZthyRL4t39r\nHoo0u9hf+crkx+gm1BkfD8fbsgVe+tLw2nHi3Hvvvdx7773pP1h0Z6xYh6yrgBtSjH878P0W71nR\n7NtnNjERns3C69FRMwjP69c3/8w995iNj5uNjZktXTrz+cZjjI2FY8TPsW9f+Mzo6JGfTWLvxo3p\nPuMML9E11vE6HagQowkdl4sXQbMwZ3wcFi0K709Pw8UXHx7m1D/zhjeE7XXrjvRQ6p7G2Biccko4\nRvwcvYQ6tVo49uSk52Cc/lG4wEgakTQLGAFGJR0taaTJuNdLOj56vQhYBdyRr7XJaHah12rw5S/D\nSPTNHn74cAGIf+bhh+HYY48McWq1IDr33x+OtW3b4eeIC1DaUMcTvU4mJHFzsnwAlwO/AQ7FHp8B\nTiLcMeAF0bi/Bv4n2vff0edGWhyz/z5hCuqhSmOY07h/166ZEKfVZ9Kcox5irV+fPtRpDL82buzu\nuzvDAQlDJO9olxFTUzPJ2LgnUt8/bx688Y2HJ2yh+WeSnANaJ4B374Y774Q3vxnmzm19rNNPn0n0\ntkpCOw4k72jnApOQfheibdwYwpHp6RDS3H//zGxRP4+3ezcsXAgHDsCsWfDoo+1FJo3AOcOLt8zs\nI2nzE0lqVXrJl6Q53p13BnGB8HzXXa2PUasFUXJxcfqFezAJSONtJK13mZqCBx8ECZYt689F3cwD\nSePBOE5S3IPpI/Up5pGRMD3czttIMlUcn5L++Mf7Z2czD2Tu3CAq113Xu7jkvYTBGXxcYFKgBFU3\nSUKfvEvz586F972vd3HxaWwnLS4wCZicnKk5aaxfaSReq9IqPIqL0Pz5YUap7LQSRfdqnHYMlcB0\nezGkTch2SpbWaiHZOn8+bN8epqvLfoE2+w3cq3E6MTQC08vFkMQrScvOnbBjBxw6NBgrmJv9Br4K\n2+lE4QKTtKNdNPYSSY9L2iPpekljSc/T68WQZAo3jYc0iGFS42/Q76n2OB56VYPCBYaZjnZr2g2S\ntBL4BHAWsABYCFyZ9CTz58No1JxiZKT/F3RaD2kQw6RGOnl23YqEh17VoXCBseQd7d4FrDGzbWa2\nlyBK70l6np074eDB8Hp6Gh57rEuDW9CNhzRoYVIzWnl2jSKxe3d7sYmLkYde1aFwgUnBEmBTbHsT\ncLyk45J8eHw8PMbGgivf76ZK3YQLWYYYRRMXic2b4YwzgtisWAFr18L69TNi0yhG8+dX93cZNkpT\nySvpKuBEM7uwxfv/DXzEzP4l2h4FngEWmNljDWObVvKmWWvTzdqjbtbyVHX9T3zx5Pz5wVObnj58\nzPg4TEyE37mxUnrJkmr+LlUhaSXvILXM3A/MiW3PAYzQvuEImrXMrLvznUha7t9I0uP3+plBoJ6f\nia8cn5wM4WCdrVsP72MTb9lZ1d9lUOm2ZeYgeTC3AD8xs09H268FbjazI+pTe12L1O+Vzk4Q7Yce\ngo9+NAgJBGG5++6Qi5o/P+TF3GMZDAbGg4m6140R62gHTJvZoYahNwE3SrqV0HjqMuDGLGzyJtj9\np1aDs8+Ge+6B22+HF74QTjvtyJ44Li7VonAPRtLlhO50cUOuJIjHFmCxmf0sGnsx8ElgFnA78GEz\nO9jkmD2vpq5qbqRIGkPPL30pLPise4pf+lIQ9OXL/TcvO95wqoLfa9BpDD3XrQurybdsgaOOgqef\nDuNe9KLgzSRdnOl3pcwfb9dQYoa1SrVxWn7ZsiAkq1cfPsP0k5+Eae0kv48X5ZUbF5icGeYLolnl\nb60G73gHLF58+NidO5MV2HlRXrlxgcmZYb8gmlX+1mqhHmbt2hAe1T2cefPybz3q9BfPweRMmu79\nw5hbaHfXhXa/kyfk88WTvDl+r7RCkOSC6LbYryq0q0UaRuEtG57kzYlucipJWj8MeyjVKvQZ5hzW\nIOIC0yNZCcGw5xZatYIYduEdNDxE6pEs74jouYUj8TtQlgPPweScg3EhyA//vYtnYHIwko6T9C1J\n+yVtl/TOFuMul/SMpH2SpqLnBfla2xy/I2L+VPD/xUpSuMAAfwccAJ4P/Anw95IWtxj7dTObY2a1\n6HlHXkY65WBqKjStqjev8iRvuSlUYCTNBs4HVpnZr83sAWAtcEGRdjnl5cEHQ6J3ejo8d9GixMmR\noj2YlxBaMzwa27eJ0B6zGW+R9AtJP5L0oezNc8rOxRe7F1Nmiu4H82xgb8O+vUCzbMZtwLXAE8Ar\ngX+StMfMbmt24GYd7ZzBZ/lyWLAgtOAE+OlPQ8LXG4Jly0B2tJP0cuB7Zvbs2L6PAWeY2XkdPnsp\n8Htm9rYm75V2qYDTG1NTQWS2bg3b9b6+nmDPl0GZRXqE0MVuYWzfUiBJ+ZQBCW5H71SJyUn48Y/D\n65ERuOYaF5cyU6jAmNlTwDeBz0qaLelVwLnA1xrHSjpX0nOi18uAjwJ35GmvUzzxCufx8dBTxikv\nhRfaRfc1ugE4B/gFcKmZ3Sbp1cBdZjYnGncr8DrgWcDPgNVmtrrFMT1EqjBeaFc8Xslbwe/lOGWh\n7zmYKIS5XdKTki6K9l0g6dReDHWcLBjWtqRlI00O5lJggnA/6BOjkOW5ZvbDTCxznC7Ju6WDi1lr\nEodIkp5jZk9Gr08DVprZ56Lt84H5wHJgq5ldmZG9ifAQabjJ88Z5w9oYrO8hUkxczgFeFROXFwHP\nMbMvA38KXNhqwaLj5EGSXjqdvI6kXon3p2lPqmnqSDhOMLO/ie0eB64AMLMDwEPAq/ploOOkpVWz\nqjqdQqhm79cFZ/fuw4Vn2BuDdcTMEj2AdwNvadj3ecJyg/HYvgngz5IeN4tH+FqO05yJCbPRUTMw\nGxsz27ix/fvr15stXRr2HX202ciI2fi42b59Yfy+feEY9e1hILrGOl6LiTwYSW8FzgPOkvQhSX8k\naR3w72Y2bWaT0bhXAM8F1vRTBB2nn3TyOhrfN5sJg55+Gg4dCqHRQw/NNCD3mpzmdFzsKOkE4BQz\nO1/SCwnVs88D/tLMvh0bdzQhVHqdmf06I3sdp2fqIVSrYr3G9yE810WmzlNPDWeCNw0dPRgze8LM\nvhC93m5mS83sRDO7tWHoKuAiM9sp6eSkBiTtaBeNvTpq1/C/kq5Oeg7HaaRWC6IwOdk8kRvvUlgX\nnH/+53AHypGR4OUcc4wneDvRl7VIkj4A3AUclHQi8AcpPp6oo52kDxLWKb0M+F3gzdF5HSc1aWtl\najU4++zQ8Op73wsruJcvT5/gHbaamZ6XCkhaAdxPWNkswirnt5nZNxN8djawB3ipRU2nJN0E/MzM\nPtUw9gHgRjO7Ptq+EHifma1oclzr9Xs51aZftTJp1kVVqWYmt3YNZjZhZqNmNmJmR0XPHcUlIk1H\nuyXRe53GOU5H+jW9nKbh+zDWzBTdDyZNR7vGsXujfY6Tmk61MlkwjDUzRbfM3A/Madg3B2gWoTaO\nnRPta4q3zHQ6Ufc+8jxfu9mrMjOoLTNnA78ElsRyMP8A7GqRg7nBzNZE256DcZyCGIiWmZaiox1w\nE/AxSXMlzQU+BtyYn7VOVRm2mZ08KToHA3ARMBv4OXAL8CEz2yrp1ZL21QeZ2bXAd4AfAf8FfMfM\nrivCYKc65N3aYdjwjnbOUJNna4cqMRAhkuMUzTDO7OSJezDO0ONNxNPjTb8r+L2cclJfUT0+PjwC\n5SGS4+SAJ4nb4wLjOD0wjOX/aXCBcZwe8CRxezwH4zg9MoxJYs/BOE7G1CuAof2K6mGuFHaBcZwu\nSJrcHfYkcKECk7Jd5uWSnpG0T9JU9LwgP2sdZ4YHH0yW3B32JHDRHkyidpkxvm5mc8ysFj3vyMNI\np/qkCWOmpuCSS2YagJ9ySuvk7rAngQsTmKhVw/nAKjP7tZk9AKwFLijKJmc4SRvGTE7Ctm3h9cgI\nXHNN6/xLEY2tykSRHkyadpl13hLdVeBHkj6UrXnOsJA2jIl7JePjsGxZ+/Fp2mpWjSI72qVplwlw\nG3At8ATwSuCfJO0xs9uaDfaOdk5S6oKxZUuyMCbemW7evCOXCVRx6UDpOtpJ2gCcQbjLQCMPAB8F\nHjCzY2Of+Rhwhpmdl+D4lwK/Z2Zva/Ke18E4qeimlqXZXQKgOncOaEfhdTBmdlbsLgONj9cAjwAj\nkhbGPrYUSJpnN8JtUhynZ5KEMY2J4MbQ6utfTz67NCwU3ZP3VoJQvB94BXAnsMLMtjYZey5wv5k9\nKWkZodXmJ83s5iZj3YNx+ko7b2XLlpDsPXgw3PkR4OGHQ7g17B5M0QJzHHADcA7wC+DSek5F0quB\nu8xsTrR9K/A64FnAz4DVZra6xXFdYJy+0qrz3dRU8Fw+/GE4dCi8t24dHHtstZcODITAZIULjNNv\n6h5MPREc90zavVdVXGAq+L2cYmmXCB62BY8uMBX8Xk7xpJmCruJ0dZ3CZ5Ecp2qkqfgd9kWOdVxg\nHCchaSp+h32RYx0XGMdJSJqFi/1c5DjI/WQ8B+M4DbTLnaRJ5vYj8dus/qYM+RxP8lbweznZU7YL\nuqx3nvQkr+N0Qa+5k6kpWL8+PPoR0gx6P5ki+8FcJOn7kg5IuiHB+EskPS5pj6TrJY3lYaczXPRy\nQU9NwYoVcM454bFiRe8iM+j9ZAoLkST9IfAbYCVwjJld2GbsSuCrwFnA48AdwEYz+1SL8R4iOV2T\nNHfSmKvZuDGEV4cOhfdHR4MolCGk6TelD5HM7A4zWwv8MsHwdwFrzGybme0FrgLek6mBztCSdGV1\nY53L+PjMYkeARYuO9IAGeUaoGwYlB7OE0O2uzibg+GixpOPkTrNcTa0GExMzOZiJicNFahiL7wZF\nYBq73+0l9IIZsIjUqQqtcjW1Gpx9dng0ekDDWHyXScvMTt3sooZTadgPzIltz4mO3fL/AG+Z6WRJ\nvG1m0jqXtK05y0TpWmYmNkC6CjixQ5L3FuAnZvbpaPu1wM1mNrfFeE/yOqWkKquuS5/klTQiaRYw\nAoxKOlrSSIvhNwHvlbQ4yrtcBtyYl62O0y+G7Q4DReZgVgFPAZcCfxy9vgxA0knRnRtfAGBmdwNf\nBDYA26PHFQXY7DhOCgoPkbLAQyTHyZbSh0iO41QfFxjHScGwFcr1iguM4yRkGAvlesUFxnESMoyF\ncr3iAuM4CRn01glF4LNIjpOCqhTK9Yp3tKvg93KcsuDT1I7jFI4LjONkgE9nBwaiZaakd0uajpYP\nTEXPaVdkO05iehEIn86eoUgPZhehM92ahOMnzGyOmdWi5/sztM0ZQPrlNfQqED6dPcOgtMx0cmJQ\nXft+eg29CoRPZ88wSDmYV0j6uaRtklZJGiTbB4IkF2lZBaifXkOvAjHodwLoJ5l0tMuA+4BxM9sp\naQnwDeAgcHWrD3hHu/Q0u0jjHfHLdlOyOP3sFtdNt7pmx6jS3QRK1dEuTcvMJB3tmhz/7cBfmNnv\nt3jf62C6oC4g9Yu0UUD6eZfBdrdn7cV+L4LLh6R1MJl4MGZ2VhbHbaDjl3PS0el/7riXcMopsH9/\nuKjTXsxZeUJV8xqqwEC0zJT0eknHR68XEbrh3ZGftcNDu5aOdQFaty5sv+ENxc2yJMkFlTVfNEwM\nRMtM4GzgvyRNAXcCtwOfz99kp1aD2bNh69YgEJs35zPLEheLpMlor0UpAWZWuUf4Wk5W7NplNmuW\nGYTnXbvSH2PfPrONG8NzkrFLl5qNjobne+4Jr8FsbCwcp5GJic5jkrBrl9m113b3HatMdI11vBZ9\nqtdJzc6dwXuBcB/mxx5Lf4w03fUbQyqpswfUj1qU3bth4UL44AfD8+7d7cc3hmQeog1WHYxTEvIu\nJGs837JlnetM+lGLcuedcOBAeH3gANx1V+uxjSHZ7t0eooG3a3C6JI8p4fhUNuQ/BV33YA4cgFmz\n4NFHYW7TW/0dOYW/ejV85CP9mdIvI96uwcmUrG8g1ugRQP/P1ymEmTs3iMp117UXFzjSy3rTm3y5\nALgH45SUfhb1NSOLWpxGr67KhX/e0a6C32tQaVW12xgCxcd0qiru1ZZf/SrU8VQ1hMkaF5gKfq9B\npJWnEN+/aFEYu23bkWO69QCaiVqzcz78cH8FbFhwgang9xpEWoU68f2jo2AWprz74U20ErVGW+oV\nyWawfLkLTBpKneSV9CxJ10vaIWmvpB9Ien2Hz1wi6XFJe6LPjuVlr9M9raa04/sXLYLFi/uXEG21\nFKHRlsWL4eMfD6HSqad2rnNxuiBJNV6/H8Bs4DPASdH2m4B9wLwW41cCjwOLgN8CNgCfa3P8HmoU\n+8+GDRuKNuEw8ranVdVuff93v7shVWVvkvMtXRoqeJcuPfyY8fPEq33B7OSTZ8YO+79ZJyhzJa+Z\nPWVmnzWzn0bb3wW2A6e1+Mi7gDVmts3M9hJabb4nH2t7p5s+GlmStz2tprTr+x966N6+Tnu3K7KL\nn2d8HBYsmHlvx44Zb2fY/836RSnqYCSdALwYaLVsbgmwKba9CThe0nFZ2+YMJkkEq1aD++6Dk08O\neaAlS4a3XiUrCu9oJ2kUuBn4qpk90mLYs4G9se29hH4wNWBPthY6VWbuXPjhD6tbr1I0hXa0kyTg\nHwkCcp6ZHWpxvP8E/srMbo+2fxv4X+B5ZnaEwEjyKSTHyRgbgI52a4DnAW9sJS4Rm4GlhD4wAC8H\nnmgmLtH5vdud45SAIjvafYUwK3SumT3TYfhNwHslLY7yLpcBN2Zto+M4vVFUHcw84ANEnkjsbo3v\njN4/rKOdmd0NfJEwPb09elxRhO2O4ySnkpW8juOUg1JMUzuOU00qLTCSviZpd7QcYZuk9xZoS+rl\nETnYdJGk70s6IOmGAs5/nKRvSdovaXs9RC6Kon+PJvaU8W8m1TVVeB1MxnwOuNDMDkp6CXCfpB+a\n2X8UYMso8Bhwupn9VNKbgG9IGjezLrra9oVdhKrolcAxBZz/74ADwPOBU4HvSvpPM9tagC1Q/O/R\nSBn/ZlJdU5X2YMxsq5kdjDZFqMtZWJAtaZdH5GHTHWa2Fvhl3ueWNBs4H1hlZr82sweAtcAFedtS\np8jfoxkl/ZtJdU1VWmAAJK2W9CtgK7AbaNO6OT8SLI+oOi8Bps3s0di+TYRlIU4TyvI3k+aaqrzA\nmNlFhErhVwPfBJ4u1qLEyyOqTuPyD6JtL9ZvQpn+ZtJcUwMrMJI2SPqNpENNHvfHx0YrzCeAk4AP\nF2lPtDziZsI/yp9nYUsaewpkPzCnYd8cYEhv8NGavP5m0pD0mhrYJG+K5QhxRskoB5PB8oi87CmK\nRwj3JF8YC5OWMrwhYzty+ZvpkrbX1MB6MJ2Q9HxJb5d0rKSjJK0E3gH8a4E2pVkekYc9I5JmASOE\ni/1oSSN5nNvMniK415+VNFvSq4Bzga/lcf5mFPl7tLGpNH8zXV1TSbpSDeKDoPj3EmYEniQkEC8s\n0J55wG+ApwhhwBShi987C7Tp8simQ7HHZ3I8/3HAtwjh0g7g7QX/zRT6e5T9b6aba8qXCjiOkxmV\nDZEcxykeFxjHcTLDBcZxnMxwgXEcJzNcYBzHyQwXGMdxMsMFxnGczHCBcRwnM1xgHMfJDBcYx3Ey\nwwXGcZzMcIFxcidaPX27pCclXRTtu0DSqUXb5vQXX+zo5I6kKwnd67YDvw8sAB4ys2uKtMvpPy4w\nTu5Ieo6ZPRm9Pg1YaWafi73/IkIz8AuLstHpDwPb0c4ZXGLicg6wuEFc/ozQNX9+QeY5fcRzME4h\nRDdZO8HM/ia+38z+FvhqIUY5fccFxskdSe8G9pvZzbF9ny/QJCcjPERyckXSW4HzgB2STiS0gLwA\n+EqhhjmZ4ALj5EZ047BTzOx8SS8E7iD0ef1LM/t2sdY5WeAC4+SGmT0BfCF6vZ1wmxKnwngOxikj\nih7OgOMC45QKSe8H/gJ4maS/kvTiom1yuscL7RzHyQz3YBzHyQwXGMdxMsMFxnGczHCBcRwnM1xg\nHMfJDBcYx3EywwXGcZzMcIFxHCcz/g+y7/7ATvdpeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc742c1ad30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked Autoencoders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's use MNIST:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train all layers at once"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders). We will use ELU activation, He initialization and L2 regularization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer() # He initialization\n",
    "#Equivalent to:\n",
    "#initializer = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "with tf.contrib.framework.arg_scope(\n",
    "        [fully_connected],\n",
    "        activation_fn=tf.nn.elu,\n",
    "        weights_initializer=initializer,\n",
    "        weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n",
    "    hidden1 = fully_connected(X, n_hidden1)\n",
    "    hidden2 = fully_connected(hidden1, n_hidden2)\n",
    "    hidden3 = fully_connected(hidden2, n_hidden3)\n",
    "    outputs = fully_connected(hidden3, n_outputs, activation_fn=None)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = mse + reg_losses\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train it! Note that we don't feed target values (`y_batch` is not used). This is unsupervised training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0319748\n",
      "1 Train MSE: 0.0179479\n",
      "2 Train MSE: 0.0129299\n",
      "3 Train MSE: 0.00947758\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_all_layers.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This function loads the model, evaluates it on the test set (it measures the reconstruction error), then it displays the original image and its reconstruction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure reconstruction_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAGoCAYAAAAHJ+8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH9xJREFUeJzt3VmsXlX1APBdOk/39naklFAZUiYpUwgFjKIkgFERRKLI\ngwY1EBNNJFEDiYn6om/Ki0FFQdEHkzoTJyAgUzGCtJgiUkplUGihw+18e1v4v/4f9tpwTm9Xr/D7\nPa5z9z7nO993z8pJ1tp7wuuvv14AgDxHHO4LAIC3G8kXAJJJvgCQTPIFgGSSLwAkk3wBIJnkCwDJ\nJF8ASCb5AkCySYnnspQWbyUTDvcFvFVt2bKl+qx47bXXwjFHHFF/j+izgl/rPJMnT67GR0dHq/GJ\nEyeGcx04cKAanzAh/mlFx1pjonsQjYmuqyW6/61jre8murb9+/dX433uc2vMWJo7d271w3jzBYBk\nki8AJJN8ASCZ5AsAyTILrgDeUFS8FBU7tfQpRGqJirGi8/QpEmuNGcsioa6fpTWmVXDVp0gq+m4m\nTaqnrOgcrfO0zh99zui3WUr336c3XwBIJvkCQDLJFwCSSb4AkEzyBYBkki8AJNNqBIwrfVqK+rTN\nRFptM9F5oraV1vn7rKHcakOKRG07fdqW+rTtRPdg3759nc8TaX1n0efvc/7W99lqd6rx5gsAySRf\nAEgm+QJAMskXAJJJvgCQTLUzMK5E1amtjRD6LLjfqpCNRNWufa65z8YO0ZhWFXR0zX02g4iqgFuV\n29F30Pr8XSvBo++/lH4V732q57uex5svACSTfAEgmeQLAMkkXwBIJvkCQDLJFwCSaTUCxpWurTGl\nxIvktxboj1pdWufpOqY1V9Qe02r16dMCE92DaK5Wm8+ePXs6xVvnnzp1ajgm0uc+Ry1lrfak0dHR\nTnO90TVU5+r01wDAQZN8ASCZ5AsAySRfAEgm+QJAMtXOwP+E1kL8Y7lJQmszhmhMn+rYqNp28uTJ\n4ZjWsUh0bXv37u0ULyWuhJ4+fXo4Jqp2blUbT5kypRqPfgPRZ2ydv89GHa1K9K68+QJAMskXAJJJ\nvgCQTPIFgGSSLwAkk3wBIJlWI2BcidpZ+ixq32eThJGRkXBMdGzXrl2dzx99nlbbzsyZM6vx1iYF\ns2bNqsbnzp1bjbc2VoiOtVqgonsQbYbROk/0nbU+f9Qe1Gopi87Tujet1qkab74AkEzyBYBkki8A\nJJN8ASCZ5AsAyVQ7l1IeeeSRavzmm2+uxpcsWRLOFVUqfvKTnwzHRFWHURzeyloVwpHWIvmR3bt3\nV+OvvPJKOGbdunXV+PPPP1+NDw8Ph3NNmzatGl+8eHE4ZmhoqBofGBgIxxx99NGdxrSqgKNNF/ps\nOLB169bw2IwZM6rxI488shqfPXt2OFdUodyqXI7uQeu32fUeePMFgGSSLwAkk3wBIJnkCwDJJF8A\nSDahT5VgT2kn6urEE0+sxqPKxrE2ODhYja9YsSLl/Fne8Y53VOM33nhjNX7MMcccwqs5aN1LcnlT\nXnnlleqzYuLEieGY0dHRarxVgRqtLfziiy+GY/7xj39U4w8//HA1/tJLL4VzRes0R2sxlxJf8549\ne8Ix0X2Lqn2jdapbx6KK6lJKOe6446rxjRs3hmMWLFhQjV922WXV+GmnnRbOFa25PNZrO0frWw8N\nDVWfFd58ASCZ5AsAySRfAEgm+QJAMskXAJJJvgCQzMYKpZRf//rX1fjq1aur8VNPPTWca+3atdX4\nX//613DMb37zm2r8T3/6Uzjm2GOPrcY3bNgQjukqKtEvJV78/YUXXuh8nqgF6Stf+UrnuXh7mjp1\najUebQRQStxOMmXKlHBMdCxa8H/evHnhXNHmAa1rXr9+fTXeaoGJNpCINn1obSwRbSzQeia++uqr\n1XjrmRg9384666xq/JRTTgnnilqAjjgifveMWtdabKwAAOOc5AsAySRfAEgm+QJAMskXAJKpdi6l\nnHzyyZ3iLcuXL6/Gr7766nDMt771rWr83//+dzgmqgZ89tln44vrqFX1GVU7R9dVSlxFedJJJ3W7\nMN6WWpvARJWmrYrWqJo/qlwuJd4MIdqcpbVJQlQ53NqMYceOHZ2uq5RShoaGqvGtW7dW463nzsKF\nC6vxJUuWhGNWrVpVjT/yyCPhmKhCO/rOWs+qaEx0/0uJq8dbY7ry5gsAySRfAEgm+QJAMskXAJJJ\nvgCQTPIFgGRajcaBadOmVeN9WnD6tEf1ES2KHi2iXkop5557bjV+8cUXj8k18dYwceLEMZurtTlI\n1DbSOn/U0rN06dJO5yglbo9qtTpFLTXRJg2lxPcgals655xzwrmmT59ejY+MjIRjHnzwwWq8dW8W\nLVpUjS9YsCAcE4nahvbv3x+OGcuWoog3XwBIJvkCQDLJFwCSSb4AkEzyBYBkqp0J7dq1Kzx2xRVX\nVONRBWcppXznO9+pxqMKSvj/Whsr9Klcjo61NmOIjvWZa3R0tPOY6B60Knf37dtXjW/fvr0aHxgY\nCOeKPudzzz0XjnniiSeq8cHBwXDM+eefX40fddRR1XjrtxFVO7eeVdF3MJYV0t58ASCZ5AsAySRf\nAEgm+QJAMskXAJJJvgCQTKsRodtvvz089vLLL1fj8+bNC8dEi8/Dm9FqJ+kzJmoN6bMZQtTO0jr/\nnj17qvFWO0vU6rNz585wTHTNUavT3r17w7k2bdpUjd92223hmCeffLIaP++888Ix7373u6vxo48+\nuhpvtQ1FrVYt0ffZagPrypsvACSTfAEgmeQLAMkkXwBIJvkCQDLVzpT169dX4zfccEPnuVatWhUe\nO/LIIzvPx9tPVLnaqkKOjrWqYCdNqj/+okrXUkrZsWNHNR5tDtJnY4XWZhDTpk2rxlufM6qEnj9/\nfue57r333mr8rrvuCsdMnTq1Gj/77LPDMcuWLavGo++sVdEcVZy3xkTfZ+ve2FgBAMY5yRcAkkm+\nAJBM8gWAZJIvACRT7Uz53e9+V41H1ZillHLVVVdV48cdd9yYXBNvX1GFcJ91mltjosrVVkVr13WS\nW5XLM2fODI9Fos8ZVWGXUsrkyZOr8ejaWv/3//znP6vxXbt2hWPe+973VuOtaueoQjr6PlvrYfdZ\ndzs61uc3GPHmCwDJJF8ASCb5AkAyyRcAkkm+AJBM8gWAZFqN3iZa7QO/+tWvqvGo3L+UUr75zW9W\n463WCjgYfRa1b22SELWNjIyMhGOihf37aG26ENm9e3c13ro30eeMWpDWrFkTzrVhw4Zq/JhjjgnH\nXHLJJZ3HRJ8n+m5a33OfNrA+m3t05c0XAJJJvgCQTPIFgGSSLwAkk3wBIJlq57eJH/7wh+GxBx54\noBr/xCc+EY6xgQLjSVSdOtYVrdF8URX0lClTOp+/tXh/pHWeoaGhajzajOHBBx8M51q/fn01/p73\nvCccc9ZZZ1Xj06ZNC8dEVd19NjyIqspbFeKR1u/JxgoAMM5JvgCQTPIFgGSSLwAkk3wBIJnkCwDJ\ntBq9xaxevboa//znPx+OmTNnTjX+jW98Y0yuCbqIFsJvtdPs37+/Gm8tuB9ptZNE1xCNia6rlPhz\ntlpwBgYGqvFok4TWee6///5qfOXKleFc0ee58MILwzHz58/vdF2llLJnz55qPPqcrU1g+rQURW1D\nrU03Wt9bjTdfAEgm+QJAMskXAJJJvgCQTPIFgGSqnf9HRdWAV199dTXeqvq85pprqnGbJ3A4RAvh\ntyqHoyrYffv2hWOi/4lW5WxU1RxV1Papdm5tEjA4OFiNR/eslFLWrFlTjd96663VeLR5Qinx8+Wd\n73xnOCb6PK1nUlQ53KrqjrSq5CPR99k6f9eqam++AJBM8gWAZJIvACSTfAEgmeQLAMkkXwBIptVo\nHGuVrn/gAx+oxv/1r39V4yeffHI419e//vVuFwaHUNQ2Ey12X0rcztIaEx1r/d9F54niu3btCueK\n2pCijU5Kia/5mWeeCcf86Ec/qsYff/zxavz9739/ONfHP/7xajza8KGUuKWqtbFC9Dmje9ZqtWq1\nbkWi+frMFZ5jzGYCAN4UyRcAkkm+AJBM8gWAZJIvACRT7TyObdmyJTx23333dZrrjjvuCI/NnTu3\n01xwKE2aVH8stRbij6pTW1Ww0XytzRCiMdF5Wov6R4v0z5gxIxwzMjJSjW/YsCEc88ADD1TjJ5xw\nQjX+2c9+Npzr7LPPrsZblcs7d+6sxlufs2u1c+s7i0SbZJTS/t1Eul6DN18ASCb5AkAyyRcAkkm+\nAJBM8gWAZKqdx4Hh4eFqfMWKFZ3n+ulPf1qNn3nmmZ3ngsNh37591XifyuWWPhXSUSV2az3oSFRt\n2/osL7zwQjW+cuXKcEz0fPnYxz5WjS9fvjyca+rUqdX4tm3bwjHRvWnd56jaObpnrfsfHWvd57G8\n5og3XwBIJvkCQDLJFwCSSb4AkEzyBYBkki8AJNNqNA7cdttt1fizzz7bea53vetd1XjXMng4XKJ2\njrH+Dbc2PYhErUZdNwIoJW512bx5czjmoYceqsbvvvvucEy0GcNRRx1VjUefsZRS9u7dW41H7WGl\nlDJz5sxqvPV9dm0dizapKKXfpgt9tDZqqPHmCwDJJF8ASCb5AkAyyRcAkkm+AJBMtXOSdevWhce+\n9rWv5V0IjHNRtfPrr78ejokWwu8zplWFGx3rU6G9Z8+eavz+++8Px/ziF7+oxqMNF0op5fjjj6/G\nlyxZEo6JRPdzaGgoHNPn3nStbG9VNPfZJKHrXH148wWAZJIvACSTfAEgmeQLAMkkXwBIJvkCQDKt\nRkkeeOCB8Nj27ds7z3fyySdX49OnT+88F4wnUTtLq20oWtS+tUB/NF+rBaXrgv+tlpnh4eFqfNOm\nTeGY+fPnV+OXXnppOOb000+vxpcuXRqOiUSfv0/bUOte9hkTib7PVttQ100SSuneHuXNFwCSSb4A\nkEzyBYBkki8AJJN8ASCZaudx7Pzzzw+P3XXXXdW4amfejsayQrpr1WopcRVuqzo3Ov/y5cvDMXPm\nzKnGR0dHwzGnnHJKNT579uxqPNrwoZRSpkyZEh6L7Nu3rxqfPHlyOCbaKCH6blrfc6RV1d5nA4XW\n5g7V83c+AwBwUCRfAEgm+QJAMskXAJJJvgCQTPIFgGQT+pRo95R2IkjQvR+FN2Xz5s3VZ0WrBSh6\njrXGRG1ArRaU1rGaru0npbTbZrq24JRSyqRJ9Y7SPpsH9LnP0T1r3ZtozFi2GrX0mS9qT1qwYEH1\nor35AkAyyRcAkkm+AJBM8gWAZJIvACTLrHYGAIo3XwBIJ/kCQDLJFwCSSb4AkEzyBYBkki8AJJN8\nASCZ5AsAySRfAEgm+QJAMskXAJJJvgCQTPIFgGSSLwAkk3wBIJnkCwDJJF8ASCb5AkAyyRcAkkm+\nAJBM8gWAZJIvACSTfAEgmeQLAMkkXwBIJvkCQDLJFwCSSb4AkEzyBYBkki8AJJN8ASDZpMRzvZ54\nLjjUJhzuC3ir2rZt25g9K444In6/eO211zrFSyll4sSJ1fjrr9cvuXX+aEzLhAn1n93IyEg4ZvLk\nydV4dG2tzx8di+5LKfHnjD5LnzF9rnnKlCnhmAMHDnS6rlLi+zk4OFi9aG++AJBM8gWAZJIvACST\nfAEgWWbBFUBvrQKdSJ+iptaYrvPt378/PBYV6PT5nJMmxY/yrkViLdFcrWvu83miIrF9+/aN2flb\nRVrRdxOd/42uoXqOTn8NABw0yRcAkkm+AJBM8gWAZJIvACSTfAEgmVYjYFyJWjZarRx91mnuuubx\nG11DTZ81j1t2797deUzXdYqj+9IaE52jNV9rTKulp6bVahW1e/VpgWqNaf1uqn/f+ewAwEGRfAEg\nmeQLAMkkXwBIJvkCQDLVzkl+9rOfhcd27dpVjT/22GPhmO9///udzv/Vr341PPa+972vGr/wwgs7\nnQPGQlRR2tqkoM/mAdGx1nmiCt2oond0dDSca9u2bdV4q6L51VdfrcY3bNgQjtmzZ081PmfOnGp8\n9uzZ4VyLFi2qxoeGhsIx06dPr8Zb1cHTpk0Lj9X0+W20KuGj30afivuIN18ASCb5AkAyyRcAkkm+\nAJBM8gWAZJIvACSb0Gdx757STnQ4fe5zn6vGv/e97yVfyZt3yimnVOMPPvhgOGZwcPBQXc7/iu6r\nsvOmbN++vfqsaLVyRMdarSF9NgmIWlp27txZjW/evDmc66mnnqrGW/939957bzX+7LPPhmOiTQei\n9qjjjz8+nOvcc8+txj/4wQ+GY84///xqfNasWeGYSHT/W61GU6ZM6Xye6DfQZzOGgYGB6iBvvgCQ\nTPIFgGSSLwAkk3wBIJnkCwDJbKzQQ1TRXMrYVjWfeeaZ4bErr7yyGl+3bl01/uMf/zic68knn6zG\nV65cGY759Kc/HR6DgxFVIbc6M1qL9HcdE21EUEopW7Zsqcb/85//VOOPPPJIONfDDz9cjT/++OPh\nmOiaL7/88nDM0qVLq/GoonfVqlXhXPfcc081fuyxx4ZjVqxYUY23Nk+Irm3v3r3VeKuiuc/vqU8l\nfNffoDdfAEgm+QJAMskXAJJJvgCQTPIFgGSSLwAk02rU8Pzzz1fjt956a+e5zjnnnPDYH//4x2p8\nxowZ4ZiotD4qhX/mmWfCuR566KFq/NVXXw3HwKESLV7fauWIfvetMdGx0dHRcMzIyEg1Pjw8XI1v\n2rQpnCsas3z58nDMFVdcUY1/6EMfCsdEm6BEGzts27YtnGvNmjXV+MSJE8Mx0QYO+/btC8dELUXR\ndxN9L6XEz8rWmGjTh9bmHl158wWAZJIvACSTfAEgmeQLAMkkXwBIptq5Iar2bS3IHVU133333eGY\nqLKuj9tvv70a/9vf/tZ5rg9/+MMHeTXQ3f79+6vx1qL2UUVtqzo1qnZuLfjfdczxxx8fzhVteHDG\nGWeEY04//fRqfGBgIByzffv2avzvf/97Nf7cc8+Fc0XPvmXLloVjpk+fHh6LRN9bdJ/7bJIwe/bs\ncEz0W5s6dWo4JvrdRrz5AkAyyRcAkkm+AJBM8gWAZJIvACRT7dxw1llnVeOtNY+jdUT7VPz1Ea07\n3VpHFcaTqKJ40qT4cdW1OraUeD3i1nkiUUXtggULwjFRhfK8efPCMdHzJapoLqWU1atXV+N33nln\nNf7oo4+Gc1100UXV+MKFC8Mx0f1sVa93rTZuVRpH5289E6Mxu3btCsd0fcZ78wWAZJIvACSTfAEg\nmeQLAMkkXwBIJvkCQDKtRj0MDg4e7ksod9xxRzW+Zs2aznNdfPHF1XhrUXg4VKJWo9bi+VHbUJ8x\nUTtP69qiuVqidprWXFHrVKttZ+3atdX4008/XY237ln0TFi0aFE4ZufOndV4qz0q+pyjo6PhmEj0\neSZMmBCOiVqXWm1orftW480XAJJJvgCQTPIFgGSSLwAkk3wBIJlq53Hs8ccfD49dd9111fjIyEg1\nvnjx4nCum2++uRqfPHly4+rg0IiqRlsVvVGFcLThQp/zt8yaNasaby34Hy3S31qgP5pv48aN4Zhn\nnnmmGt+wYUM1ftppp4VznXvuudV4q0L8lVdeqcajZ1UppcyYMaMaj6qdW5XLfTa1iX43rd9g19+a\nN18ASCb5AkAyyRcAkkm+AJBM8gWAZJIvACTTajSOrVq1KjzWKtOvuf7668Njy5Yt6zQXHEpRq09r\nw4GozaPVNrR3795qvNViF80XtcDs2bOn81ytxfujlqJf/vKX4Zjf//731fiSJUuq8csvvzyc69RT\nT63Gp06dGo6JnlULFy4Mx0T3c9++fdV463uO2pZaY6Jj0cYab3Ss+ved/hoAOGiSLwAkk3wBIJnk\nCwDJJF8ASKbaeRy49tprq/Gf//znnef64he/WI1/+ctf7jwXHA6tRfIj0YYDrSrc6DytKtjoPFFV\nc6tyOVrwf8eOHeGY6JmwcuXKcExUhXvxxRdX4xdccEE419FHH12NR9XJpcSbLkRVyKWUMjw8XI1H\nldOtSvioQrr13UQbKLQ2T7CxAgCMc5IvACSTfAEgmeQLAMkkXwBIJvkCQDKtRkl27twZHvvDH/5Q\njUcLv5dSyqJFi6rxm266qRqPyv1hvIlaY6L2j1LilqKozaSUUqZNm9btwhrXEG3GMHv27HCuqG3p\niSeeCMfceeed1Xi04UIppXz0ox+txq+88spqPNo8oZS4Paq1gUR0b1otZVHrUvRMHBoaCueKtNqT\nopamVntSV958ASCZ5AsAySRfAEgm+QJAMskXAJKpdk5y1VVXhcc2bdrUeb4vfOEL1fjcuXM7zwXj\nSbRAfas6Naocbokql1sbK0SV2H1EHRD33XdfOObpp5+uxhcvXhyOWbFiRTV+0kknVeOtKvDNmzdX\n4617Fm2gsGXLlnBMVD0dXVvrmqPfTes3E1Vityq0u24I4s0XAJJJvgCQTPIFgGSSLwAkk3wBIJlq\n5zH22GOPVeOtCsbIRz7ykfDYDTfc0Hk++F8WVUGXEleatiqko3Wfo/WLW9cQrZ3euuannnqqGv/L\nX/4SjonWHD7jjDPCMaeddlo13mdt66hCuE8V8O7duzuff+bMmdV4tLZ3KXFVe7R+dCn9qtq7Vtx7\n8wWAZJIvACSTfAEgmeQLAMkkXwBIJvkCQDKtRj1Ei36XUsqNN95YjUdtDS1nn312eCxqbYD/da1F\n+ruK2kxKKWXy5Mmdx0StS1EL0IYNG8K5fvvb31bja9euDcdEbVCXXHJJOGbp0qXVeLThQevzR60+\n0b0sJW7BGRwcDMdEbVDRNbfahvbu3dvpulrnb7WutdrKarz5AkAyyRcAkkm+AJBM8gWAZJIvACRT\n7dzDLbfcEh675557Os937bXXVuM2T+DtqLVIf9cxrerUSGuR/sjGjRur8dWrV4djVq1aVY23Koev\nu+66avyiiy4KxwwNDVXjUbVvq9p89uzZ1XirQrpPp8fAwEA1HlUU79ixI5xreHi40zlKiX9PXTdP\naPHmCwDJJF8ASCb5AkAyyRcAkkm+AJBM8gWAZFqNerjpppvGdL5vf/vb1bjNE3g7ilpdjjii+7tC\nn7alVntS1Gqybdu2avzRRx8N54rakM4555xwTNRSNHfu3HBM1OoTtef0aQ1qfTfR99l6vkUbVWza\ntKkaf+GFF8K5ovakVktVdG2ta+76+/TmCwDJJF8ASCb5AkAyyRcAkkm+AJBMtfM4sHPnzmq8T3Vn\nH9FC8q2qz2gh9ahKsWXPnj3V+M0339x5rpbo87Sq11uL3HNotBbpj0QVra3fcJ9NF6JK4K1bt1bj\na9euDefau3dvNR79P5RSyksvvVSNr1+/PhwTXXNUIdy6/9F9njVrVjgmqhBv3f/omfjnP/+5Gp82\nbVo410knnVSNL1iwIBwTPcdaz4NW9XSNN18ASCb5AkAyyRcAkkm+AJBM8gWAZJIvACTTajQOLFmy\n5LCe//rrr6/GjzrqqHDMyy+/XI1/97vfHZNrytS6/5/5zGcSr4RSSpk0qf5Yam2SEI1ptX9EbTNR\nC1BrvunTp1fj8+fPD+eKWgn/+9//hmN+8IMfVOODg4PhmOhz7tq1qxqfN29eONfw8HA1PmfOnHBM\n1DrVatvZuHFjNR5tBnHiiSeGcy1btqwajzbDKKWUoaGhajy6l6V03wjHmy8AJJN8ASCZ5AsAySRf\nAEgm+QJAMtXOPVxzzTXhsdtuuy3xSsbGLbfcknKeqCK1zwL3n/rUp6rx8847r/NcF1xwQecx5GtV\nLo+OjlbjfSqko40IWvMtXry4Gr/00kvDuaJrXr16dTjmxRdfrMbXrVsXjomqinfv3l2NR1XQpcQV\nvVG1dymlHHnkkdV4tKFLKfF3HVWPH3fcceFc0Xczd+7ccExro4ZIVIkdbTrhzRcAkkm+AJBM8gWA\nZJIvACSTfAEg2YRWBeEYSzvR4fSTn/ykGm9VUPaxZs2aanws11b+0pe+FB474YQTOs932WWXVeML\nFy7sPNc4EJfRclC2bdtWfVZEayGXElfHttbijars9+/fH46Jqp2j9aBb6wc/+eST1fhzzz0XjonW\nVB8YGAjHbN++vRrfunVrNb5ly5Zwrug8ixYtCsdEa0W3qp2j6unoWRGtxVxKfM2texatld3Kl1El\n+MDAQPVH480XAJJJvgCQTPIFgGSSLwAkk3wBIJnkCwDJtBpBP1qNDpHh4eGUZ0XUNtR6JkZjovjO\nnTvDuaI2qJkzZ4ZjogX/o1anUkrZvHlzp3irLTI6f6ttJzrW2vQiaiuLNqM4cOBAOFfUUtbaPKHP\nZi+RwcFBrUYAMB5IvgCQTPIFgGSSLwAkk3wBIJlqZ+hHtfMhsmPHjuqzolXR2kdUbRwtkN8aEz1H\nW9XOUeVua8OBqEK3VbkbXVtUbdxnM4pW5XKfMdG9iT7L8PBwOFd0b/pUW7cqwaPPOWfOHNXOADAe\nSL4AkEzyBYBkki8AJJN8ASCZ5AsAySYd7gsA+P/6tD9GY1ptQ13bWUppt+HUTJoUP2Kj8+/fvz8c\nMzIyUo1v3749HBPNF13brFmzwrmiFpxWq1PX9qzWsWiu6dOnh3P1EZ2/9X125c0XAJJJvgCQTPIF\ngGSSLwAkk3wBIJlqZ2Bc6bPgf1SFG1UUt/Spdo7O31q8f2BgoNuFNfSpwo2qoFvV1tG9iT5/KfGG\nA63vM7pvXSu33+g8keg8rc/Z+q6rc3X6awDgoEm+AJBM8gWAZJIvACSTfAEgmeQLAMm0GgHjSp/W\nkD6itpFWq03XuaI2m1Li1pTWmKjVpzUmMnny5E7nKKXdahM5cOBA57m6bsbQaimLNtdo/c6i+9ln\nM4iIN18ASCb5AkAyyRcAkkm+AJBM8gWAZBO6VmgBAAfHmy8AJJN8ASCZ5AsAySRfAEgm+QJAMskX\nAJJJvgCQTPIFgGSSLwAkk3wBIJnkCwDJJF8ASCb5AkAyyRcAkkm+AJBM8gWAZJIvACSTfAEgmeQL\nAMkkXwBIJvkCQDLJFwCSSb4AkOz/AEv90du6wMxnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc73ceb5c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in multiple graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "There are many ways to train one Autoencoder at a time. The first approach it to train each Autoencoder using a different graph, then we create the Stacked Autoencoder by simply initializing it with the weights and biases copied from these Autoencoders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's create a function that will train one autoencoder and return the transformed training set (ie. the output of the hidden layer) and the model parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size, learning_rate = 0.01, l2_reg = 0.0005, activation_fn=tf.nn.elu):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        n_inputs = X_train.shape[1]\n",
    "\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "        with tf.contrib.framework.arg_scope(\n",
    "                [fully_connected],\n",
    "                activation_fn=activation_fn,\n",
    "                weights_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "                weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n",
    "            hidden = fully_connected(X, n_neurons, scope=\"hidden\")\n",
    "            outputs = fully_connected(hidden, n_inputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "        mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = mse + reg_losses\n",
    "\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        init.run()\n",
    "        for epoch in range(n_epochs):\n",
    "            n_batches = len(X_train) // batch_size\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                indices = rnd.permutation(len(X_train))[:batch_size]\n",
    "                X_batch = X_train[indices]\n",
    "                sess.run(training_op, feed_dict={X: X_batch})\n",
    "            mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        hidden_val = hidden.eval(feed_dict={X: X_train})\n",
    "        return hidden_val, params[\"hidden/weights:0\"], params[\"hidden/biases:0\"], params[\"outputs/weights:0\"], params[\"outputs/biases:0\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train two Autoencoders. The first one is trained on the training data, and the second is trained on the previous Autoencoder's hidden layer output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0129975\n",
      "1 Train MSE: 0.0129642\n",
      "2 Train MSE: 0.0130323\n",
      "3 Train MSE: 0.0133927\n",
      "0 Train MSE: 0.00313369\n",
      "1 Train MSE: 0.00342821\n",
      "2 Train MSE: 0.00363107\n",
      "3 Train MSE: 0.0037358\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150)\n",
    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we can create a Stacked Autoencoder by simply reusing the weights and biases from the Autoencoders we just trained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG1NJREFUeJzt3UtsVeX3xvEXgUKhN0BaaGm5iNqKokBQvAyMAxxoNGIY\noAONmmhMNNFEjSYm6kRnysSo0XgfmGjUeA9eiEhQbqKiRhAQgVIEQUqvtMX/7D9Zz/q5Nz3ntKv9\nfoYr6+yze2pZ7pwn6x3z77//JgAAIjhjqG8AAICsGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBha\nAIAwGFoAgDAYWgCAMMaV8L1YvYFSGDPUNzAaHDp0iL9nFFVdXZ38W+ZJCwAQBkMLABAGQwsAEEYp\nv9MCgCExZoz9esQ74UL1njp1SvaecYb9/37Vq67pyXPyhnr/vNeIhictAEAYDC0AQBgMLQBAGAwt\nAEAYDC0AQBikBwEMG3lSforXm+caKv2XJ2k4duxYU/NSfuq91OtTSmlgYGBQ96Ve713Du191XS8Z\nWawEI09aAIAwGFoAgDAYWgCAMBhaAIAwCGIAGDbUl/eFCGLk6VWBBS8c4dUHcw8nT56UvWVlZYN6\nLy9c0d/fn7k3z+c72FCNhyctAEAYDC0AQBgMLQBAGAwtAEAYDC0AQBikBwGElOdgRZWG85J/48eP\nz3xddY2+vr5MtZRS6u3tNbXu7m7ZO2HCBFObNGlS5vvy0oeqVyUKvbr3OXoJRCXXwZeZOwEAGGIM\nLQBAGAwtAEAYDC0AQBgEMQCElGflkwpCqBBESilVVFRk7lXvV15ebmoqROHxAibjxtl/rtV7paTD\nJOrsLq/uhSjUdfOc08UaJwDAqMLQAgCEwdACAITB0AIAhMHQAgCEQXoQwLCh0mXeWiN1WKKXkOvo\n6DA1b13T33//bWoTJ06UvSplp36Gqqoq+Xp13TyHNXo/r0r0dXV1yV71fl6CMc9qJnUPeVZveXjS\nAgCEwdACAITB0AIAhMHQAgCEMaqCGN9++62prV69WvY2NDSYmrcy5ZZbbjG1qVOnyl6vDoxUeVb3\nqNCFCkaklFJra6upnThxQvaqwIK3fqinpyfT61NKadq0aaamAh5e6EMFMX7++WfZq1ZRTZkyRfaq\nf7+8s7dmzpxpaurnSkmfnaUCMSnp37sX5Mh1NlrmTgAAhhhDCwAQBkMLABAGQwsAEAZDCwAQxphC\nHMqVUcneyHPuueea2s6dO4vyXtXV1bK+bNmyorxfscyZM8fUHn74Ydnb1NRU5LvJZPB7YvCfDh06\nlPnvWaXLVEIvpZSOHj1qaocPH5a9e/fuNbX9+/dnvq73b5868LGzs1P2qntT65a81ytekk6tglJp\nvpR0KnH27Nmy98orrzS1xYsXy151QKb6eVPSh1Z6P5uq19XVyWaetAAAYTC0AABhMLQAAGEwtAAA\nYYyqNU7vvfeeqW3btk32LliwwNS89Srfffedqb3//vuy97PPPjO1uXPnyt49e/bIelbqi9CU9NqW\nffv2Zb6uCmeklNJDDz2U+RoYPdQX9d7qH8ULG6gghReuUPeQ58wotUIpJX+1W5b3T8lf75T1vbwV\nV5s3bzY1FTBJSYfDvLO3VMCjrKxM9qrP0VuHxRonAMCIxNACAITB0AIAhMHQAgCEwdACAIQxqtKD\nLS0tmWqehQsXyvqqVatM7amnnpK9f/zxh6l56cHdu3dnvjfFS/Wo9KB3D2pNTXNz86DuC6OLSoZ5\nyVaVkKupqZG9am2Yd3ihSs55CUZV91KC7e3tppYnqVhbW2tqXqLw0KFDpvbxxx/LXnUYppdgzEMd\nnOl9jhMmTDC1QqwN5EkLABAGQwsAEAZDCwAQBkMLABDGqApilJJad5JSvhBDnpBIHmrt1JEjR2Tv\nJZdcYmrLly8v+D1h5FKBIC8kVFlZaWpTpkyRvWeeeaapeauK1EohFaJISQch1FlWKaXU3d1tat4K\nJKWhocHUvH871q1bZ2reOV1q1Zp3nlZjY6Opqd9DSvpzVOEMr65enxdPWgCAMBhaAIAwGFoAgDAY\nWgCAMBhaAIAwSA+OYF6y6IYbbjA173C2Z555xtSyHnwHpKQTY16KTKX/vJVPKmXn/beZ59DJ6dOn\nm5p3v2pdkrqH6upq+XpV9xKQ6lBY72+8vr7e1BYvXix7VVLQSzCqz8FbUaX+TfF686x34kkLABAG\nQwsAEAZDCwAQBkMLABAGQYwR7JVXXpH1trY2U/POIfJWvwBZqS/fveDP5MmTTa2np0f2qgCB90W/\nWq3khQ3UPXiritQ1BhtWaG1tlb3bt283NS+0oVbAnX/++bJXrclSZ2GlpH82L+Sier3fu/f5yOtm\n7gQAYIgxtAAAYTC0AABhMLQAAGEwtAAAYZAeHCF27dplavfff3/m12/YsEHWZ8yYcdr3BBSCl2RT\nvBRaf3+/qXkHUao03PHjx2XvP//8k+m6XmpOvf6jjz6Svb/88oup9fX1yd758+ebmjrcMiW9JstL\nVnrvp6jVTHlSgh6etAAAYTC0AABhMLQAAGEwtAAAYRDEGCE++OADU/O+NF25cqWpzZs3r+D3BBSC\nFyBQX+p7X/SPHTs2Uy0lHdrwqMCCWgPlhUm2bdtmanv37pW9O3fuNLWLL75Y9i5atMjU6urqZK9a\nh+V9BurfFG+Nk/d7GyyetAAAYTC0AABhMLQAAGEwtAAAYRDECMYLV7z77rum5n35++STT5qa96U0\nkFWebQdqW0JK+twqr1ed1+T1qnqec6C8v4+pU6ea2qRJk0yts7NTvl6dkbVmzRrZq/6ely1bJnvV\nRoza2lrZqz4HL0ShPhuvV31m3maQPHjSAgCEwdACAITB0AIAhMHQAgCEwdACAIRBejCYl156SdbX\nrVtnajfddJPsZWUTSkkl91QKLSWdOPMSs+rcqt7eXtnb1dVlauocqZRSqqioMDXvfCl1jfLyclP7\n6aef5Ou//vprUzt48KDsXbBggaldddVVsrepqcnUvHSnWjulUpxe3ftdqqSgdw9e6lPhSQsAEAZD\nCwAQBkMLABAGQwsAEAZBjGFMnbVzzz33yN6amhpTe+KJJwp+T4DH+zJdffnurfNRQQrvy/v29nZT\n88IVqu6dGdXT02NqXjBBBUf2799vau+88458/caNG02tqqpK9q5YscLUGhsbZa+6X+9zVJ9DR0eH\n7FU/rxfEUPU8q748PGkBAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA9OAx0d3fL+qpVq0zNSzHdfPPN\npsa6JkSjkmxeKlEl2bz0oEoreqk3tbLJO1D1+PHjprZp0yZTU4e0pqQPh1y5cqXsXb58ualVV1fL\nXsU79FKtyfIOdlTX8A7IzJMeZI0TAGBEYmgBAMJgaAEAwmBoAQDCIIhRYuoL4WuuuUb2/vbbb6bW\n0tIiex9//PHB3RhQQl6gSP19eL0qMOGFDfKc7ZTnum1tbab2wQcfmNq+ffvk6xcuXGhq1157reyd\nOnWqqXnrsNT9ej+vF6QYbK8KYuQJXLjXHfQVAAAoEYYWACAMhhYAIAyGFgAgDIYWACAM0oMldvTo\nUVNbu3Zt5te//vrrsq6SRcBw4CXcFJU489YtKWVlZbLurSVS1Fojta4ppZS++OILU/vqq69MTR3S\nmlJKV199taktWrRI9lZWVma+L/WZe59BnpSfWmeV5/dbCDxpAQDCYGgBAMJgaAEAwmBoAQDCIIhR\nJN4XpMuWLct8jTfeeMPUvC9pgeFKrQ/y1gGpXu8sKxUA8IIY6rrq7K6U9BlXO3bskL3vv/++qR07\ndszUVqxYIV+vzsiqra2VveqsMHWvKenVVypgkpIOaHi/nzzrsBTWOAEARhWGFgAgDIYWACAMhhYA\nIAyGFgAgDNKDRfLyyy/L+u7duzNf44orrjC1PEkdYLjy/jtWCTlvTZBKuHnpNFXv6uqSvQcOHDC1\nL7/8Uvbu2bPH1CZNmmRqs2fPlq+fNWuWqalDKFPS6T/v51U/m5fCVPKszsrzmRcCT1oAgDAYWgCA\nMBhaAIAwGFoAgDAIYhTAzp07Te2xxx4r/Y0AwamARp4v9NX6opR0MKGjo0P2qpVNra2tsnfmzJmm\nps7Oam5ulq9XYRJvNVNvb6+peauZKioqMveq0IUXlClWuCIPnrQAAGEwtAAAYTC0AABhMLQAAGEw\ntAAAYZAeLIB169aZWnt7e+bXt7S0yHp5eflp3xMwnOVJp3krhdQhjt3d3bK3p6fH1Lw1Tqo+ZcoU\n2atWrc2YMcPUvIMdVdrxxIkTsvfkyZOmpg5wTEmnEr3PUdXzrGYqdaKQJy0AQBgMLQBAGAwtAEAY\nDC0AQBgEMUrssssuM7U1a9bIXoIYGKm8L+9VQEMFLlLSa4nUeVwp6TOqvBCDWrnU1NQke9UZVWqF\nkneWlQpMeOeHDZYXflHvN5zP7eNJCwAQBkMLABAGQwsAEAZDCwAQBkMLABDGmBKu4Bj608MwGgzf\n2NMIcujQoSH/e1YrkPKkEr21Ruq6XprOu0ZW3qGViroHlT7M8/qUhsfBjkpdXZ28YZ60AABhMLQA\nAGEwtAAAYTC0AABhlDKIAQDAoPCkBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EF\nAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAI\ng6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACGNc\nCd/r3xK+F0avMUN9A6NBa2srf88oqvr6evm3zJMWACAMhhYAIAyGFgAgjFJ+pwUAw96pU6dM7Ywz\n9P/fjxljv3b591/7dZ+qea/H/8aTFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM0oMARiWVEkxJJ/36\n+vpk7/jx401NJQ3Hjh2b+b5Onjwp6729vZnvq6yszNS8pKK633Hjhu9o4EkLABAGQwsAEAZDCwAQ\nBkMLABDG8P22LZA333zT1Do7O2Xvli1bTO2FF17I/F6PPvqorF911VWmduWVV2a+LjASDAwMyLoK\nLHiBh56enszvp8INWcMZ3nt5/3aon80LeNTW1ppaeXm57FXBE+9zVD9HqVdR8aQFAAiDoQUACIOh\nBQAIg6EFAAiDoQUACGOMdzhZEZTsjYrl7rvvlvXnn3++xHdinXfeeab2zTffyN7q6upi385Q4lS9\nEmhtbS3Z37O3bkkl3Lq7u2XvP//8Y2o7duyQvXv37jW19vZ22fvXX39l6lWJwpRSqqmpMTXv77O5\nudnUzjnnHNk7ffp0UzvzzDNlb39/v6l5c0ElLr1kZJ7VVUp9fb38W+ZJCwAQBkMLABAGQwsAEAZD\nCwAQBmucHCp0UYjAxaJFi0ztxhtvlL07d+40tVdffVX2/vLLL6b29ttvy97bb7/9f90iMGRUAMBb\nt6RCFyoYkVJKv/76q6lt3bpV9u7atcvUjh07JntV6GLatGmmNmnSJPn67du3m1pTU5PsVQENFcBK\nSQc8Jk6cKHtVEOPEiROyV/1+vKCMCmh4vXlCGzxpAQDCYGgBAMJgaAEAwmBoAQDCYGgBAMIY9enB\nP//8U9ZffPHFzNdYunSpqX366aeyV6WIysrKZK9aU/P777/L3vXr15vakSNHZC8w1LxDBhV1gGNK\nKXV0dJial3pTqULv8EKV3rvwwgtl7wUXXGBqVVVVprZx40b5+s2bN5ua+rlS0gdGegdWqjSel8JU\nn4PXq4wbp8eI+h0X4sBInrQAAGEwtAAAYTC0AABhMLQAAGGM+iCGF1ZQ60pU4CKllD7//HNTq6io\nGNyNpZReeeUVU9u0aVPm119//fWDvgcgq0Kczaeu4a34qaysNLXe3l7ZO2/ePFPz1iXNmDHD1Lx1\nSbW1taZ24MABU9uyZYt8fVtbm6l5K6Muv/xyU/PO6VLX8D7H8vJyWVdUSMT7vU+ePNnUvHVWefCk\nBQAIg6EFAAiDoQUACIOhBQAIg6EFAAhj1KcHFy9eLOsqVeitW8qTvslDrZLKs14FKKU8K3q81Ju6\nhjpM0LuGdw8qyeb93TY0NJiaOoAxpZQ6OztNTa2M8hKBKmU8Z84c2Ttz5kxT8w52HGyS03u9+jfQ\nS2wWC09aAIAwGFoAgDAYWgCAMBhaAIAwRn0Qw+N98VoMr7/+uqz/8MMPma+xfPlyUzvrrLNO+56A\nQlFBCi8woVYNeWdvqet6wQQVIPBWCuUJGxw8eNDUNmzYYGp5ghgqcJFSSueee66peT9vnpCK4n02\n6nPwrqvO2Tp16pTs9VZMKTxpAQDCYGgBAMJgaAEAwmBoAQDCYGgBAMIgPVhi33//vandeeedslcl\ndbxk0erVq03NW5UDlJJKjHmrmfr6+kzNWynU3d1tankOOvSu29XVlamWUkoffvihqW3bts3U/v77\nb/n6+fPnm9qSJUtk79y5c03NWy2nfrZC/HugEoHqd+b1FgJPWgCAMBhaAIAwGFoAgDAYWgCAMAhi\nlJha8ZLnPJq77rpL1s8555zTviegmFTowlvNpOre6h8VNvDCFf39/abmhTZ6enpMbceOHbJ3y5Yt\nprZp0yZTu+iii+TrZ82aZWrnn3++7FWrlbx/O9Tn6IU2JkyYYGpeuEL9Lr1e9bvIs67Jw5MWACAM\nhhYAIAyGFgAgDIYWACAMhhYAIAzSg0Vy2223yfpbb72V+Rr33XefqT344IOnfU/AUFApMm+NU55e\nlXrzetVhid5qJrVy6csvv5S9Kg2s7ksd9piSXi81depU2atSlJ2dnbJXJQVVKjIlvd7JO1xSfWZe\nr0ps5jmI0sOTFgAgDIYWACAMhhYAIAyGFgAgDIIYBdDR0WFqn3zyiexVX4bW1dXJ3kceecTUvFUs\nwHClwhXeaiYVCjh58qTsVYEHLxSgVg151z148KCpbdy4Ufaqn6OxsdHUZs+eLV+/dOlSU/POvVIh\niOPHj8ve6upqU8sTfvFWM6l/v7zVTN77DRZPWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwSA8WwMqV\nK03tr7/+yvz6e++9V9a9dS5AdF6yTNW9NN24cdn/+VLJu8OHD8verVu3mppKCKeUUm1trampRGBL\nS4t8vToEcvLkybL32LFjpuZ9BkePHs30XinpFKX3mavfj3fwZiFWNsl7KMpVAQAoAoYWACAMhhYA\nIAyGFgAgDIIYOWzZskXW165dm/kaK1asMLX777//dG8JCMkLYqiVQF7YIM8X/W1tbaamAhcp6ZVN\n3lqjhQsXmlp9fb2pVVVV/dct/j9vFZWqe70qxOV9XurcK++cLtU7MDAge70wx2DxpAUACIOhBQAI\ng6EFAAiDoQUACIMghqO7u9vUHn74YdnrncujLFmyxNQ4IwujjXeelgoL5Nme0d7eLnt37dplatu3\nb5e9aptEQ0OD7F2wYIGpNTc3Z3692iZx5MgR2as+M+/fDrVVo7e3V/aqbR/qrLKU9L+LeUIm3u89\nz9lbPGkBAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA96HjuuedM7Ysvvsj8+ttuu03WWdkE+GmxPEla\ntT5IJf9SSmnPnj2m9vPPP8tetTZq9uzZsve8884zNXVulZfGU+uhvJSfukZ5ebnszfpeKenP0Ttr\nTK3Z8s7/UsnIQpyxxZMWACAMhhYAIAyGFgAgDIYWACAMghiORx55ZFCvf/rpp2WdlU0YbfJ8+a7O\na1Jf/nu93gokFcTYu3ev7J03b56p1dbWyt6amhpTU/eb5yyrEydOyF4VuvCuq4IU+/bty9zrnYU1\nZcoUU6uoqJC9KjjinY2WB09aAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDBIDxaJOlgtpXyHneWhkjpe\n6kqtbfFWxyjqILiUUlq9enXmayje/aokp5duwvCjDv6bOHGi7FVpOO/gQHX4aldXl+xVh0N6f6Nq\nFZSXStyxY4epqfv1/r7UQZRewq6lpSXzdTds2GBq+/fvl73qGuq9Ukpp2rRppub9LtW/daxxAgCM\nKgwtAEAYDC0AQBgMLQBAGAQxiqShoaGk73fXXXeZWn19vexta2sztWeffbbg91Qo6rO84447huBO\ncDrUl+9egEB9qZ8nvHTgwAFZV0EKLxSgrrF582bZe+zYsUy9ra2t8vUqTOKdvbV7925T84InP/74\no6mpMEpKKV100UWm5q2bU0EM735VoESdsZUXT1oAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMEgPOm6+\n+WZTe/nll4fgTrJ57rnninJdlQDy1i0pt956q6xfeumlma9x+eWXZ+5FDF4iUKXLvDVO6hpnn322\n7D3rrLNMbfLkybJXHYr4xx9/yN7169ebmvqb8VYzqZ9tzpw5srezs9PUvPSgSmc2NzfL3vnz55va\n3LlzZa9Kd3opzEIkBRWetAAAYTC0AABhMLQAAGEwtAAAYYwp1pdlQsneqFhee+01WVerWPL44Ycf\nZH2wq5UeeOABWVdfvHquu+46U6utrT3teyqBwR/Yg//U2tqa+e85zxlKan2Qt1Kop6fH1FSIIqWU\n/vzzT1PzztM6ePCgqalzs1LSK5tUCKK8vFy+fvr06aZWWVkpe1XwxDvLqrGx0dRmzJiRubeqqkr2\n1tTUmJoXaPHCJ1nV19fL/3B40gIAhMHQAgCEwdACAITB0AIAhMHQAgCEQXoQIw3pwRLIkx7MQyXk\nvLVhqu4ledVBhV4qUa1G6uvrk73qYEV14KS3ikqtZvKo+62urpa948ePNzUvzac+c69X3YO3kivP\nujeF9CAAIDyGFgAgDIYWACAMhhYAIAyCGBhpCGKUwGCDGF4wQa188r7oVyuMvACBCibk4d2Dej/1\ns6mVUymlNDAwYGpemESFQbz7Utfw/q1X1/BWb3nvVwwEMQAA4TG0AABhMLQAAGEwtAAAYTC0AABh\nDO6ULgA4DV4KTSXcVMIuJZ2m83pV3UvTqV4vlahWPqkDH720pLceSvGuoahkpfd6Vfc+R/WZ5Tnk\nsxB40gIAhMHQAgCEwdACAITB0AIAhEEQA8Cwob7U977oVwGC/v5+2asCD955Typs4K1hUoESdUaW\nt0ZK1Xt7ezPflxcmUZ9Dns9xsGdhFRNPWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwSA8CGNbyrAnK\nsx7KSxrmuW7W3kKsl8qT6MtzuG8pD3YshFh3CwAY1RhaAIAwGFoAgDAYWgCAMMbk+cIOAIChxJMW\nACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAg\nDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIIz/\nA5HF/ehEPsQ3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc73d00a828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in a single graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Another approach is to use a single graph. To do this, we create the graph for the full Stacked Autoencoder, but then we also add operations to train each Autoencoder independently: phase 1 trains the bottom and top layer (ie. the first Autoencoder) and phase 2 trains the two middle layers (ie. the second Autoencoder)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "weights4_init = initializer([n_hidden3, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "\n",
    "with tf.name_scope(\"phase1\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase1_outputs = tf.matmul(hidden1, weights4) + biases4  # bypass hidden2 and hidden3\n",
    "    phase1_mse = tf.reduce_mean(tf.square(phase1_outputs - X))\n",
    "    phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n",
    "    phase1_loss = phase1_mse + phase1_reg_loss\n",
    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
    "\n",
    "with tf.name_scope(\"phase2\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase2_mse = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
    "    phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n",
    "    phase2_loss = phase2_mse + phase2_reg_loss\n",
    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=[weights2, biases2, weights3, biases3]) # freeze hidden1\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.0077799\n",
      "1 Train MSE: 0.00749564\n",
      "2 Train MSE: 0.00773762\n",
      "3 Train MSE: 0.00780849\n",
      "Training phase #2\n",
      "0 Train MSE: 0.00227434\n",
      "1 Train MSE: 0.00267451\n",
      "2 Train MSE: 0.00249404\n",
      "3 Train MSE: 0.00275205\n",
      "Test MSE: 0.00303329\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "mses = [phase1_mse, phase2_mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                sess.run(training_ops[phase], feed_dict={X: X_batch})\n",
    "            mse_train = mses[phase].eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG1RJREFUeJzt3VlsVeX3xvEXASmUAm0NFEURVBRxQkVEjeHnLESJGhKH\nC42aaEw00UQNJibqjZfKjVM0OF6YGHFWJoMRpwgiDqhhjIoUoS1TW0op/K//Wc8y++UMZbXfz+XK\nu/fZ5xxOFzv7yXoHHDp0KAEAEMFRvX0BAAAURdMCAIRB0wIAhEHTAgCEQdMCAIRB0wIAhEHTAgCE\nQdMCAIRB0wIAhDGoiq/F6A1Uw4DevoD+oK2tjd8zKqq+vl7+lrnTAgCEQdMCAIRB0wIAhFHNZ1oA\ncMQbMMA+SvF2w1BrK0Vdw5FwXdXGnRYAIAyaFgAgDJoWACAMmhYAIAyaFgAgDNKDAPo8laYbOHCg\nXNvT01Po+JRSOuoo+/9+tfbgwYOFr8tz4MCBQq+fUkrd3d2FX2vQINsGvFSikrO2HKlG7rQAAGHQ\ntAAAYdC0AABh0LQAAGEQxAAQUk4AQFGBC48XICg68ikniJET+vCo1/OuQQU8Bg8eXPi1cq631O8s\nJe60AACB0LQAAGHQtAAAYdC0AABh0LQAAGGQHgQQkkqneQm5HDljnNTr7d+/39RUQs8776hRo+Ra\ndV4vUeiNqCp6Dd7nmHPeciQFFe60AABh0LQAAGHQtAAAYdC0AABhEMQAUHVesEE9vM8JQXijmVSA\nwAtHqHMMHz5cri1qyJAhsq7CFTt37pRr6+rqCr/e0KFDTc37bNTeW5UKXHhrc/bZ4k4LABAGTQsA\nEAZNCwAQBk0LABAGTQsAEAbpQQBV540JUmOJVMIuJZ2G2717t1yrEnJeYq2+vr7wNaiUnUoKepsq\n1tbWmlrOKCovjafO4SUYc5KV6vvZt2+fXKveM5tAAgD6FZoWACAMmhYAIAyaFgAgjH4VxPj2229N\nbf78+XLtcccdZ2pqNEpKKd1+++2m1tDQINd6daCvUg/fvWCDCgB0dnbKtX///bep/fbbb3Ltjh07\nTK2rq0uuPfHEE03N2+NqzJgxpqZ+414IQv1N8d6vCpN4IQj1OXphEBU8qampkWvV+/DCFerzHTRI\ntxzGOAEA+iSaFgAgDJoWACAMmhYAIAyaFgAgjAHlGKtRUNVeyHPqqaea2rp16yryWiNHjpT1Cy+8\nsCKvVykqSTVv3jy59oQTTqjw1RRSPIaEw9bW1mZ+zzl/Szo6OmRdJef+/fdfuVYlBX/99Ve5ViUN\ncxJyXvKupaXF1NQIJS+pqMYitbW1ybXHHnusqQ0bNkyuVaOZvLWXXnqpqV1wwQVyrdoMM2fElXq/\nKen0YH19vfwtc6cFAAiDpgUACIOmBQAIg6YFAAijX41xeu+990ztxx9/lGunTJliat5D3u+++87U\n3n//fbl20aJFpjZhwgS5dtOmTbJelDcyZezYsab2119/FT6vCmeklNKjjz5a+BzoP3JG9ChHH320\nrKuw08SJE+Va9Vvwfh/KP//8I+vNzc2mpsY4tbe3y+O3bt1qal5YQb1fb9+rjRs3Fjo+JT2ybsaM\nGXKt+sxU4CIlHUjJ2SvMw50WACAMmhYAIAyaFgAgDJoWACAMmhYAIIx+NcapmrzN2TZv3mxqXnpQ\nJYByeKkrlR70rmH79u2mtnDhQrl2zpw5GVdXMYxxqgI1xiknJeilyNRIIG8EktrY0UvpKd6mrmo0\nU2trq1yrRiOp8VBeym/btm2m5qX8GhsbTe3jjz+Wa1V90qRJcu3VV19tajfeeKNcqzbD7OnpkWvV\nppU5/YYxTgCA8GhaAIAwaFoAgDBoWgCAMPrVGKdq8vbqOe200wqfY/LkyeW6nP9HjZ1SD7VTSmn6\n9OmmdtVVV5X9mhCf95BdBTS80IYKR3jjlmpra01NPfz3zuEFlXICBGrkkgpn7N27Vx6vqPeVkg5z\nLF68WK7dvXu3qXkBj9NPP93UvNFM3uerVCrkx50WACAMmhYAIAyaFgAgDJoWACAMmhYAIAzSg32Y\nN9LmhhtuMDVvrM6zzz5rat74G/RvXiJQpci8dJpa66UHVXLPO6+qe6OV1Ot5aWD1u1HnHTJkiDxe\nfWbe+12/fr2prVu3Tq5V79cb46Q2gVQJyJT0e8v53j05I8C40wIAhEHTAgCEQdMCAIRB0wIAhEEQ\now979dVXZb25udnU1F49KaU0fvz4cl4S+jDvwbuqe8GfnAfyKhSg9uNKSYc2VC0lHYTw9sdT16ve\nr3e8Cjxs3bpVrl2wYIGprV27Vq695JJLTO2KK66Qa8eMGWNq3vejAh4547vKgTstAEAYNC0AQBg0\nLQBAGDQtAEAYNC0AQBikB/uIDRs2mNpDDz1U+PhvvvlG1puamg77moBcKonW1dUl1/b09Jialwgs\nmvLzXs/bMFIlDdVGid5YJHW9K1eulGs//vhjUxs8eLBce91115laThLYGyWlPkf1PaSk35uXSszB\nnRYAIAyaFgAgDJoWACAMmhYAIAyCGH3Ehx9+aGrqgXBKKc2dO9fUJk6cWPZrAlLSD+9zHsh7e1Gp\nAIB3XhW68MIVinfezs5OU1MBBC+IoUaqeXtkqc9x2rRpcu3UqVNNzdsHL2fckvrMvSCG+szKsfcW\nd1oAgDBoWgCAMGhaAIAwaFoAgDAIYgTjhSsWLlxoat4D7KefftrU1D45QDlkPWTP2PdKndd7LW/C\nQ1EqcJGSnkihfnfePl/r1683tZ9++kmunTBhgqldfvnlcu3YsWNNzQtBqCCF9/cgJ1SjzqH2QPuv\na1O40wIAhEHTAgCEQdMCAIRB0wIAhEHTAgCEQXowmFdeeUXWv/zyS1O79dZb5VpGNqG3eSk/lWTz\nkmU5SbZSR0l516BGI9XW1prapk2b5PHvvvuuqa1evVqunTVrlqnNmDFDrq2pqTG19vZ2uVYlkr3E\npkoEep+NlxQsFXdaAIAwaFoAgDBoWgCAMGhaAIAwCGIcwX788UdTu//+++XaUaNGmdpTTz1V9msC\ncqkH9V4QQ631RiDl7IeVM0pKBTS8MVBqbWtrq6ktXbpUHr9kyRJT897XpZdeampNTU1ybVdXl6l5\n+16psVM5Y5y80Ib63soxLo47LQBAGDQtAEAYNC0AQBg0LQBAGDQtAEAYpAePAN4Gc7fccoupeQmg\n2267zdQY14QjgUrueaN/VBrPS6ep34KXvFPn9ZJsKnmnNntMSb8Plfr95JNP5PHqtz937ly59rzz\nzjM1LxWpRih5fzvUyCdvreKlO73vrVTcaQEAwqBpAQDCoGkBAMKgaQEAwiCIUWXqgfDs2bPl2j/+\n+MPUJk+eLNc++eSTpV0YUCHqgXzOWCVvXyZ1Xi9codZ651VhDi/gsWXLFlNbuHChqX300Ufy+Cuv\nvNLUvL8HjY2NpuYFJlQ4ImcUlReUUbzvUtUZ4wQA6FdoWgCAMGhaAIAwaFoAgDBoWgCAMEgPVpna\nIG758uWFj3/jjTdkvaGh4XAvCai6nPSgt7a7u9vUvJSfSg96aTpVV6+VUko///yzqX3xxRemNmXK\nFHn8zTffbGqnnHKKXKs2a9y9e7dcO2zYMFPz0pKq7qX81HfhjbhSqcRy4E4LABAGTQsAEAZNCwAQ\nBk0LABAGQYwK2bVrl6xfeOGFhc/x5ptvmtrUqVMP+5qA3qBGAuWMCVL7PaWkH/TnjHzyAh7qvJs3\nb5ZrV6xYYWpqtNNdd90lj//f//5nanV1dXJtR0eHqXkhCPV+vTCJer85o5k8KsyR8717uNMCAIRB\n0wIAhEHTAgCEQdMCAIRB0wIAhEF6sEIWLFgg6xs3bix8jksuucTUypG+AapJJc68f8cqDZezNmd0\nkHfebdu2mdqiRYvk2vfff9/U1PudNGmSPF6NX/PGS6n0n/d+VYrSSxp2dXXJulKOTRxLxZ0WACAM\nmhYAIAyaFgAgDJoWACAMghhlsG7dOlN74oknqn8hwBEoZ/SP0tPTI+sqQOCFGFToorOzU65dvXq1\nqX399ddyrQpWXXPNNaY2btw4ebwazaRGMKWk34MXxFCfec734F1DzjisSuFOCwAQBk0LABAGTQsA\nEAZNCwAQBk0LABAG6cEy+PLLL01t9+7dhY+fPHmyrA8dOvSwrwmIKGfkk0oKemvV+CEvldje3m5q\nQ4YMkWtnzpxpamr8WlNTkzxejVbyxip5mzgq6nPMGcHkfY7VTgoq3GkBAMKgaQEAwqBpAQDCoGkB\nAMIgiFFlF110kaktWbJEriWIgf4m50F/zt5yan8pNUIppZTq6+tNTYUrUtIBi1NPPbXQOb3r8gIT\naoSS9xmo8U45IZUjGXdaAIAwaFoAgDBoWgCAMGhaAIAwaFoAgDAGVHEsR+/P/0B/UDxShsPW1tYW\n/vfsbaCoxih5a9V4JzWayTtejaLy/iZ75+ir6uvr5W+ZOy0AQBg0LQBAGDQtAEAYNC0AQBjVDGIA\nAFAS7rQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAA\nYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQ\ntAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABhDKriax2q4muh/xrQ2xfQH7S0\ntPB7RkU1NjbK3zJ3WgCAMGhaAIAwaFoAgDCq+UwLAMpmwIDSHl8eOqQfy1XivAcPHiz8Wt51HXWU\nvcfwrrWnp6fQ8R7vvN61VRN3WgCAMGhaAIAwaFoAgDBoWgCAMGhaAIAwSA8C6PNUes9L06nkXc55\nhwwZYmqDBuk/teq1uru75drBgwcXev2UUjpw4EDhteq8AwcOlGsrldjMwZ0WACAMmhYAIAyaFgAg\nDJoWACAMghhl8NZbb5lae3u7XLtq1SpTe+mllwq/1uOPPy7rl112manNnDmz8HmBaFSwwAtR7Nu3\nz9S8UEBXV5ep7d+/X67t7Ow0NfXb9/4eqGs45phj5Nr6+npT8wIeOSOf1DWoMEnueXNGVOXgTgsA\nEAZNCwAQBk0LABAGTQsAEAZNCwAQxoAqburV+7uHlei+++6T9RdffLHKV2KdfvrpprZixQq5duTI\nkZW+nN5U2pwZFNLS0lKR37NKnHmJQJXo27Ztm1y7c+dOU1uzZo1c+9NPP5na77//Ltf++eefptbR\n0WFqo0ePlsc3NDSY2uzZs+XaadOmmdo555wj16r0nzeaKWfDSNUvcnqIN0pKnaOxsVH+lrnTAgCE\nQdMCAIRB0wIAhEHTAgCEwRgnhwpdlCNwMXXqVFO76aab5Np169aZ2muvvSbXrl271tTeeecdufau\nu+76r0sEyipnDyb1oN57eN/W1mZqGzZskGt/+eUXU1u+fLlcu3nzZlPzwkvnn3++qY0bN87UWlpa\n5PHr1683NXWtKaU0fPhwU5s4caJce+KJJ5qaF2hRe29535kaW6X24/J41+AFP+TawisBAOhlNC0A\nQBg0LQBAGDQtAEAYNC0AQBj9Pj2oxrCklNLLL79c+BxqvMpnn30m1w4bNszUjj76aLlWJW1U2iil\nlL766itT27Fjh1wL9DYvRZazVqXeuru75do9e/aY2tChQ+XaWbNmmdoVV1wh16r0oErCqXRvSik9\n//zzpua9B7U5pXpfKaXU3NxsaiNGjJBr9+7da2peYtOrKypxmZMk9XCnBQAIg6YFAAiDpgUACIOm\nBQAIo98HMbywgtrfRQUuUkpp6dKlpqZGruR69dVXTe37778vfPycOXNKvgagVOq3lLOvktobKqWU\nBg2yf75U0CklvW/VjBkz5NrLLrvM1E455ZTC51X7aXmBry1btphaXV2dXKuCI95aNVpJ7SmWkv4u\n1Gebkg6/qFpKKbW3t5ua9/3k4E4LABAGTQsAEAZNCwAQBk0LABAGTQsAEEa/Tw+ee+65sq5Shd64\nJW8cTKnUKKn9+/dX5LWAUqnkX0p6dM/AgQPlWpVk89Yef/zxplZTUyPXNjU1mZqXSmxsbDQ177ev\nfo+tra2mtmTJEnn8tm3bTM0bW3XmmWeampdSVu9NpRpT0klD73NUvA0cve+tVNxpAQDCoGkBAMKg\naQEAwqBpAQDC6PdBDI/aC6ZS3njjDVlfs2ZN4XNcddVVpnbSSScd9jUBuXL2SvLWqsCDF3RSo4a8\n8UMqWOCFmlSgxAsbqNFIy5YtM7VVq1bJ49VeVt7eXfX19abmjVBqa2szNRW4SMkPpCjqc/BCGypQ\n4o3vygltcKcFAAiDpgUACIOmBQAIg6YFAAiDpgUACIP0YJWtXr3a1O655x65tqury9TGjh0r186f\nP9/UvLQQUAleIlCl8by0mEqneWu7u7tNzRtVpFKF3tgplYbbs2ePXPvrr7+amtqo1UvNnXPOOaZ2\n7bXXyrUTJkwwNbXRYkopbd++3dRGjRol16rEpje2SiUN1feQkn7PXgrT+y4U7rQAAGHQtAAAYdC0\nAABh0LQAAGEQxKiyb775xtRU4MJz7733yvqkSZMO+5qAavOCCSow4e0vpX43+/btk2vVuKMxY8bI\ntSpY8M8//8i1H374oaktXrzY1NR+XimlNGvWLFObPHmyXKtCDLt27ZJr1T5bXqAl5++P4o3ZUt+x\n913mjADjTgsAEAZNCwAQBk0LABAGTQsAEAZNCwAQBunBCrnzzjtl/e233y58jgcffNDUHnnkkcO+\nJqA3qBSZN2JMpcu80T9Fj09JJ9y886rRSF988YVcu3z5clNTSUVvY8ezzz7b1EaMGCHXtra2Fl47\nbNgwU/NSgup6vc001caZ3gimnNFMObjTAgCEQdMCAIRB0wIAhEHTAgCEQRCjDPbu3Wtqn376qVyr\nxsx442Qee+wxU/P2uQF6m/fgXY3oyRnbox7+e2pra2VdBT927Ngh127YsMHUlixZItd2dnaa2lln\nnWVq48ePl8dPmTLF1LyAiApS1NXVybU5e1mp0IV6Xynp/bRUkMO7Bm+UFPtpAQD6JJoWACAMmhYA\nIAyaFgAgDJoWACAM0oNlMHfuXFP7999/Cx//wAMPyHpDQ8NhXxNwpFCpNW8TSJUi89JpajSTNx5K\nvV5zc7Nc+/nnn5vaypUr5drjjjvO1C6++GJTmzZtmjy+pqbG1Hbu3CnXquSd2rAyJf1+vY0s1d8Z\n72+PSkrnfOY5CVMPd1oAgDBoWgCAMGhaAIAwaFoAgDAIYmRYtWqVrKs9dTw33nijqT300EOHe0lA\nn6JCFypwkZIeP+SFAjo6Okzthx9+kGuXLl1qal6AYNy4caZ2wQUXmNro0aPl8Xv27JH1otra2mRd\njVvy9t5S780b46TWqtF0npy90dxzlHwGAACqhKYFAAiDpgUACIOmBQAIgyCGQz2InDdvnlybs9/P\neeedZ2rskYX+xpuIoR7Ue+EKNQ3C+y2uXbu2UC2llDZv3mxqTU1Ncu2sWbNMTU3J8Kj3pkIUKen3\n63026nPwghhq0ob3OaoghgrEeJiIAQDoV2haAIAwaFoAgDBoWgCAMGhaAIAwSA86XnjhBVNbtmxZ\n4ePvvPNOWWdkE/oblRjz0mKqnpMs27p1q6wvWrTI1Lzfs0rTTZ8+Xa5VI5vUXlReQljtTzVy5Ei5\nVo2z8sZAqaSgel/eNXj7dPX09JiaN5pJpQq99KBXV7jTAgCEQdMCAIRB0wIAhEHTAgCEQRDD8dhj\nj5V0/DPPPCPrjGxCf6OCFN7D+5wghtrHqbm5Wa5Vo5n+/vtvuVbtkXX22WfLtWpkkxrD1N7eLo/v\n6uoyNS8EoT4Hb+ST4u29tWvXLlPz9tNSYQ4v4OHVS8WdFgAgDJoWACAMmhYAIAyaFgAgDJoWACAM\n0oMVokajpOSnpkqlUkReekeNYlEpJo+XLJo/f37hcyje9aokp7f5HXqPl/JT/968sT05ibOc35JK\nFarUXEp6jJL3Wlu2bDE19Vvq6OiQx7e2tpqa9znW1dWZmvdbPHDggKmpBKXHSzCqZKWXYFS/0Zqa\nGrmWMU4AgD6JpgUACIOmBQAIg6YFAAiDIEaFqPEulXTvvfea2rHHHivXqofSzz33XNmvqVzUZ3n3\n3Xf3wpWgXA4ePFh4rRc2UIEFb0yaChB4D/9bWlpM7YMPPpBrly9fbmpqPNSoUaPk8cOHDzc1L/RR\nW1tral5gQo1sWr9+vVyrwi9XXnmlXNvU1GRqXuhMXa8XoMoJ1XCnBQAIg6YFAAiDpgUACIOmBQAI\ng6YFAAiD9KDjtttuM7UFCxb0wpUU88ILL1TkvIMG2X8iOaN27rjjDlmfMWNG4XNcfPHFhdfiyJOT\nDFP/3rykoTqvSuOllNIxxxxjatOnT5dr//zzT1P7/fff5VqVbNy+fXuh109Jp/zGjBkj155wwgmm\n5o1QUptOesnKsWPHlnQNDQ0Ncq16vXKMseNOCwAQBk0LABAGTQsAEAZNCwAQxoCcfUxKVLUXqpTX\nX39d1vfv31/SedesWSPrpY5Wevjhh2X95JNPLnyO66+/3tRGjx592NdUBXozIpRVS0tL4d+ztz9U\n0bXe8aqu9pFKSY8w8n53aj8sb1TRzp07C53XO17VzzjjDLl2woQJpuaNh1IBDW9PL3WO0047rfDa\nESNGyLVq/y/vu1R9qLGxUS7mTgsAEAZNCwAQBk0LABAGTQsAEAZNCwAQBulB9DWkB6ugUulBNcbJ\nG/2j/nZ5mwyqVKGX+lXn8N6DOq/aRFJtDJlSSj09PaZWX18v16pxSd7fbzVCyRuHpc7hfeZqhJv3\nmed876QHAQB9Ek0LABAGTQsAEAZNCwAQBkEM9DUEMaogJ4iRQwUAvP3b1N8u7++ZGmGkxjV51+Dt\n01VTU2Nq6nq98VIqMOEFGFTgwQuTdHd3m5oXxFBhEG9tqXL6DUEMAEB4NC0AQBg0LQBAGDQtAEAY\nNC0AQBh2ZgoA9BKVLvOSdzlUok+NjEpJp/e8NN2ePXtMTaXxvATk0KFDTc1L2KmNHb216jMrxxgn\ndY4qJtBTStxpAQACoWkBAMKgaQEAwqBpAQDCIIgB4IjmPej3wgKKCkKoYENKOohR6mupcEZKKbW3\ntxd+LTWayQt4qMBEznvwrjdnj6xK4U4LABAGTQsAEAZNCwAQBk0LABAGTQsAEAbpQQBHNC+xlrMJ\npDqH2lTxv86hqPSeOt5L+VVKTlJQXW/OZ15t3GkBAMKgaQEAwqBpAQDCoGkBAMIYcCQ8WAMAoAju\ntAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQA\nAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh\n/B+IqTJVP1pY6QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7153a3588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_one_at_a_time.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Cache the frozen layer outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.0072861\n",
      "1 Train MSE: 0.00790681\n",
      "2 Train MSE: 0.00777492\n",
      "3 Train MSE: 0.00778348\n",
      "Training phase #2\n",
      "0 Train MSE: 0.00234328\n",
      "1 Train MSE: 0.00248873\n",
      "2 Train MSE: 0.00271144\n",
      "3 Train MSE: 0.00268969\n",
      "Test MSE: 0.0028896\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op, training_op]\n",
    "mses = [phase1_mse, phase2_mse, mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        if phase == 1:\n",
    "            mnist_hidden1 = hidden1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if phase == 1:\n",
    "                    indices = rnd.permutation(len(mnist_hidden1))\n",
    "                    hidden1_batch = mnist_hidden1[indices[:batch_sizes[phase]]]\n",
    "                    feed_dict = {hidden1: hidden1_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "            mse_train = mses[phase].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG15JREFUeJzt3UtsldX3xvHNpfYK5RQKBUFFCwgYEEEFMWqaiBgSiRoG\n6kCDJBoTTTRRA4mJOnGoTIwajXgbmJAomqgIGowIIqBcVISiqIACBQoFCm25/Eb/yX8/y7y750JX\n+/0MV9Y55z2nLcs353HtfhcuXAgAAHjQ/2JfAAAAWTG0AABuMLQAAG4wtAAAbjC0AABuMLQAAG4w\ntAAAbjC0AABuMLQAAG4MLOFrsXoDpdDvYl9AX9Da2srfcwJr81C/fvy6WnK5nPxwuNMCALjB0AIA\nuMHQAgC4UcrvtACgx1PfM50/f75kr5/y/Zf1nVhvPr2DOy0AgBsMLQCAGwwtAIAbDC0AgBsMLQCA\nG6QHAbiUb0Kuf3/93+wqKWj1nj17NtN1DRyo/6k9d+5cVBswYIDsTUkPqutNSUBavT1hgwd3WgAA\nNxhaAAA3GFoAADcYWgAANwhiAOjRrMCFCiyoYEMqFTawntcKaPx/1ntQj095v1YwQgVEurq6ZK96\njqzv67+uoVi40wIAuMHQAgC4wdACALjB0AIAuMHQAgC4QXoQgEsq0ZeS0rMScvmm9FLWJanrtV6r\ns7MzqlmpxjNnzkS1srIy2avq1iqpUh6GaeFOCwDgBkMLAOAGQwsA4AZDCwDgBkEMAL1GSlDAOuNK\nPUfK2V3qeVWIIgQdmDh9+rTsHTx4cFSrrq6WvVVVVVHNWuOkwhzFCqnkewZaCNxpAQAcYWgBANxg\naAEA3GBoAQDcYGgBANwgPQigx1DpMpVCC0Gn7KzknUq9WUk2tdaopqZG9l5yySWZahUVFfLxKhFo\nrWZSKT31WpaOjg5Zt1KFSkr6T11vIdZAcacFAHCDoQUAcIOhBQBwg6EFAHCjTwUxvv/++6i2dOlS\n2XvppZdGtcrKStn74IMPRrW6ujrZa9WBvsT6Ql590X/w4EHZu3379qh29OhR2dve3p752nK5XFS7\n7LLLZK9alzRkyJBMfSHof1OOHTsme9UqqPLyctmrAhpqZZR1DcOGDZO96udjBTlUEMNa+WTVFe60\nAABuMLQAAG4wtAAAbjC0AABuMLQAAG70K8ShXBmV7IUsEyZMiGrNzc1Fea3a2lpZnzlzZlFer1iu\nuOKKqLZ48WLZayWsSix7DAnd1traGv09WwkwtZbIWrd06tSpqPbnn3/K3l27dkW148ePy962trao\nZqX01BonK32oknPqvVkpP8VKQKpU49ChQ2WvShqOGDFC9jY1NUW1iRMnyl6VNLQOuEw5MFLJ5XKy\nmTstAIAbDC0AgBsMLQCAGwwtAIAbfWqN08cffxzVtmzZInsnT54c1X755RfZu2HDhqi2YsUK2bty\n5cqoNnbsWNm7Z88eWc9q4ED94x05cmRU27t3b+bnVeGMEEJ49tlnMz8Heh8r1JWyzqd///i/o60V\nSOosqurqatmr1hpZK9VU6MIKUuzfvz+qtba2RrXhw4fLx6swifVa6u/WWoe1c+fOqGatcTpw4EBU\nU6G1EPTPWAVXrF7r554SCOROCwDgBkMLAOAGQwsA4AZDCwDgBkMLAOBGn1rjVEpWUketpLHSg3/8\n8Ude16ASUyHoFJJ1DS0tLVHto48+kr3z589PuLqiYY1TCag1TinUaqcQ9Bona+WTeg7rQMKOjo7M\n16bWEllrnNT1qtVK//77r3z8kSNHotqYMWNkr1rDtG7dOtmrktKXX3657L3vvvui2o033ih7VbLx\n7Nmzslf9fFjjBADoUxhaAAA3GFoAADcYWgAAN/rUGqdSqqiokPWrr74683NYZ9rkS62dOnz4sOxV\nX8jOmTOn4NeE3kt9+W6tGFOrmay1RipoZAU81HNYoQC1Gsl6XhVkU+dIWUEndV1DhgyRvSogsmrV\nKtmrgiPqsw1Br5iyVjOp0EXKSq5CBP+40wIAuMHQAgC4wdACALjB0AIAuMHQAgC4QXqwF1MrZkII\n4e67745q1mFyr7zySlSrrKzM78LQp6Ss7lHpMiuJqxJu1u+xSvRZK59U3UrTqfem3oP1N6PqVlJx\nx44dUa25uVn2Ktdcc42sq7VuFvX5qpRgCPpzSDko1MKdFgDADYYWAMANhhYAwA2GFgDADYIYvdiy\nZctk/cCBA1FNnQEUgn0GD5BVvl/IW1/0q96Uc7NSVgpZAQ+1SqqtrS2vazh06JDs/eSTT6Latm3b\nZO+sWbOi2vXXXy97q6urM13Xf9WVlHBFCu60AABuMLQAAG4wtAAAbjC0AABuMLQAAG6QHuwlfv/9\n96j21FNPZX78+vXrZb2hoaHb1wRYrBSaWmFkHRh5+vTpTLUQ9BonK5VYVVUV1VJSiSlroFQqcffu\n3bJ306ZNUc1acXXLLbdEtcbGRtmrPgdrlZRi/SzVeytEopA7LQCAGwwtAIAbDC0AgBsMLQCAGwQx\neolPP/00qlnnBS1YsCCqXXnllQW/JiAE/UW9tRZJ9Vq/xyosoAIXIegAgFrBlPq8KqChwhEq3BGC\nXvn07bffyt6DBw9GtenTp8teVbfO9FI/i5SfjxWuyPdcMgt3WgAANxhaAAA3GFoAADcYWgAANwhi\nOGN9Kf3RRx9FtfLyctn70ksvRTXri2YgXylnZKkQRMp2BnU2VAhpAQIVFujs7JS9aluHCnhY7/ev\nv/6Kajt27JC9Ksxx++23y151Pp4VrlDvzfrM1WdmPa8KpFi9KbjTAgC4wdACALjB0AIAuMHQAgC4\nwdACALhBetCZt956S9bV6pf7779f9rKyCcWQtIrHSNOdPXs283OodKz1vCq1lvJaVq96PZUe3Lt3\nr3z8J598EtVUojAEfUbWtddeK3vVyiYreZyybkmxEoHq98E6e4s1TgCAXomhBQBwg6EFAHCDoQUA\ncIMgRg+2ZcuWqPb444/L3iFDhkS1F198seDXBFisL9mVlCCGFQpQq8esazh58mRUU2uGQtChAGvN\nmXoOdcbWN998Ix+/bt26qJbL5WSvCmKMGDFC9qo1TOq6Qkhb4aZCF9ZnrurWzz3pdydzJwAAFxlD\nCwDgBkMLAOAGQwsA4AZDCwDgBunBHuD06dOyft9990U163C2Bx54IKqxrgk9Qco6H/X7rQ5atHpT\nDna0UnOqV61msvz8889R7euvv5a9+/bti2qLFi2Svddcc01Us96DWs1kfTaqbvWq17NWXKmfccrK\nJwt3WgAANxhaAAA3GFoAADcYWgAANwhilJj6InLevHmyd+fOnVFt4sSJsveFF17I78KAIlG/89YX\n7yrwYIWPVEDDCgWoa7BCAWrdkRXE+OOPP6La8uXLo5pa1xRCCDNmzIhqTU1Nsre+vj6qWe9XvQcr\ntJESxEhZcaWujTVOAIA+haEFAHCDoQUAcIOhBQBwg6EFAHCD9GCJHT16NKqtWbMm8+Pfe+89Wa+r\nq+vuJQEll5Lys6hUoZVOU6uZrNdSz2ElGLdt2xbV1OGt48aNk49fuHBhVJs0aZLsVSm9zs5O2asO\np0xJBFpSDum0fhb54k4LAOAGQwsA4AZDCwDgBkMLAOAGQYwiOX78uKzPnDkz83O8//77UW3atGnd\nvibgYlBfyFtre6yVQFmf1wpMqOe1AgQqmLB7927Zu3nz5qh2+PDhqDZ37lz5+MmTJ0e1mpoa2avO\n3SvEaqaUQItaZ5USnklZ12ThTgsA4AZDCwDgBkMLAOAGQwsA4AZDCwDgBunBInn77bdlXR0aZ7n5\n5pujWsrKFaAnUImxlJVCViJQJdm6urpkr6pbqTe1am3lypWyd+3atVFNpf+mT58uH6/Wr1nXpT4H\n63NUn7n1vOqzsVKJ6hqsFGYhkoIKd1oAADcYWgAANxhaAAA3GFoAADcIYhRAc3NzVHv++edLfyGA\ncyoAMHCg/mdKhRAqKytlrwohnDp1SvZu3bo1qq1bt072/vDDD1GtqakpquVyOfl49d6ss8bUZ2OF\nVDo6OmRdUZ+jFdpQ11CswIWFOy0AgBsMLQCAGwwtAIAbDC0AgBsMLQCAG6QHC+Dbb7+Nam1tbZkf\nP3HiRFm3klCAdylrnKwDCVXdSt6pNJ2VsFPXMGrUKNk7b968qHbHHXdEtXHjxsnHqxVK1vtV12Wt\nrbKeQ1GJQOtzzHpdIbDGCQAAhhYAwA+GFgDADYYWAMANghgldtNNN0W1VatWyV6CGOhrUs6BUvWU\n8+as3uHDh0c16zwsdebdhAkTolpFRYV8vApSWAEGa2VTVtYZWepztHoV1jgBAGBgaAEA3GBoAQDc\nYGgBANxgaAEA3OhXwuRHaSMm6Kuyx8fQba2trT3y7zllpZDVq9J01r+TqlfVrLVK+ab0rOtSr5eS\nrOwJcrmcvGDutAAAbjC0AABuMLQAAG4wtAAAbpQyiAEAQF640wIAuMHQAgC4wdACALjB0AIAuMHQ\nAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIA\nuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB\n0AIAuMHQAgC4wdACALgxsISvdaGEr4W+q9/FvoC+oKWlhb9nQ79+8a/ghQv640rp7Wvq6+vl3zJ3\nWgAANxhaAAA3GFoAADdK+Z0WABSM+j7Ior4nSnl8vq+V8rzWa6W8h/Pnz2e+hqzXZV1DqXGnBQBw\ng6EFAHCDoQUAcIOhBQBwg6EFAHCD9CAAl1SSzUq39e8f//e5qlnP0dXVlfm6VPKuEK+lEoFWyk+9\n3oABA2TvuXPnMvf2hG0f3GkBANxgaAEA3GBoAQDcYGgBANwgiFEAH3zwQVQ7deqU7N28eXNUe+ON\nNzK/1nPPPSfrTU1NUe22227L/LxAT5CyLkkFCFTNqnd2dsreEydORLWWlhbZ29bWFtVUkKK6ulo+\nvqamJqpVVVXJ3vLy8qhWUVEhey+55JLMvWVlZbKupKySKtbKJ+60AABuMLQAAG4wtAAAbjC0AABu\nMLQAAG70K+GhXhf/9LA8PfbYY7L++uuvl/hKYpMmTYpqa9eulb21tbXFvpyLKXv8DN3W0tJSsr9n\nK5129uzZqHb8+HHZq+q//vqr7N2wYUNU27Ztm+zdt29fVFMpPyvVeNVVV0W1G264QfbeeuutUU39\n3YcQQmVlZVTr6OiQvVZaUVHroaznzTdpWF9fL5u50wIAuMHQAgC4wdACALjB0AIAuMEaJ4MKXRQi\ncDFt2rSodu+998re5ubmqPbOO+/IXvWl8vLly2Xvww8//F+XCBRUvmcwqXOkQgihvb09qh08eFD2\n/vLLL1Ft06ZNsleFLqyAx5VXXhnVRo8eHdXUuqcQQti1a1dUGzhQ/7M8fvz4TLUQ7OCHotZOqTVQ\nIeifhfWzVEGZlHO6LNxpAQDcYGgBANxgaAEA3GBoAQDcYGgBANzo8+nBv//+W9bffPPNzM9x/fXX\nR7UvvvhC9qqVKVZSRyWAdu/eLXu/++67qHb48GHZC3iiUmgh6IMZ//rrL9mr/m6s5506dWpUU6nf\nEEKYO3duVBs8eHBU+/rrr+XjrTSwohKMR48elb3qgEtrXZNKK1opP3VAppX8s1KfWa/Bwp0WAMAN\nhhYAwA2GFgDADYYWAMCNPh/EsMIKajWJClyEEMLq1aujWk1NTX4XFkJYtmxZVNu4cWPmx8+fPz/v\nawCySjkrKeU5zpw5I3tPnjwZ1SoqKmTvuHHjotqUKVNkb0NDQ1SzzrgaNGhQVFOrpA4dOiQf39ra\nGtWs96DOrVKBixB0uEsFRELQgQlrDZQKTFg/35SAR8rvCHdaAAA3GFoAADcYWgAANxhaAAA3GFoA\nADf6fHrwuuuuk3WVKrTWLVVWVhb0mv6PWiVlpYWAUlIpv/799X8DqySalSJTSUErTVdbWxvVhg4d\nKnsbGxujmvX3XF9fH9Wsv3G1CkqlB601Tnv27Ilqt9xyi+y9+uqro5o6hDIE/ZmrQzND0D8LKz1Y\nVlaWuVd9vtZqJ9KDAIBeiaEFAHCDoQUAcIOhBQBwo88HMSzqS95iee+992R969atmZ9jzpw5Ue2q\nq67q9jUB/0V9cW59Ia++6LdCGyrwYK2HGjVqVOZrUAEC6xrUCjYrQKBWSa1fvz6q7dy5Uz5erXEa\nMWKE7FXv17ouVT9y5IjsVe/XWkOnVkmlnAeYEriwcKcFAHCDoQUAcIOhBQBwg6EFAHCDoQUAcIP0\nYIn99NNPUe2RRx6RvSqpM3LkSNm7dOnSqKYSU0CxWCk/ldJL6bVSfnV1dVFNrVUKQa8wsnpV6q2r\nq0v2/vDDD1FtzZo1Ua2lpUU+/sYbb4xqN998s+xVf/vWeqmUwyXLy8tlXUlZWWd9vvniTgsA4AZD\nCwDgBkMLAOAGQwsA4AZBjBJTK15U4MLy6KOPyvr48eO7fU1AKhWksFb0qLoVxEhZ86PWQ1l/S2qt\nkbUCSZ3ptWvXLtn75ZdfRrVNmzZFtdmzZ8vHz5gxI1MtBL1ayQo7qM/GCmKoNUxW+EV9vlaQI+V3\nJAV3WgAANxhaAAA3GFoAADcYWgAANxhaAAA3SA8WycKFC2X9ww8/zPwcTz75ZFR75plnun1NQKGk\nJAJVws1KnOWb8rMOgVSsa1DPu3btWtm7evXqqKbWp02dOlU+XiUFc7mc7FXvbeBA/U94dXV1VLN+\nPmpFlTrcMgT9mVkJRvU7YqUHrWtTuNMCALjB0AIAuMHQAgC4wdACALhBEKMA1JeWn3/+uexVX/KO\nGDFC9i5ZsiSqqZUrQE+mvmS3whWK9UW/CiFYz1tVVRXVrFVFO3bsiGpqNVMIIfz5559R7bbbbotq\nU6ZMkY9vbGyMaseOHZO96r0NHz5c9qowiBWuUJ+vWhkVQlrQRV2v9Zmn4E4LAOAGQwsA4AZDCwDg\nBkMLAOAGQwsA4AbpwQJYsGBBVDt06FDmxz/xxBOyXldX1+1rAoopZe2O6rUSgSr1ZiXOVDpNHX4Y\ngl5VZKX01EGt//77r+wdNmxYVJs5c2ZUGzt2rHz8oEGDopqV0FPv1zr08tSpU5keb12DtR5Kfb6d\nnZ2yNwVrnAAAvRJDCwDgBkMLAOAGQwsA4AZBjASbN2+W9TVr1mR+jnvuuSeqPfXUU929JOCisM5F\nytprBSYU60t6FdCwQgzt7e1R7bPPPpO9GzdujGptbW2yd/bs2VFNrXEaPXq0fLwKnqhrtVRUVMj6\nkSNHopoKXIQQQmVlZVRTwZUQQjh9+nRUs1bLpYQrUn6fuNMCALjB0AIAuMHQAgC4wdACALhBEMOg\nvnBcvHix7E35P8KnT58e1TgjC72B9cW7+pLd+qJfBSlUUCAEHUKwNkTs378/qv3000+y98cff4xq\n1dXVsnfatGlRraGhIaqVl5fLx6vz9axQgnq/VvBE/ZtifY7qOaztGSdOnIhqtbW1slf9PlgBnJTz\n1bjTAgC4wdACALjB0AIAuMHQAgC4wdACALhBetDw2muvRbWvvvoq8+MXLlwo66xsAuykoapbZzup\n1K46RyqEELZv3x7V1LqmEPRZX/Pnz5e9qj506NCoZqX8VHrQSkBWVVVFNeuzUdeQci6ZWgNl9VrX\nqxKMVkqQ87QAAL0SQwsA4AZDCwDgBkMLAOAGQQzDkiVL8nr8yy+/LOusbEJfowIA1jofFYKwQgxq\npdCePXtk77Zt26Lavn37ZG9jY2NUmzFjhuytqamJauo8rOPHj8vHq/VOKeuWrM9GBR6sM8EOHDiQ\nqRaCDn6MHz9e9qqzwlICFxbutAAAbjC0AABuMLQAAG4wtAAAbjC0AABukB4skpMnT8q6tUolXyqF\nZCW0VOLIWsWiqAMyQwhh6dKlmZ9Dsa5XJTlVMgk9k/qdt/4O1AGIVuLs4MGDUW337t2yVyUFjx07\nJnvVKiirVx0YqVKNgwYNko9Xv8fWIZAquVdXVyd7W1tbo9pvv/0me9XnaCUN77zzzqhm/XugFOLv\nljstAIAbDC0AgBsMLQCAGwwtAIAbBDGK5NJLLy3p6z366KNRbdSoUbJXrWh59dVXC35NhaI+y0WL\nFl2EK0F3qCCFFTZQYZyuri7Za4UbFBX8sM6i2rt3b1RbsWKF7FVrp1TNWt+m3oP1focPHx7VrJBK\nS0tLVDt06JDsVeGICRMmyF61HkoFT0IIobq6WtbzxZ0WAMANhhYAwA2GFgDADYYWAMANhhYAwA3S\ng4YHHnggqr399tsX4Uqyee2114ryvCphZa1bUh566CFZnzVrVubnmD17duZe9DwqKahSaBZr5VMu\nl4tqVupt0qRJUc1aa6TWEm3cuFH2/vPPP1EtZW2VSthVVVXJ3iFDhkS1wYMHy161lk0dWBlCCGPH\njo1q1ueo1sVZKUHVWwjcaQEA3GBoAQDcYGgBANxgaAEA3OhnrVMpgpK9ULG8++67st7Z2ZnX827d\nulXW812t9PTTT8t6Y2Nj5ue46667oppaJ9OD6L02KKiWlpbo79laKZQS3FG91rolpb29Xdb37NkT\n1TZv3ix71Xlz6sypEEL4+++/o9qZM2eimnU+lXq/lZWVsletZVNhlBB00EUFOUIIYcyYMVGtoaFB\n9qrgh7VOywp+ZFVfXy9/objTAgC4wdACALjB0AIAuMHQAgC4wdACALhBehC9DenBElDpwRTWWiOV\nerPSh+pgRXUAYwg60WetH1IpSOt5Dx8+HNVUmthKD6rnHTlypOxVyb0jR47IXrWKykr5qZ9FRUWF\n7FXJSqs35fBPhfQgAMA9hhYAwA2GFgDADYYWAMANztMCUFTqC3nrPC31Rb/15b0KV6QEMdRrhaDX\nRqnQRwh63ZEKeHR1dcnHq8/BCjaogId19pZ6b9b7Vc9rBWXUZ1PCMF8IgTstAIAjDC0AgBsMLQCA\nGwwtAIAbDC0AgBukBwEUlUqXFeLAyKyvFYI+kNBKyKlDGK3rUmm6jo6OTLUQ9PVayUrVa6Ul1XNY\nveo9WNdg/dzy7U1JIHKnBQBwg6EFAHCDoQUAcIOhBQBwgyAGgJIrxOqf8vLyqFZWViZ7U0IBKoRg\nBRPa29szPaf1flPOnLJWQSkp55Kp9U4pn5elWOuduNMCALjB0AIAuMHQAgC4wdACALjB0AIAuEF6\nEECPZiXZUtJpKauk8k3pWeuhFHUN1nWpesprWVKeI+UzT0lGpuBOCwDgBkMLAOAGQwsA4AZDCwDg\nRr9irdoAAKDQuNMCALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdAC\nALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4\nwdACALjB0AIAuPE/kFyNEKxXsbEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc73ce3d9e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_cache_frozen.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Tying weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "It is common to tie the weights of the encoder and the decoder (`weights_decoder = tf.transpose(weights_encoder)`). Unfortunately this makes it impossible (or very tricky) to use the `fully_connected()` function, so we need to build the Autoencoder manually:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0167529\n",
      "1 Train MSE: 0.0164158\n",
      "2 Train MSE: 0.0162873\n",
      "3 Train MSE: 0.0171566\n",
      "4 Train MSE: 0.0161259\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGyFJREFUeJzt3UtsleW3x/GnUHqjFyhtpYJUKooFQUG8gQOjkYmJRo0D\nZaBRE42JJpqo0cREnehMmRg1Gu8mJibe49+gAYIGL0GKYgHRSlsQKKWl95YWOJMzOOc8v3XyvuxL\nu9rvZ7iy9t7v3qVdvNm/rKfgzJkzAQAAD2ZM9AUAAJAUQwsA4AZDCwDgBkMLAOAGQwsA4AZDCwDg\nBkMLAOAGQwsA4AZDCwDgRmEeX4vVG8iHgom+gOmgs7OT32fkVF1dnfxd5k4LAOAGQwsA4AZDCwDg\nRj6/0wKASe/06dNRraBAf1Vq1c+2z3r9bDzvVMGdFgDADYYWAMANhhYAwA2GFgDADYYWAMAN0oMA\npqU0Kb0zZ/QCEFUvLIz/rFqPV9cwc+ZM2auSglZ68NSpU7KeKfU+rPeWq2Qjd1oAADcYWgAANxha\nAAA3GFoAADcIYgBwyQoAKDNmxP8/Lykpkb0nT55M9HirniasoF4rTRDDuq7x8fGoVlRUJHvThDbS\nhEFyhTstAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBukBwFMGVYSTqXsRkdHZa9K2ak0nkWl6WbNmiV7\nVT1N0lDVLFavSita12ClFRX1OaRJfFq40wIAuMHQAgC4wdACALjB0AIAuEEQA8Ckoc6XstYEDQ8P\nRzVrVZEKIajHh6DDEdZqJVUvLS2NasXFxfLx6r2NjY3J3jQhCMUKQaj3kOa10qySygbutAAAbjC0\nAABuMLQAAG4wtAAAbjC0AABukB4EMGmkOVCwsDD+83XixAnZ29fXF9WstUYqPThnzhzZW1VVlejx\nKhUZgk4KWmk89X5nz54te1Vyz1pxlWbdkkoapjlEMhu40wIAuMHQAgC4wdACALjB0AIAuDGtghg/\n/vhjVNu4caPsXbBgQVRT61lCCOHuu++OatXV1bLXqgPTiRW4UF/qDw4Oyt7Ozs6odujQIdl7+PDh\nqGaFNiorKxPVQgihpKQkqqnfces99Pf3RzXr7C31N8n6HFXIxPr7pd5bbW2t7FXrqNKckWUFUlKt\njUrcCQDABGNoAQDcYGgBANxgaAEA3GBoAQDcKEiT/MhQ3l7IsnTp0qi2f//+nLyWWu8SQghXX311\nTl4vV84///yo9tRTT8neRYsW5fhqEkm+BwhnrbOzM/p9tv6WqISbdUBgb29vVDt69KjsbW9vj2ot\nLS2yt6urK6pZhy2qhNvQ0JDsVWlHVWtra5OPP378eFSrr6+XvcuXL49q1uc4Ojoa1RoaGmTvunXr\notqqVatkb3l5eVSzEoFKmjVddXV1spk7LQCAGwwtAIAbDC0AgBsMLQCAG9NqjdOnn34a1Zqbm2Wv\n+tLzjz/+kL0//fRTVPvss89k7zfffBPVFi9eLHv/+ecfWU9Knb8Tgv6it6OjI/HzqnBGCCE8+eST\niZ8D04f6ol4FBay6CmeEEEJ3d3dUs87IKioqimpWKEBdQ5rrVddghT7UWiTrjKyBgYGodvDgQdm7\nZ8+eqNbU1CR71d+DZcuWyV61CspaO2WFRDLFnRYAwA2GFgDADYYWAMANhhYAwA2GFgDAjWmVHlTp\nGStRo6xcuVLW77zzzqj24osvyt4DBw5ENSs92NramvjaFJWYCkGnhaxrOHbsWFS7+OKLM7ouTE1W\nGk+td7KSrepAwoqKCtm7cOHCqKYOZQxBJ9ys1JtK/w0PD8telZCbOXNmVLPShyo9OHfuXNmr1ll9\n++23slclDa1kpboG67NR7836WarUqLXqK80qKO60AABuMLQAAG4wtAAAbjC0AABuTKsgRj5ZXwin\nCTGkCYmkodZOqfOGQgjhqquuimrr16/P+jVh6lJf3qszp0LQAYCamhrZqwIaVqBIXYNaSWT1WiuJ\nZsxI9v9+9Zwh6JVNVqBFrZHbtm1b4uedP3++7FVr2axrUO/XCniocEWa57VwpwUAcIOhBQBwg6EF\nAHCDoQUAcIOhBQBwg/TgFDY4OCjrt956a1Sz1qi8/PLLUc1KXWH6UOt4rGSYSgpavapurXFSK5+s\nFJpK86r1RSHopJ91ver3Rh34aP3OqLr1e6tWqvX09Mhe9dlYyWW1DqusrEz2qpVNIyMjsjdXuNMC\nALjB0AIAuMHQAgC4wdACALhBEGMKe/vtt2X9yJEjUW3evHmyt6GhIZuXhCnCCiYk7bXOVVK91gok\nFaSwzpBTz2GdGaWuTYUrQghhaGgo0fNa71eFGDo6OmTvjh07opoVxFDrrFasWCF71ZosKziifj7W\n52h9ZpniTgsA4AZDCwDgBkMLAOAGQwsA4AZDCwDgBunBKeLvv/+Oao899ljix2/fvl3WrYPjgKRU\n4sxat6RWPpWXl8te9RxWelAdVGgd7Dg6OhrVTpw4IXvVGid1AKP1ftXKpq+++kr2btq0Kar19/fL\n3htvvDGqXXbZZbI3zUGU1ro3RSU20zzewp0WAMANhhYAwA2GFgDADYYWAMANghhTxBdffBHVrDUq\nd9xxR1RrbGzM+jUBFiuYoL6ot9Y4qeew/s2runUOlAo3WKENFRJRNSvY8O+//0a1H3/8Ufbu3bs3\nqq1cuVL2rlq1KqrV19fLXhVeUcGVEHRQxpLp+i4Ld1oAADcYWgAANxhaAAA3GFoAADcIYjhjfdH8\nySefRDV13lAIIbzwwgtRzfqyG8iU+vLd+kJehSusL/9VOMIKTKjnSBMqsLZyJD2Lqq+vTz5+z549\nUa29vV32qjPv1q5dK3vV9ouqqirZmyYwof5OWL2qbgVwrOdQuNMCALjB0AIAuMHQAgC4wdACALjB\n0AIAuEF60Jk333xT1rdt2xbV7rrrLtnLyibkU5rztNJQSUHreVWvtTqopKQkqlVWVspe9XpDQ0NR\nbffu3fLx6vfWShquWbMmql111VWyt6GhIapZZ42p88OsNU7qM0uTPLbO02KNEwBgSmJoAQDcYGgB\nANxgaAEA3CCIMYk1NzdHtYcfflj2zpkzJ6o9//zzWb8mIBvSrFCyVjOlWROkWF/+q7CAdb3qTK6/\n//47qn344Yfy8d9//31UUyGKEEK44YYbopoKZ4SQ7oys4eHhqGa9X7UazjqXTAVaWOMEAJhWGFoA\nADcYWgAANxhaAAA3GFoAADdID04CKr0TQgh33nlnVLNSPRs2bIhqrGvCZKBSeta/Y9Wr1iKFoJNs\nVjpN1a3Emkq9WY4fPx7Vvv7666j2+eefy8erFUjr1q2TvTfeeGNUO+ecc2SvOixWrWsKQa+Nsj4b\nlUC01jipFGY21ndxpwUAcIOhBQBwg6EFAHCDoQUAcIMgRp6pLydvuukm2btv376o1tTUJHufe+65\nzC4MyFCaM5Gsc5VUACDNeU1WgEBdW1lZmewtLIz/LKpgQwh6ZZM6I6ujo0M+/rrrrotqV199texd\nsGBBVLOCDSp0MTg4KHvTrK1Ks5opzb+HNLjTAgC4wdACALjB0AIAuMHQAgC4wdACALhBejDPuru7\no9qWLVsSP/69996T9erq6rO9JCAr0hzkN2vWLFlXKT2V5rOew0olqudQByWGoNOKPT09svfXX3+N\nart27Ypq5eXl8vFXXnllVLv00ktlr0ruWSk/VbcO01SfjZX8S9PLGicAwLTH0AIAuMHQAgC4wdAC\nALhBECNHent7Zd1a0aK8//77UW3VqlVnfU3ARFABDSswoc7IsoIYihVMUKENK5igzpf67bffZO/m\nzZujWldXV1SzwhXXX399VKupqZG9KsSgzrcKQX++ac4Js34+Sa8rBNY4AQDA0AIA+MHQAgC4wdAC\nALjB0AIAuEF6MEfeeustWW9tbU38HNdee21Uy1UiB8gna41Tmn/fKg03MjIie1Wq0Er4trW1RbWt\nW7fKXrWySSUgrdRvQ0NDVLOSe+o9WIdTKlYKU6U7raSh+vlYh3SmWeuVBndaAAA3GFoAADcYWgAA\nNxhaAAA3CGJkwf79+6Pas88+m/8LASYh9eW9FTZQK4HShA2stUZDQ0NR7dixY7L3999/j2p//fWX\n7J09e3ZUU6ELa32bCjEMDw/LXhWksMIOKrRhBTHUZ2b1qp+ltTorG2dnyefNybMCAJADDC0AgBsM\nLQCAGwwtAIAbDC0AgBukB7Ng27ZtUU0dJGdpamqS9dLS0rO+JmAys9Y1qTSclUJTqbc0K4Ws1Jsy\nf/58WS8vL49qajVTbW2tfLy6XmsVlVoPlY30oFqplWadVr5Xy3GnBQBwg6EFAHCDoQUAcIOhBQBw\ngyBGnq1duzaqbdq0SfYSxADslU+KFUxQv0uVlZWyd+nSpVGtvr5e9qrA1XnnnRfVFixYIB8/Pj4e\n1awgRprPQYUr1GuFYAc0JivutAAAbjC0AABuMLQAAG4wtAAAbjC0AABuFFhpmxzI2wthWsvvTplp\nqrOzc8J/n3P1t0sdOmkdLllSUhLV1FqjbFyrel5rhdJkXsOUVF1dnbww7rQAAG4wtAAAbjC0AABu\nMLQAAG7kM4gBAEBGuNMCALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4\nwdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQ\nAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4wdACALjB0AIAuMHQAgC4UZjH1zqTx9fC\n9FUw0RcwHbS2tvL7jJxqbGyUv8vcaQEA3GBoAQDcYGgBANzI53daADBpFBTorz/PnMns6zr1vGle\nK9PXn+q40wIAuMHQAgC4wdACALjB0AIAuMHQAgC4QXoQwJRhpfSUNCk9q3fmzJmJruHkyZOJX2vG\nDH0vMTY2FtWKiooSP6/1HlTduoY01OeQjWQkd1oAADcYWgAANxhaAAA3GFoAADcIYmTBBx98ENUG\nBwdl744dO6La66+/nvi1nnnmGVm//vrro9p1112X+HmBqeD06dOyfurUqUS1EEI4ceJE4l71eips\nUFxcLB8/NDQU1WbNmiV71TWUlJTIXqWiokLW1etZgRYV0LB6rZ9FprjTAgC4wdACALjB0AIAuMHQ\nAgC4wdACALhRkMcDx9yfbPbQQw/J+muvvZbnK4ktW7Ysqn3//feyt6qqKteXM5GS7/HBWWttbc3J\n73Oa1T8qnTYwMCB7VZq3tbVV9h44cCCqdXR0yN7+/v6oNjw8HNWslJ+q19fXy97GxsZEtRBCqKmp\niWrWaiZ1DVbyTyUNrefNND3Y2Ngof5e50wIAuMHQAgC4wdACALjB0AIAuMEaJ4MKXWQjcLFq1aqo\ndvvtt8ve/fv3R7V33nlH9ra0tES1jz/+WPbed999/98lAhNGhS6sL/R7e3ujWl9fn+zdu3dvVPvl\nl19kb1tbW1RT65ZCCGF0dDSqlZaWRjXrPajzuKwVcJWVlVFNBbCsXuvsLfXerHCF+vmkWXGl3m9a\n3GkBANxgaAEA3GBoAQDcYGgBANxgaAEA3Jj26cH29nZZf+ONNxI/xxVXXBHV/vOf/8jesrKyqGal\nelQq56+//pK9P/zwQ1Tr6uqSvcBkpRJnaVYK9fT0yN6DBw9GNZX8C0GvUbrgggtk79KlS6Oa+r39\n888/5ePVobDWKiq1Hso6gFElGAsL9Z979flaKczx8fHEz6v+rlk/SyutKHsTdwIAMMEYWgAANxha\nAAA3GFoAADemfRDDCiuodSUqcBFCCN9++21UKy8vz+zCQghvv/12VLNWzyi33HJLxtcA5IJ1Rpa1\nEkgZGRlJ/HgVrpg3b57sXbFiRVRbvny57FVnUXV2dkY1FQQJQa9s6u7ulr0q9GGtfFJhjoqKCtmr\nzgRTn63F+lmqcIUVuCCIAQCYkhhaAAA3GFoAADcYWgAANxhaAAA3pn16cPXq1bKuUoXWuiW1MiUb\n1CqpkydP5uS1gExZK4WsujI2NhbVrGSZ+n20UruqXl1dLXtVSm/OnDmyV6X0Dh06FNV+//13+fh9\n+/ZFtblz58pelf6zDlVU65asRKBaraQeH4L+WViJTbVmS9XS4k4LAOAGQwsA4AZDCwDgBkMLAODG\ntA9iWKqqqvL2Wu+9956s79q1K/FzrF+/PqpZZwABE81a/ZPmDCYVQrBCDCoMcu6558re2bNnRzUr\nxKDOyfryyy+j2p49e+Tj1fVa4bDLLrssqi1atEj2qsCEOo8rBP2zsMIzKqBhBdFU6CJNKMfCnRYA\nwA2GFgDADYYWAMANhhYAwA2GFgDADdKDebZz586o9sADD8je0dHRqKYOswshhI0bN0a1bKxMATKl\n0mlpDgNUvweW4uJiWVeJQGvlk1pLdPToUdm7adOmqLZ79+6oZr0HdbjkmjVrZG9TU1NUq6yslL3q\nYEdrBZxKbKY5lNFab2etmMoUd1oAADcYWgAANxhaAAA3GFoAADcIYuTZ9u3bo1qaL5offPBBWb/o\noovO+pqAXFJrmKwv+tOsFFJBIyuIoVYNWaukVIihublZ9ra0tES148ePR7Xa2lr5+AsvvDCqXX75\n5bJXnell/e3o7e2NatZnrj4z6zNX551Z52mpIIb1vNbPQuFOCwDgBkMLAOAGQwsA4AZDCwDgBkML\nAOAG6cEcuffee2X9o48+Svwcjz76aFR74oknzvqagFyyEmAqtZbmYEcrcWYlBZOyUm8HDx6Maj//\n/LPsVeud1HtYsmSJfHxjY2NUsw6gHRoaimoqJRiCPqzRel61zkqlBEPQh2FavdZ6p0xxpwUAcIOh\nBQBwg6EFAHCDoQUAcIMgRhYMDAxEta+//lr2qi8yzznnHNn79NNPR7VcfbkJ5JO1UkiFLqxetcbJ\nOsNJhUSsEMNPP/0U1dra2mSvsnjx4qhWV1cne6urq6OaFWhR66GsM7LUOVvqTDHr9ayQSpp1S7l4\nfAjcaQEAHGFoAQDcYGgBANxgaAEA3GBoAQDcID2YBXfccUdU6+zsTPz4Rx55RNZVsgjwRiXGrBSZ\nWu9krWsqKSmJaoWF+k9ad3d3VPvzzz9l7+7du6OaSu6FoBOM9fX1Uc1a46SShmneg0U9h5WsVCuf\nrNVM6tBJ63lVEpT0IABgWmFoAQDcYGgBANxgaAEA3CCIkcKOHTtkfcuWLYmf47bbbotqjz322Nle\nEuCS9YW8+vLeWj+kVppZ64fUuVfbt2+XvZs3b45q1iqpNWvWRLWVK1dGtdWrV8vHq9BGX1+f7C0r\nK4tq2ThrTK2COnHihOxVAQ0rOJKN0IXCnRYAwA2GFgDADYYWAMANhhYAwA2CGIbh4eGo9tRTT8le\n60wb5fLLL49qnJGFqcwKC2T6ePVFv9rYEIIOYrS0tCTuXbRokexVGy0uvPDCqFZbWysfr96DtY1C\nBVKskIra1DE0NCR71bYP9RmEoN+H9fdLhWKs7RlpcKcFAHCDoQUAcIOhBQBwg6EFAHCDoQUAcIP0\noOHVV1+Nat99913ix997772yzsomwE4EqtSbRZ0DZZ1jt3PnzqjW1tYme8vLy6PaJZdcInuXL18e\n1RYuXCh7la6urqg2MDAgeysqKhI/r/psrERga2trVFPp6RBCqKqqSnwNucKdFgDADYYWAMANhhYA\nwA2GFgDADYIYhqeffjqjx7/00kuyzsomTDdqVdHp06dlr1phNDIyInvVWiIVKgghhH379kW1np4e\n2Tt37tyo1tDQIHtV6EK930OHDsnHq7p1dpcKr1ifjXpv1nmA6uysc889V/aqgIa1xq6kpETWM8Wd\nFgDADYYWAMANhhYAwA2GFgDADYYWAMAN0oM5Yq1isZJBmSouLo5q1oFr6nA26/A8xVrxsnHjxsTP\noVjXq5Kcadb9YGKp1JuVOCssjP8kWf/eVEJu7969slelB7u7u2WvOlixv79f9qq04j///BPVrPVS\nKgGpfpdDSLfySV1De3u77FWJ5rKyssS96mcWgv65q2Sl1WvhTgsA4AZDCwDgBkMLAOAGQwsA4AZB\njBxZsGBBXl/vwQcfjGrWKpYjR45EtVdeeSXr15Qt6rO8//77J+BKcDbUl+wqDBSC/qLeCi+pVVBp\nvtBXIYgQQvj333+j2tatW2XvgQMHopparWRdlwpdWGdWqc/s8OHDsre3tzfxNaxevTqq1dTUyF4V\ngEoTikrz87FwpwUAcIOhBQBwg6EFAHCDoQUAcIOhBQBwg/SgYcOGDVHtrbfemoArSebVV1/NyfOq\nFS3WuiXlnnvukfVrrrkm8XOsW7cucS8mH5Xysw4IVOlBq7eysjKqWYc1qn9vVoJRpQdVSjCEEJqb\nm2X9/7IOf1UpPXWwZAh61ZqV3Kuurk70WiGEsGTJkqi2ePFi2aveh5UIzEZSUOFOCwDgBkMLAOAG\nQwsA4AZDCwDgRoF1vkkO5O2FcuXdd9+VdetsoKR27dol65muVnr88cdlXX3xarn55pujWl1d3Vlf\nUx7k5ttf/C+tra0Z/T5bIQh1jpM63yqEEMbGxqJaX1+f7FXhip07d8pedRaVFcRQv7tq7ZQVXrr4\n4ouj2vz582Wver/Wyqd58+ZFNSukooIfFRUVsletnbLO/7LO2UqqsbFR/i5zpwUAcIOhBQBwg6EF\nAHCDoQUAcIOhBQBwg/QgphrSg3mQaXrQStOpurUCSSUNrVSiSvQNDg7KXrUuaWBgQPZ2dHREtaNH\njya+LvUerNSdSlFaKT+1+spah6VYP59M17qlQXoQAOAeQwsA4AZDCwDgBkMLAOAG52kByDsrmKCo\nYIRFBS5C0KuGrBVI6jnGx8dlr1qNpM4Ps96vem/WWjgVeLB61XuwrkG9NytckaszstLgTgsA4AZD\nCwDgBkMLAOAGQwsA4AZDCwDgBulBAJOGSrhZqTdVt1YgDQ8PJ74Gte5IHcBoXUOatUalpaWJH69S\nftb7Vb1W8i/TwxrzjTstAIAbDC0AgBsMLQCAGwwtAIAbvr6BA4D/ptYlWYEJtS7JWg+V5iwqdR7h\nyMhI4serc7rU+wpBBylydR6iFdrI4/mLJu60AABuMLQAAG4wtAAAbjC0AABuMLQAAG6QHgTgkjro\n0Eq3FRUVRTVrfVGa51VJP/W81ioqKymYC1YiMJ+pxGzgTgsA4AZDCwDgBkMLAOAGQwsA4EbBZP7C\nDQCA/4k7LQCAGwwtAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBsMLQCAGwwt\nAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBsMLQCAGwwtAIAbDC0AgBsMLQCA\nGwwtAIAb/wX3le7dY++pZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc73cf04518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Unsupervised pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "logits = tf.matmul(hidden2, weights3) + biases3\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n",
    "loss = cross_entropy + reg_loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Regular training (without pretraining):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.953333 Test accuracy: 0.9271\n",
      "1 Train accuracy: 0.966667 Test accuracy: 0.9261\n",
      "29% Train accuracy: 0.946667 Test accuracy: 0.9468\n",
      "3 Train accuracy: 0.973333 Test accuracy: 0.9543\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now reusing the first two layers of the autoencoder we pretrained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.9\tTest accuracy: 0.9102\n",
      "1 Train accuracy: 0.966667\tTest accuracy: 0.9387\n",
      "2 Train accuracy: 0.953333\tTest accuracy: 0.9488\n",
      "3 Train accuracy: 0.96\tTest accuracy: 0.9482\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # Freeze layers 1 and 2 (optional)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked denoising Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import dropout\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.00001\n",
    "keep_prob = 0.7\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "is_training = tf.placeholder_with_default(False, shape=(), name='is_training')\n",
    "\n",
    "X_drop = dropout(X, keep_prob, is_training=is_training)\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X_drop, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0184146\n",
      "1 Train MSE: 0.0127358\n",
      "2 Train MSE: 0.0102313\n",
      "3 Train MSE: 0.00965831\n",
      "4 Train MSE: 0.00924804\n",
      "5 Train MSE: 0.0088919\n",
      "6 Train MSE: 0.00885094\n",
      "7 Train MSE: 0.0090364\n",
      "8 Train MSE: 0.00841659\n",
      "9 Train MSE: 0.00865683\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, is_training: True})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch, is_training: False})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGzRJREFUeJzt3Vls1PXXx/Evm13phgKK/isIAqIouKCigJhAonElEpcL\njZpoTDTRRI0mJuqFXio3Ro3GfUuMa4yigBqFKloBF9S0IosL1gqUpS2l6HP1XDzP+Rz/vx+dmfbQ\n9+vy5Mz0N1Omx1/m4/kO+eeffxIAABEM7e8LAAAgK4YWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyG\nFgAgDIYWACAMhhYAIIzhJfxZrN5AKQzp7wsYDDo6Ovg8o6hqa2vlZ5k7LQBAGAwtAEAYDC0AQBil\n/E4LAEIaMiT7V6WcnFFc3GkBAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA9COCgpxJ9XiIwT/ov6/MO\nH67/1Pb29mb++ep5e3p6ZO/+/ftNbcSIEbLXu7aBijstAEAYDC0AQBgMLQBAGAwtAEAYsb6BA4B/\n4YUY/v77b1MbNmxY5l6PCjcMHWrvBbzrUr0qnJGSDld4r0E9b57giXp8Svq9ybPiqhC40wIAhMHQ\nAgCEwdACAITB0AIAhMHQAgCEQXoQwICRJ4nmJdyy8pJ3ajWSl/7LuoZJJf9S0ulD7z1QP8vrLdah\nleoaDjnkkD4/bx7caQEAwmBoAQDCYGgBAMJgaAEAwiCIAWBA8wIXKkiRZwWSRz2vFyrIGjbwfn5n\nZ6epeedelZeXm1qeEMS+fftk7549e2RdUa/DO48rzzqrPLjTAgCEwdACAITB0AIAhMHQAgCEwdAC\nAIRBehBAyRXi4MCuri5T85Jw3d3dpqbSeCmlVFFRkamWkk76qfSh93jVm+cARu/ASvX+eik/1bt7\n9+7MvXnkOYjSw50WACAMhhYAIAyGFgAgDIYWACCMQRXE+Oyzz0xtyZIlsnfcuHGm5n2ZevXVV5ta\nQ0OD7PXqwGDirTVSX8hv375d9ra2tpramjVrZG9bW5upjR49WvbW1taa2oQJE2RvWVmZqam/E17o\no7q62tR27dole1XowltbpUIXhx12mOytrKw0NS8Mol5HnjVbedZpebjTAgCEwdACAITB0AIAhMHQ\nAgCEwdACAIQxpBCHcmVUsh/kmTx5sqm1tLQU5WepBFJKKZ1++ulF+XnFcvTRR5vaXXfdJXv/85//\nFPlqMun7fiD8Vx0dHZk/z2p1j7d+SK0P+vnnn2Xvhx9+aGpff/217F2/fr2p1dfXy151sKJaA5VS\nSnv37jW1UaNGmZpKCaakE3bbtm2Tvep5vVSi+vvjfT5nz55tao2NjbK3qqrK1LxDK/P83lW9trZW\nfpa50wIAhMHQAgCEwdACAITB0AIAhDGo1ji98cYbprZ27VrZO23aNFP77rvvZO/nn39uam+++abs\nXbp0qamNHz9e9npfQGflnZ9z+OGHm9qWLVsyP68KZ6SU0p133pn5ORCbCnDlOStJBRBS0muR6urq\nZO+MGTMyX4P6jO3bt0/27ty509S2bt0qe1WA4NBDDzW1np4e+Xj1vDt27JC9at2SF2xobm42tSOP\nPFL2qus96qijZK8KqXirmdR6Jy/4l+ecLu60AABhMLQAAGEwtAAAYTC0AABhMLQAAGEMqjVOpeSt\nfdm4caOpeenBDRs29OkaVNInJZ0e9K7hzz//NLXXX39d9l500UU5rq5oWONUAmqNU56/Jd6/TZVE\n8w467OzsNDXvc6ee11uXpFKF3vOqNUrqtXmvV12DWg2Vkk4KLl++XPa+/PLLpuYdQPvAAw+Y2ty5\nc2WvSiR7v3eVHsxzCGRNTQ1rnAAAsTG0AABhMLQAAGEwtAAAYQyqNU6l5J1zM2XKlMzPMXXq1EJd\nzv+h1k61t7fL3lmzZpnaggULCn5NiM9bxaMCBN4X8upsJu+8JvUZ80IbKkAwZswY2asCBB71mtV1\nec+p3gfvvVHrnVauXCl7VfBj5syZsledM+hdr3q9XnBEXUMhgn/caQEAwmBoAQDCYGgBAMJgaAEA\nwmBoAQDCID14ENuzZ4+sX3LJJabmHSb38MMPm1pFRUXfLgyDSp7DANW/Qy+VqNYteYcteqlCJc/6\nIXWYZZ6fpVKN3gGZ6qDW1tZW2asOjJw+fbrsrampMTXv96PeX+96i7UikDstAEAYDC0AQBgMLQBA\nGAwtAEAYBDEOYk8//bSsb9261dRGjRolexsbGwt5ScC/yhPEyHM2kzoPywsUqWCBt0pKhQ3yBDHU\na2hra5O9b7/9tqk1NzfL3lNOOcXU5syZI3tVaCPPaqY8a68KgTstAEAYDC0AQBgMLQBAGAwtAEAY\nDC0AQBikBw8SP/30k6nddtttmR/f1NQk62PHjj3gawI8XsJOJfe8dJpK7pWVlclelf5TSTiPt6pI\nvQ6VgPReb2dnp6l9/PHHsvf11183NS9Zec4555jahAkTZK96jq6uLtmrXptKH3q8682z8ok7LQBA\nGAwtAEAYDC0AQBgMLQBAGAQxDhJqxYs6byillC677DJT876kBfqb9yW9Cl14X/R7dUWFLrxggnpe\ntTLK8+uvv5raihUrZK9av3bxxRfL3oULF5qaOjcrJX3unrfGqaqqytS8kIpaUVWIM7a40wIAhMHQ\nAgCEwdACAITB0AIAhEEQIxgvXKH+b3lvO8CDDz5oat6XqUAx5Dkja/hw/Weqr2dveZ8P9RlTYQWv\nV53T5YUzWltbTe3rr7+WvSeddJKpLV68WPaOHj3a1NT7lZIOXVRXV8vePEGKQoQuFO60AABhMLQA\nAGEwtAAAYTC0AABhMLQAAGGQHgzmySeflPVPPvnE1K688krZy8om9LdCpNBU4lWlBL26l+hTvV7y\nTqUVVXpw+/bt8vErV67M/LNmzZplalOnTpW96jX09PTIXvUavMRmnjPIioU7LQBAGAwtAEAYDC0A\nQBgMLQBAGAQxBrC1a9ea2s033yx76+rqTO3+++8v+DUBxaRCAb29vbJXBRa80IZat+StfFIBj8rK\nStmrAgudnZ2m5p2RtXr1alObNGmS7F2wYIGp1dfXy17FOyOrvLzc1Ly1bt4aOUW9N97vMg/utAAA\nYTC0AABhMLQAAGEwtAAAYTC0AABhkB4cALq6umT9iiuuMDVvTc1VV11laqxrwkCgEn1ecm/oUPvf\n0V4iUKXTvBVIKrXmfZbUNaiEXUo6Tffpp5+a2ksvvSQfv2XLFlNbtGiR7J08ebKpeeuWVILRe8/V\nc3jvuUogequdvJ/XV9xpAQDCYGgBAMJgaAEAwmBoAQDCIIhRYuqL4vPPP1/2/vjjj6bmnZ9z3333\n9e3CgBJSYYeU9OfD681zDpTq9YICeYIj7e3tpvbWW2+Z2rfffisff84555jawoULZe/IkSNNzVvN\npF6DFybJ+viUUhoxYkSmWkp+KEbJE9rgTgsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBunBEtu2bZup\nffTRR5kf/9xzz8l6Q0PDgV4SUFQq/eclAlX6r7u7W/aqdJqXHlQJNy+xpupemq6pqcnU1q1bZ2rj\nx4+Xj7/88stN7bjjjpO9au1UT0+P7K2urpb1rM/rJf/UyqY8yb9CrHbiTgsAEAZDCwAQBkMLABAG\nQwsAEAZBjCLp6OiQ9dNPPz3zczz//POmNmPGjAO+JmCg8IINKhTgfXmvetX5Vt5zlJWVyV4VQvj+\n++9l74oVK0xNvbazzz5bPv6ss84ytWHDhsletbLJew1ZH5+Sfm+8oIzihTY4TwsAMOgxtAAAYTC0\nAABhMLQAAGEwtAAAYZAeLJKnnnpK1jds2JD5OVSyqFiJHKBYVJrOSw+qf99emi4PlXBT6cOUUtqx\nY4epffjhh7J32bJlplZVVWVqJ554onx8TU2NqXnvjVrZVFFRkbnXW/mk3l8vEZhnHZb3OvqKOy0A\nQBgMLQBAGAwtAEAYDC0AQBgEMQqgpaXF1O69997SXwgQnAoAeKEAFaSorKzs8zVs3brV1L755hvZ\nu3nzZlObP3++qU2fPl0+XoUYvPPDlN7eXllX54qps7BS0u+jClx4ihW48HCnBQAIg6EFAAiDoQUA\nCIOhBQAIg6EFAAiD9GABfPLJJ6a2c+fOzI+fOnWqrHsrWgDkOwRSrSpqb2+Xve+//76pNTc3y16V\nyJs4caKpjRo1Sj4+z2tQiT4v5acSfd66JZU0LHUiMA/utAAAYTC0AABhMLQAAGEwtAAAYRDEKLEz\nzzzT1D744APZSxADg40KCwwdqv/bury83NS8lU95ztNSQYp58+bJ3gsuuMDUzj33XFNT15qSDl14\nr0G9N94apzzn7g3k0IXCnRYAIAyGFgAgDIYWACAMhhYAIAyGFgAgjCElTI7EiqggquyxKRywjo6O\ng/bzrA5h9JKGKhWo0o5eAlKl/Ly/yXl6oyUCldraWvlZ5k4LABAGQwsAEAZDCwAQBkMLABBGKYMY\nAAD0CXdaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoA\ngDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAw\nGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMIaX8Gf9U8KfhcFrSH9fwGCw\nadMmPs8ppSFDsv1z++cf3q68Ghsb5ZvLnRYAIAyGFgAgDIYWACCMUn6nBQADhvd9lPr+aehQ/d/3\nqreU319515XH33//XYArKR3utAAAYTC0AABhMLQAAGEwtAAAYTC0AABhkB4EcNDIuqEiJT/l19PT\nY2r79++XvXv37jW18vJyU6uurpaPLysrMzUvEaiuS9W85/CeV71nXqJwIGz24E4LABAGQwsAEAZD\nCwAQBkMLABAGQYwCeOGFF0xtz549sre5udnUHn/88cw/65577pH1+fPnm9q8efMyPy8wUBVihVJ3\nd7epbd++XfZu3LjR1H744QfZu23bNlMbMWKEqXlBjPr6elObMWOG7K2trTW1iooK2Tts2DBTy7Py\nKU+gpRCrpPLgTgsAEAZDCwAQBkMLABAGQwsAEAZDCwAQxpASruXo//0ffXTTTTfJ+mOPPVbiK7GO\nO+44U/v0009lr0ohHUSyx55wwDZt2lSyz3OedNq+fftk/bfffjO1zz77TPa+8847pqZSvyml1NDQ\nYGojR440tV27dsnHq/TgKaecInvPPvtsU5s9e7bsPfTQQ02tq6tL9qr3zOsdPtwGzlUtpb6vfGps\nbJSfZe60AABhMLQAAGEwtAAAYTC0AABhsMbJoUIXhQhcqBUtixYtkr0tLS2m9swzz8je9evXm9qr\nr74qe6+77rp/u0Sg6Lw1Qaqu1iKlpAMEO3fulL1NTU2m9uKLL8reTZs2mdq0adNk75w5c0xNXa9a\nDZWS/tyuWbNG9lZVVZnaxIkTZW9lZaWpeWdvqffRO09L/X688IsX0Ogr7rQAAGEwtAAAYTC0AABh\nMLQAAGEwtAAAYQz69ODmzZtl/Yknnsj8HKeeeqqpvffee7JXpXoOOeQQ2bt//35Ta21tlb0rV640\ntfb2dtkLlJJKnHnpQZW881JonZ2dpuZ9nr/88ktT8w5qPf/8801t8eLFslelClWabvXq1fLx6nBK\ndbBkSinV1NTIuqLWMHlrlXp7ezM/r0ogeunOPAdJ5ln5xJ0WACAMhhYAIAyGFgAgDIYWACCMQR/E\n8MIK6otBFbhIKaVly5aZWnV1dd8uLKX09NNPm9oXX3yR+fEXXXRRn68ByCrPaqZhw4bJXlX3vuhX\nn9GysjLZO2rUKFPz1qedd955pnbCCSfIXvU5V39TvFVHKkwyduxY2Ttp0iRTGz16tOxV74MKdqWU\n0u7du03Ne89VKCbP7z3PeigPd1oAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMAZ9enDmzJmyrhJA3rql\nioqKgl7T/1KrpLyD3ID+5q3iyZUMG2r/O9p7XpVE8z6L8+fPN7Xjjz9e9h599NGmVl5eLnv37t1r\nam1tbab2008/yceruvc3Sb02bwWTSgR6f7/q6upMzUs7qt+Fl0pUv0uvN8+BkdxpAQDCYGgBAMJg\naAEAwmBoAQDCGPRBDE9tbW3JftZzzz0n6+vWrcv8HAsWLDC1Y4455oCvCSgUFZjIs84nT9Cpvr5e\n1hsaGkzNW5eUJ0DQ0dFhahs2bDC1VatWycerIMfhhx8ue/MEJhTvfVQhCG8dllo75b03KrThhXI4\nTwsAcFBiaAEAwmBoAQDCYGgBAMJgaAEAwiA9WGJr1qwxtRtuuEH25kkWLVmyxNS8g9yAUlJpvDwH\nB3qritRqpTypRC+xpq63u7tb9ra2tpra0qVLTe2PP/6Qj58xY4apTZkyRfbW1NSYmneYpjqc0vt7\noJ4jTyrRW8Gkfhfqvc2LOy0AQBgMLQBAGAwtAEAYDC0AQBgEMUqsqanJ1FTgwnPjjTfK+rHHHnvA\n1wTklSfYkPXxKemVQN45UOpsOW/9kAoFeGGDrq4uU2tpaZG9b731lqmpz3hVVZV8/KRJk0ztyCOP\nlL0jR440NRW48Hq9QIv6vXmBCVX3wi9eSKSvuNMCAITB0AIAhMHQAgCEwdACAITB0AIAhEF6sEiu\nvfZaWX/llVcyP8ett95qanfccccBXxPQH7x0mZJnhZJK3Xqpt8rKSlPz0oPqYMfly5fL3tdee83U\nVKrxrLPOko8/4ogjTM07gFYlEL1e9T6o60op32GNXr2UuNMCAITB0AIAhMHQAgCEwdACAIRBEKMA\ndu/ebWrvvvuu7FVfKo8ZM0b23n333abmrbQBSqkQK5uyPq/3eHWOk7c6SJ1FpQIXKaW0fv16U1u1\napXsVZ9ntZppwoQJ8vEzZ840tXHjxsleFa7wVjPlWQ2nAine3xm1ZisP799Nnn8j3GkBAMJgaAEA\nwmBoAQDCYGgBAMJgaAEAwiA9WACXXXaZqbW1tWV+/C233CLrDQ0NB3xNwEChEmNeyi/P4ZKqnucQ\nyL/++kv2rl692tRUojAlnbKbPn26qZ122mny8Y2NjaamDnBMSb833momlR70EnrqOXbt2iV71aGT\n3uos9Tv2VmflwZ0WACAMhhYAIAyGFgAgDIYWACAMghg5NDc3y/pHH32U+TkuvfRSU7vtttsO9JKA\nkPKs7fFWB40YMcLUVFAgpZT+/PNPU/vqq69kr/qce2uR5s6da2pqNdMJJ5wgH6/O+VKvKyUdYvDO\nKlPvmReuUIEJ7/ejgi5er7o2L7SR68y1zJ0AAPQzhhYAIAyGFgAgDIYWACAMghiOrq4uU7vrrrtk\nr/d/pSsnn3yyqXFGFgYbL1yhQgHeRgz1pb4XmGhpaTE1L0ClghjHHHOM7FUBi1NPPdXU6uvr5ePV\n9e7cuVP2qk0Z3t+ezs5OU/MCE3V1daaWJxjhBUfUa8vzvB7utAAAYTC0AABhMLQAAGEwtAAAYTC0\nAABhkB50PProo6a2fPnyzI+/9tprZZ2VTYCvt7fX1LykodLe3i7rq1atMrWlS5fK3uHD7Z/FadOm\nyV61xqmqqsrUvFRjd3e3qXlpPFVXP8v7eXnWYan3wOM9r5f67CvutAAAYTC0AABhMLQAAGEwtAAA\nYRDEcNx99919evxDDz0k66xsAvx1Pmo1k/eFfkVFhan9+uuvsvfHH380NS+0MW/ePFObNWuW7FXr\nmVQwQQUuUkppz549plZTUyN71XlaeUIbeVY+dXR0yN4dO3aYmlq9lVJKo0ePNjV1flhK+UIb3GkB\nAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA9WCS7d++WdZWOKoSysjJT81I9Kt3krZlR1AGZKaW0ZMmS\nzM+heNerkpxeagr9xztkUCUFvdU/6vPhJW7V8+7atUv2/vLLL6bm/RtS17Zt2zbZ29TUZGrl5eWZ\nainpNN1vv/0me9UhkN5nZvPmzabmpSXVoZNbt26VvSoFqQ7CTCnf++D921G40wIAhMHQAgCEwdAC\nAITB0AIAhEEQo0jGjRtX0p934403mtoRRxwhe9WXrI888kjBr6lQ1Ht5/fXX98OV4N94q3hU3Qsk\n5TkzSq018oIJ1dXVpuatS9qyZYuprVixQvaqAJRazaRWGqWkwyReQES9Bi/w1dbWZmpeSEWFIH7/\n/XfZe/zxx5valClTZK/6vecJXHi40wIAhMHQAgCEwdACAITB0AIAhMHQAgCEQXrQcdVVV5naU089\n1Q9Xks2jjz5alOcdPtz+E/ESWso111wj62eccUbm55g9e3bmXgw8eVaXqXSZ93hVHzt2rOydM2eO\nqeVZgbR+/XrZu337dlNT6b+6ujr5ePUaent7Za9aL3XYYYfJXpUq9K5h/PjxpjZ9+vTMvWPGjJG9\nav2WlzBljRMA4KDE0AIAhMHQAgCEwdACAIQxxPtirAhK9oOK5dlnn5X1np6ePj3vunXrZL2vq5Vu\nv/12WZ84cWLm57jwwgtNzVtJM0D0fU8M/qtNmzb16fPshSvUWiS1viglHUzwVhX98ccfpqbWNaWU\n0saNG03NO+Pqr7/+MjW1xsk7R0pdb0NDg+xVdS9MosIR3mq5+vp6U1Prpbyf5wU81GuuqKiQvUpj\nY6P8LHOnBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD9CAONqQHS6CU6cHKykrZq1aMeYcidnd3m5p3\n2KKqq5+Vkk7ZqcMp1bqnlPTKJrX+yPtZ3t9vldLzDtNUvLVKXjqzGEgPAgDCY2gBAMJgaAEAwmBo\nAQDC4DwtACWnVjCllNLevXtNzVuTpkIbKnCRkl6t5K1AUiERbw2TCk2o6/XWIqnQh3pd3s9Sr8vj\nhSvU+9DV1ZW51/tdFgt3WgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwSA8CKDkvyeal7BSVNPQSgSqR\n561mUs/hXZdKCqoEo1rXlJJO/3lpPNXrvY8qAeklK1Wv93rz/H6KhTstAEAYDC0AQBgMLQBAGAwt\nAEAYBDEAhJRnfZAX0FBUYME7i0qFGFTAwzu7S52H5b0u1eudS+YFP5Q8wZOBgDstAEAYDC0AQBgM\nLQBAGAwtAEAYDC0AQBikBwEcNPIkClUaLyWd/vMOovSe4//z1i1l/fke7+d7qUJlICcFFe60AABh\nMLQAAGEwtAAAYTC0AABhDMn6RSIAAP2NOy0AQBgMLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYDC0A\nQBgMLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAY\nDC0AQBgMLQBAGAwtAEAYDC0AQBgMLQBAGP8DktLxeebue9AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71427d320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Visualizing the extracted features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_stacked_denoising.ckpt\")\n",
    "    weights1_val = weights1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure extracted_features_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAABYCAYAAABPlQhTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHbtJREFUeJztnVmMlEX3xp/umWaakd1hEFlcUFzBLSpG1C8qcYkalzu9\n88pEY2K89NprE2O8NfFCg4maeKEGNdGIGhcU16goKCMooMM2wCw909/F/H9V1eftF2b4K134nedm\nlu5++31Pnap6nnNOVVWazaYcDofD4cgR1U7fgMPhcDgcZfBJyuFwOBzZwicph8PhcGQLn6QcDofD\nkS18knI4HA5HtvBJyuFwOBzZwicph8PhcGQLn6QcDofDkS18knI4HA5HtvBJyuFwOBzZwicph8Ph\ncGSL7hP1Rffff3/bTQLZO7BSqRz3tf+OaxwPXnjhhRP7hQZPPPHEP7LxorXjidrf8cknn+yoPZ96\n6qmmVHzeanWSy01MTJT6mPXBo9nsWPY91jXavV52X4899ljHbPr000+3fYDUnrmCe+NewaOPPtox\ne7799ttNqdjW3GtXV1f43frjsXyqq6srXIv38NPaoGy8bfe5su/j/+vWrTumPU/YJFWG9EGnO9m0\ne1/ZZ6c68P7bNtydyqDJ+8bHxyVJ3d3dLZ9JO0F6zaNddyp2nOq9nSiU3Uc6mJZNRnRk/saGFthY\nah0Y2t0H1+C7Go3GcT9DTkifl2ccHR2VFG2CnbArNpg5c6YkaWxsLPy0E4q1l7Vju/uwsANzjuB5\nUt/DLrVare178Q9sc+jQIUnR7rNmzQr2wj7WnqkPp++bMWNGy/vT38sm/akg/5ZwOBwOx/8sOq6k\nUhxLmlqGz4yeMkze09PT0/I31+QzIyMjkiKDsKqh2WyeFKz0WChTp5YddXV1FZSSZU6wXdhQGl6A\n4dpwg/2b70j/lxvK/K+dgoQ9AmxjldWRI0ckTdqQ6/T19UmaZK+StH//fknS4cOHJUW1YL8z/Q6r\nBk50yHs6sHadmJho8T8p9lfri/gX/z/jjDMkTfro0NCQpOinf/zxR8v3cs3e3l5J0b7YMVVxViWA\nnJTV0cJtVkHhd9iPv1FQv/32m6Q4Hvb09Ojqq6+WJC1ZskRS7MO816oivrNer0uK7XDkyJG20Zf0\nGaaCfCzvcDgcDodBVkrKMgTLSq0qOnDggCRp3759kiaZ0Pz58yVFZmDj0pap2fg/LCFNIFo1cDIi\nVT/pT9hPV1dXeA/qgNew765duyRFNjZnzhwtXLhQUmRmsFrsDstKGVVOrDSFZfpHUyU8F88ye/bs\nlv9jO1g7Nj1w4EDwJ97DNbDZ3r17JUnDw8Mt94Nvc62xsbGCLXNWUu36d1muxPoiz37aaadJirYb\nHh4OSvSvv/5qeQ2FxDVoI3wUv4b51+v14Ke0Y472tHke+38p9jfUI6r8p59+kiRt3bpVkvTdd99J\nin66bt26YGPsSr/nM9h38eLFkibHASm2Weq39l6tcp4K8hwtHA6Hw+FQZkoKpGpGivklfsJ8mOlh\nSOPj42EWX7RokaSokHbu3NnyWVgWMz2MA9bVaDSyqz6bDqwahbnwfPyN4jx48GBgj/PmzZMUlerP\nP//ccq0FCxZIkhYuXBhYlFVSvPeUU06RFO2L3VPkYl97H+0q+YjtYyurbvbs2SMp5kX6+/slTdpX\nmvRHmOr555/fco0dO3ZIkr7++mtJ0u+//y5J2r17t6TYVvj7rFmzAjO1ubEc0S7nbPPLNmqBqsQ2\n9G/86Mwzzwz5KT7Le7A9jB+f37Ztm6To1/jssmXLgm9bRVWWq+oEbKm3rcpN/4d/DA4OSoo2or/y\nfOShbr31Vp1++umSpD///FNS7MOXXnqppNgWdlxm3ECVVSqV8P22309HoWY1SdnEPI2AERjo+D8S\nlpDT4OCgvvjiC0lRgl5wwQUtn3n//fclRal/6qmnSpJWrFjRci/twlI2/Jcr0nJyYG1JyA6Hajab\nYdK3nZ3BFCdk8ExJgS1IgSTg8Djr0NBQ+D1XOx4t7IcPMrEw+bz33nuSon8xEV155ZWSoq/u3r07\nFEx89dVXkmKbYH+uvXLlSkkxgY2NuZ+xsbEw+NB3uFZOYSprz9Su3DeDJZMtfc+G8nhebLRw4cLQ\nh3nN9m0+u3HjRklx8MWfmcyOHDlSCEuBHELUabtL0VbpxE5/ZMIiNHfOOedIki677DJJ0iWXXCIp\nTthc688//9Q777wjKfo2pAi/hTR9/PHHkqSlS5dKin7KeLxgwYJwr1yL9p7KcgrQecs7HA6Hw1GC\njiuptNwZpgi7h0UxQ8PSSeQhzVFBo6OjgWHBeGEMMAqYA6wAKcvfsJKDBw8Wkri5Mv+jwbIuy2TS\npDy2gJHSHrCzH3/8UVK0/8yZMwvXtayL12HKjUYjtC/IJZRSpj6492azqblz50qK/oUtCEuh3G+6\n6SZJ0uWXXy4pss5msxme14bxUP/4MLZEjRJuwZcHBga0adMmSdE3uVdbvt4JHGsB5/j4eKFoBAUO\nG6fv41+UQf/666+SJn3zgw8+kCRt375dUrG4gnZFYf3yyy+SpNtuu01SVBc7duwIdiMEmEsoWipf\nSJ4utidETx/GvihsfIrXN2/eLCnaZs2aNSF8ipLnmqRMaIuLL75YUuwDKLCzzjpLknTjjTeGtuA+\nuNZ0+rwrKYfD4XBki44pqXbxaRgCDAjWSgyZ2Z6ciC1J37dvX0iGosJIQqMCUAUwfb7D3k+z2QyL\nK0mWl22vkgvaLUTmeW2BCDaGudbr9cBmKUu1Zbh2QWVvb2+h9Jrvpa34fhQVakGKrCo3hWp9AaUz\nY8aM4BPcM4wQxog9UUOw+x9++EHSZP7j7LPPlhSLLLA7qgGVgN9jb2yJrf/666+gCrjHZcuWSYrt\n3EnYZR72/5VKJahq2DnPjF/xNzb69ttvJbUuJSGnhG/RNpRM0yZEXpYvXy4pql6Uw9jYWFBQ+Ct/\n54Cybcvwh6GhoZD7oQ9TIAHIkdL/UaxEo9auXRvGTjuGEC3gM/ykrfBn2mjFihXhe/BPW3cwFbiS\ncjgcDke26HhOKi2RJm7K/2AFMAhYLMyB2ZkFagcOHAgKyZb1fvLJJ5KK5e3M9MSi0+oYcjTAbrSa\nC9ptOQLz5Ll4D7aDTcIUBwcHA2u1VX6oTWuzRYsWFTYIhU3B4FBUsN2+vr6CQsHmuQG/o70PHToU\nYvk2H4CS4icsnlJ08h6rV68OtqI0GoZK26CoiPVjww0bNkiK9tq6dWuoKkyr3aQ8lJS1n91QN92K\nCzaObYiAYBN8krGBXF8a8cBOXANFiqpAMVk1wTizd+/eYNt0A1sp5nQ6Cfoa/caq+Wq1WrqMZNWq\nVZJiHgk74pco2p6enmA/qiMB16QtALkr+sQzzzwjSVq/fr3uuuuuls/il56TcjgcDse/AidMSZVt\n5ZGyLFtNRw4EFs5P2Hq7hXf8j1n9+++/lxTj1bxuWYmtdqvX64FRlC3yzA3pM2FDGAtsEZtSJUXF\nzvDwcFBMsFqYJiwdxk+8vtlsFramgmXZhYAwqfHx8eADdqPPXFB23MbY2Fiwo80XEdOHcaNQUUUo\n/LGxsfAZ1klRYXXuuedKiiqXn+S1UAZpO9hFxfYZcoDN8aU5XvwCu7GmiegINoDVW6XaaDTCZ6zP\n46eoBZQrapj7oO8PDw8X1F+O4LlY68g9n3LKKcFeVm3hM/y0mx0wttbr9dD/GQ/smkCqUhkfGFtZ\nEPzqq69Kmox8MT7QzseTf85rdHA4HA6HI0HHq/uI+baLUTJDMwvDTm21CUyoUqkEdgHTtetaYCF8\nn62YQfENDw8XNkPMjfGXVRlWq9VgK3JD9igN4vio0kqlEhgU6sceMMdn0p0/YEowYf7mWu1yY7RR\nWfVXp2DzTHaHk5kzZwY/4bmw1YUXXigp5kD5P2rxzTffDNdivRPVZTBUGCuqlPbFZ1FrXLvZbIY1\nKdwjSjanClSrSLm3dFsnmP+WLVskFdd54ccof9TTyMhIGA8YL2ze1VZaokitUmg0GgUFUhYB6iTs\nLh2gVqsF30AZUWlHfpPqTzumoTL7+vqCP9qt1egPrDXlPvBjxosbb7xR0qTa5T6OJxcF8hp1HQ6H\nw+FI0DElBbuCMR05cqRQVQbLgp2SR4Gdwgp4/8jISPgMbJ1cgFVMdvPadrkyu4eg3Tut07AKxObV\npOIGqAAWhH36+voC84RpYhsUlN0r7PDhw4X1V6gAe9gazLe7uzvbKklgq9CwQ39/f2CcsHeYK++1\n62rwGfJOtVotqB/UAdckD2M352VdFUqKdjh06FBYF4jdc6hCAzZHZFXoyMhIiyKSol1R4tiVdWbk\nlK+//vrwOuMDtkcJ4Ld8/2effdbyOoo2XTtIvyDfMp095v5p2INDbT/q7u4OygW/wL5U9+FDdo1e\nulE04yz7/dGHP//8c0lxPKYtPvzwQ0nRT9khRSrmSq1PTAWupBwOh8ORLU64kiqL8dbr9QIzYGcJ\nu27Kbg0Ps9y3b194D6yDHAnMHnbFmgmruKhcqdVqQeXZ+HSuwG6NRiOwQ+LWsCGeH5vClubNmxeY\nKEoVVgvrYg0OrLfZbJYqNN4DI03XRlk75p7r4++5c+cGG8MEYarvvvuupGgza1/s/+WXX4aKU4Cf\n0zawYRgptqQdqMxK95dst4NLLihjzXv37g25Y/wUxm+P5cGPqNSj7+/duzcwe3bjRnVx7W+++UaS\ndN5550mKfR+lmh5jYw9HzfnkA5tHnzNnTvAH7IZKR+XwGXJVfJZ+u3z58tD/8Uvr09ge32eMRY2h\n6pcsWRJ8mfWCx3OAbF6jg8PhcDgcCU6Ykipba5QyU2Zzu3sBDB6mZCvzYKnz5s0r7I5A3J/YKKyA\ntRIwC1sp2NXVVdidwla75Iqenp5C5RzMFPvYPEG1Wi3d0cOeCYV62r9/f2D1tAHslTZknVC6h6Bl\np7nY06oPu26vVquFe0ep4jewSV7HzqzvwYannXZaaBvaBCXAZ1mlT8z/+eeflxTbDLWR3u/xMNR/\nGrZ97RHxjUYj/G4PggQ8M6ye8+LeeustSZO5K3afwG9tXgs7/ec//5EUlRT7HpK/6enpCe+1hzDm\n4KPcm83l8tyjo6NBcWID/qZ/khtNowNS3IVj6dKlYQ0aUSmujyrDfuzsT/SJtqSKcv/+/UF1AesL\nU0HnLe9wOBwORwk6Vt1nWWtvb2+Yie1R73Y/PmKh5JPSyiB7XhIxUfJLxKFtpR6fS485T89AOhkA\nS6lWq+GeeZ70SOcUKKn9+/cH28CuUKHYH5WEali5cmV4L/ujwaLIxXB9vn/27NnZKlKbz8FH8J0d\nO3YEdUOFHhWR9phu9p5DJcFkV61a1bIXoBTVJkwfZUCVFT+xdZqb5bpcyx7R3knY9rVnm3V1dRV2\niMHm5EdRMqgiWD5+vXjx4rBOB/9E3fMeq3KxH9Vq6WkH6T54Uh52BFZ9MD4xDm7fvj08M3k4fA21\ng6JP80ZSHBf37NlTqO6l73It1BHVqPglr/P+3t7eYEfu0eawp4KObzBrBwQpOggPz0CHc7GYDMdN\nt4Fn0RoGJplny9vTrX1SMNj09PQUwn05Jk9TpOEU7MrgRae3hRTYqV6vB+fCNtib//OZdEAhfECH\np00YvO0GopVKpZDwzSlEJRWLELDh6Oho4VgNBgWIFZ9hEiPckpZBY18GUwYFrr127VpJsa2uvfZa\nSbEf0JZDQ0MhnMgEQBvlcpBkCp473U6Hvsz9U85sDyfFVmvWrJEUQ6GHDx8O/ZKQIGQW++GDn376\nqaRISAkd0oYrVqwI92gXIOdgT9tfeA78YcuWLYEk2T7LJIS/3HLLLZKi37KIetGiRYHY02fffvtt\nSdGnuTbhPg6WpQ1pm/Hx8VAwYTfq9XCfw+FwOP4V6LiSSpPozPqwJhgs4Q4UEzO5DcPt2rUrzNQk\nYi2DgCnxWaS+XbyXbtKau4KypfvNZrPAuu0x3fwN+/zjjz/CZ7ARYRISpXwP0v3gwYOBgRHeu+66\n6yTFNoSppRsJ53g0dwobMsMPf/nll5DYRyniszB/wiocTU6xD8n5F198MdiGsmAS+KhS2CfXOvPM\nMyXFaEC61Y0to86B8VuUlfSPjo4G2xLCp28TjrKHZ+KDHGLa398f7MY4wWdQYSz6Z2zA52H8KK/F\nixcXCqrscRidhA2JohR5rl27dgV7pmF8KW5mfP/990sqHuXC+9OQJ+FT2sRutcZiaEr7+Rw+v2XL\nlvBexlcwnTCqKymHw+FwZIuOKykwMTERGA2xUJgkMzqvw2b5P2y2r6+vsBAYNoWygqWS4LfHfcBW\nDxw4UEj0EgPODbYkNT32hOeB5dgDB7FTrVYrHBtNO8CkUgUlTdqDjVVvv/12SdJFF10kKbJamCms\n79ChQ+E6ll11GjbmbxeRb968OSSN8Q3rT/gkbB7G+sYbb4RrwWZRqJRTo0Zh7+RnUFLcD8prcHAw\nXD+n7ZCAzTdz/2mREkoQn+BZeS76PNdCuaNcFyxYEPo210BpolT5vnvuuUdSzGtR7GOXAEitC9al\nPLbwssfFp8cKSZM5PmyNP2zatElSVPJEOlCXLKHAnzds2BDGU8YG+jBtQh/gaA5sQ/4/Va4oXrZj\n4r12A+GjwZWUw+FwOLJFx4/qSNmrZdh2QRrVJTB8ZniY7tKlSwMD4idKiTyALQ1mgRo5q3TjxbLF\nvLnkUsriuumiPpgVz5duJJn+vXDhwmBX4tgwebsdFcx0zpw5uvnmmyVFVoXd+YwtgR8bGwtMMDd7\n2vvg/vCJnTt3hueCZWJHFkOiBFCyzz33nKRoyzvuuCMweVgu6uDBBx+UFKtaeR11Rg6A9uFepvIs\nnQA2SDeAllo3GaWSDBWJn6Kg6L82T4ianJiYCOMGiopxgzaiVPqaa66RFI9Qp4/A6vfs2dN2XMoF\n2M0u1EeNnnHGGSHqY8vx6X/4IW3Cs6Pmd+7cqdWrV0uKm/jSBnbjXtqOdiVqlS6qxo7WFzwn5XA4\nHI5/BTqek2q3DREzM9UkMEtmX+KcMFBm9Isvvjis16EyjWvCBrg2rIvP2jVAKQvIaYFkijLmP2vW\nrPDc6ZHy6d8wKPJQc+bMCbFmGCgMFdhKp/7+/pCT4l6wP0wOVgsLGxkZye6wwzKgGPGNlStXBqXE\nJqbkoh555BFJ8QC41157TVJUQ7D4xx9/vFClBwPFF8nFkjNhTRaqjc/Pnj27kJegnXNAWtEpRV8g\nXzkxMaF169ZJikoKFm4Xn/O33YaqXq8HtY5quPvuuyVJGzdulCQ9++yzkuI6KbsOLa10sxtg2/VS\nOcBuPJBuSMxiZ7aSYwstbE7fJkfFWjHsO3fu3HBoIYqT99pKXYDd8WP8dNu2bQUbH89Y6krK4XA4\nHNmiYzTBHn6VbjBrq8tgCsRCYets/fHwww9LmjzCmyMTYFxUlcCA+duqhXTXCmlypkdt2B0SclNU\nIF0LBfu325HY9Twoyvnz5wcGZHMHsDLUEFur3HnnnSGH8Mknn0iKuQK+j+9Pt6uy685y2R7J5iH4\nG4V53XXX6YYbbpA0eeSGFNk6yopnW79+vaSoHlCc1WpV77//viSFn3wG/yY/QF7GbrCMPw4ODha2\nWMpxOx+ALdhh4+effw7rzbAxCtz2NWyD+iQ/2tvbG+xBH6eqj91nqAbmiBTGj/RoG2nSrlw3x63Q\nyrbtwq69vb0hT091KbtBUEmKn9LveR0bXXnllcGXbN4aO5PH5/WVK1dKihGvtFYAlWXvfTr2zWN0\ncDgcDoejDTpe3Qfjr1arhUMGmcEBbBwGwQFoDzzwgKRJBgVzP/fcc8P/0mulR59LxUMAURETExOF\nfaZyrPhJkR4vgK14XtgN78FOabUN7IZqKGL42JJ8y3333SdpMs/Ckdx8pqzyEjSbzZMmJ2WPRkiP\n2YA9oqzsCn9blQazfemll4JaQF0Ry0edwVDZaJaqVqIDfL5er4c2y13lS63H3kuTueWXX35ZUlT2\n6bEoUlQJjA3Yirzc4sWLQ9+2CgP/Irf93nvvSYrVcKgH2nRkZCTco4305GDXsnEIW82bNy8oF3JT\nqFZ8zR63wW4RKNrff/89KE6uRVUpbUBky24AbsfvU089Nbxm18pNZ92ZKymHw+FwZIuOl67ABGfO\nnBmYAayUmDKqh9fJK8FiqZz64YcfQp4KpsV6KGZ3KlSsooJBpTHpsoMac0HZDgnVarVQ5cV7eE4Y\nFcqn0WiENWSvv/66pMhu7733XknSQw89JCmqiJ07dwaFwPVYU2RzJOkK89zsWAbYHmp/eHg4MFR7\nFAyMkWcjPwJjxa8GBgZCG+DHqC6q+ewuEuRPUVCo0/nz5wcVkUteLwXs2dqRqrGBgYGgPHlGKhnp\nvyhy9vajipHXa7Va8DkUBa9hL/oC1+L/jA3k/tIKY4scfNYeGEj/p9Kx0WgE9WN3Hcdu7AaBz9nd\nTbZu3RrswbW4PnbmmvQFxulbb7215bsvv/zyML7+f/Y/zc+zHQ6Hw+H4P3RcSYGRkZHA6mFEsETi\nqgMDA5KKsXxe37BhQ5j1LVtCQVGBAttK12lJrXmdHOLQR0PZWVjd3d2FuLw9W8fuBr979+5QmYc6\nYsU5ipVKPtrhxx9/bDlKXorK1O4g3+5489x2mW+3m7wUmWOqpFh3gv1gmdjDKkryICMjIyH+j1Il\nn4fP8hl7/DkMNd392sb4j2dF/z8Fu04KO/L89957b1A35DvZ7YD+Sb+1pxjQNrt27Sr4Fn7MuEBe\nmvwL1anYN/VDojj28NTjOazv74b1A+xKzq1SqYQ8GyrIVk+ioPBf+jI+le6F+Morr0iSPvroo5Zr\nYS/a5oorrpBUPM+rWq0Ge5IDP57qyY5PUulRHXROG3ojvITxMDSLzEgoDwwMhEbiGnZ7eRKxZZtf\ntltcfLIgDQekob9276ETpqXiPDuJUgYMJn4W/mHDoaGh0GGwM7AHB6YhirTNc0JZYQz/T48xILzJ\nRGOPRCE8hc/yc/v27XrppZckxeUAfJbBE3LAoED4jwGdsFm60NSGfHMITwF7zAXPs3z58jBxUGbP\nT/ySvk0yn4knLaQgpGqXXTAWsFCYNoEwpMRBmvRrfBs75jA5AbvZrV1cnx7KiX8SUmbS4rOQdexN\nyG7btm2BiEE8KdzhWoyldrE/trMbKUjF8XU68HCfw+FwOLJFx5UUaDabYeZlRrZHwCMjOR6BRD+y\nvV6vhxAg4QHYKmEoe7AfMteGDHJj+dNByqJtkpVQnlVY6bEPsDAYvt1SJS3TtWWntrQ9181526Fs\nMW+qTrAR/oUSgI3D9ElUw2AJZ5111lnBF6+66ipJUVnwGRh+egy3FNkvbLidksrJvrZwAqSqD9ad\nbvIqRYVKX8eO2Az79/b2BvuxTRohfuyJnbAfbD4N7afvyxXt/NH+H99h7CSEbAspeB+FKhQA9ff3\nh4X5KFHegz1RWNiTn1ZZ9fT0lEYlpgNXUg6Hw+HIFllRB7twjrwJDCfdpFSKjCI92M/mopjdiW3D\nIEDZ9vcnM1K2YgsoeF671VNfX19IrhLbx5YwfxRuyuTtYXv2+9oVTOSKY91jpVIpqH27hQx+h0KH\nybJQNy2cYJsj/BxfhenzN21mF5e3u+ccCiZA2YLNVGXbRfTYz26PRDQF2+GjtVotqC5ydvbIH7sw\n2OZlQbVaLVX8OURWjrWpQK1WC/3aHlTK+IcP2eK0dKE1eSrGVWyNPfleCiawI9dOxwDsZoukprNk\nwpWUw+FwOLJFVkoKWNZiK0LsQjR7xIcUY7DM+nymbFuOHJjS34V2Zd7AsjDsND4+XlCw/CxTm2Nj\nY4W2sWzPMvuTQVGV5aZSZphujCxFFknuhJ/2+WfMmBHsaY+jwL6w3GOp0rRty+45R6Ss2t4nbJ2y\ncRtdoaK3XQ4ONWAr2bAbr9s2TNl9mfrLabG0beP0/m15ui1bt+MiVX3p82Enok6oM1srgP1sVCb1\nX5s/O57q03ws73A4HA6HQeVkYLYOh8Ph+N+EKymHw+FwZAufpBwOh8ORLXyScjgcDke28EnK4XA4\nHNnCJymHw+FwZAufpBwOh8ORLXyScjgcDke28EnK4XA4HNnCJymHw+FwZAufpBwOh8ORLXyScjgc\nDke28EnK4XA4HNnCJymHw+FwZAufpBwOh8ORLXyScjgcDke28EnK4XA4HNnCJymHw+FwZAufpBwO\nh8ORLXyScjgcDke28EnK4XA4HNnCJymHw+FwZAufpBwOh8ORLXyScjgcDke2+C9Pvc7+Fcpz9wAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71427dcf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i + 1)\n",
    "    plot_image(weights1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Sparse Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ageron/dev/py/envs/mltst/lib/python3.5/site-packages/ipykernel/__main__.py:3: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sparsity_loss_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcjXX7wPHPNTKYDEaN7Ma+hZBKZUlpoWwpIsnyPHqS\n6kGFNku0/RCylFLTQhKKaLWTtZAlWce+NoxlBrN8f398z8wzM2bGrOc+y/V+vc6rzjn3uc917jHn\nmu96iTEGpZRSytMEOB2AUkoplRZNUEoppTySJiillFIeSROUUkopj6QJSimllEe6xukAcpOI6JRE\npZTyIMYYye5rfa4FZYzRWxq3119/3fEYPPmm18d/rs306YYWLfT6uOOWUz6XoJRSKiPjxsEzzzgd\nhcoMTVBKKb+xZg2cPAlt2jgdicoMTVB+onnz5k6H4NH0+qTPl67N2LHw7LOQL1/undOXro+nkdzo\nJ/QUImJ86fMopXLPgQNQvz5EREBwsNPR+AcRweRgkoRPzeJLT1hYGPv373c6DOWQChUqEBER4XQY\nymHvvw/du2ty8iZ+0YJyZXEHIlKeQH/+6vx5CAuDDRvsf5V75LQFpWNQSimf9+mn0Ly5Jidv4xdd\nfEop/5WQYKeWf/KJ05GorNIWlFLKpy1YAMWKwR13OB2JyipNUEopnzZ2LPz3vyDZHglRTtEE5aPC\nw8Np0qRJ0v3g4GCdyab8zu+/w+7d8MgjTkeiskMTlMMqVqzI4sWLk+5/9dVXFC9enBUrVrB//34C\nAgJISEjI1rkl2Z+M586dI0xHiJWfeftt23rKn9/pSFR2aILyIOHh4fTr148ffvghqfUjHtwvER8f\n73QISqVr925YsgT+9S+nI1HZpQnKQ3z44Ye88MIL/Pzzz9x6661Zfn1kZCRt2rShaNGi3HbbbezZ\nsyfF8wEBAezdu5e1a9dSqlSpFOuC5s6dS7169QC7G/xbb71FlSpVCA0NpXPnzpw5cwYgqUU3bdo0\nKlSowN133w3AZ599RlhYGKGhobzxxhspWoWZOd9nn31GhQoVKFGiBKNGjUqKKyEhgVGjRlGlShWK\nFi1Ko0aNOHz4MAA7duzg3nvv5brrrqNmzZrMmjUry9dM+bbRo+Gpp6BwYacjUdnm9Hbsuby1u0lL\neo97grCwMPPwww+bkiVLmi1btqR4LiIiwgQEBJj4+PirnqdTp06mU6dOJiYmxmzdutWUKVPGNGnS\nJOn5gIAAs2fPHmOMMVWqVDG//vpr0nOPPPKIeeedd4wxxowdO9Y0btzYHDlyxFy+fNk89dRT5rHH\nHkuKR0RM9+7dTXR0tLl48aLZvn27KVy4sPntt99MbGysGThwoAkMDDSLFi3K9Pn+/e9/m0uXLpnN\nmzebAgUKmB07dhhjjHnnnXdM3bp1za5du4wxxvz5558mMjLSXLhwwZQrV86Eh4ebhIQEs3HjRhMa\nGmq2b9+e5rXx5J+/yhvHjhkTEmLM8eNOR+LfXL972f9Oz8mLPe2W3QQFuXPLjrCwMFO0aFHTrl27\nK57LbIKKj483+fPnNzt37kx6bMiQISkSlIgkJahXXnnF9OzZ0xhjzNmzZ821115rDh48aIwxpmbN\nmmbx4sVJrzty5IjJnz+/iY+PT4onIiIi6fnhw4ebLl26JN2Pjo5OkaAyc74jR44kPX/LLbeYmTNn\nGmOMqV69upk/f/4Vn3fmzJmmadOmKR7r06ePGT58eJrXRxOU/xkyxJj//MfpKFROE5R28ZFb6Sn7\n7z9lyhR27txJr169svX6kydPEh8fT9myZZMeq1ChQrrHd+nShblz5xIbG8ucOXNo2LBh0mv3799P\n+/btKV68OMWLF6dWrVrkz5+f48ePJ70++fscOXKEcuXKJd0vVKgQ1113XdL9zJzvhhtuSPr/oKAg\nzp8/D8DBgwepVKnSFfHv37+fNWvWJJ0zJCSE6dOnc+zYsUxdL+Xbzp2DDz6AAQOcjkTllCYoD1Ci\nRAkWLVrEihUrePrpp7P8+tDQUK655hoOHjyY9NiBAwfSPb5mzZpUqFCBhQsXMmPGDLp06ZL0XPny\n5fnhhx+IjIwkMjKS06dPc+HCBUqVKpV0TPKJG6VKleLQoUNJ92NiYvjnn3+ydL70lCtX7oqxtMTH\nmzdvnuKcZ8+eZeLEiVc9p/J9U6fC3XdD5cpOR6JyShOUhyhZsiSLFy/mp59+on///kmPG2O4ePEi\nly5dSrqZVM21gIAAOnTowNChQ4mJiWH79u2Eh4dn+H5dunRh/PjxrFixgkeSLRLp06cPQ4YMSUpw\nJ0+eZN68eSniSa5jx47Mnz+fNWvWEBsby+uvv57i+ayeL7nevXvz6quvsnv3bgC2bNnC6dOnefDB\nB9m5cydffPEFcXFxxMbGsmHDBnbs2JHhZ1a+7/JluzD3xRedjkTlBk1QDkveGilbtiyLFi1i9uzZ\nvPzyy0nPBwcHExQURKFChQgKCmLJkiVXnGfChAmcO3eOUqVK0bNnT3r27Jnu+wB07tyZZcuWcffd\nd1O8ePGkx5977jnatm3LvffeS9GiRbn99ttZt25duuepVasWEyZMoFOnTpQuXZqiRYtSokQJChQo\nkK3zJb/fv39/Hn300aTX9u7dm5iYGAoXLszPP//MV199RenSpSldujSDBg3i8uXLGV9s5fNmzIDq\n1aFhQ6cjUblBy22oXHXhwgWKFSvG7t27MxwHcyf9+fuHhASoUwfeew9atnQ6GgVabkN5gO+//56Y\nmBguXLjAgAEDqFu3rsckJ+U/5s6FoCC45x6nI1G5RROUyrHvvvuO0qVLU7ZsWfbs2cNXX33ldEjK\nzxgDI0bAa6/pprC+RLv4lM/Tn7/vmzcPXn8d/vhDE5Qn0S4+pZRfS2w9vfKKJidfowlKKeXVfvoJ\nYmKgfXunI1G5TROUUsprGQPDh9vWU4B+m/kc/ZEqpbzW4sUQGakFCX2VJiillNcaMQKGDIF8+ZyO\nROUFTVBKKa+0fDkcOgTJtpJUPsaxBCUiISIyV0TOi8g+EXksneMCRWSKiBwTkVMi8p2IXH2nUS8R\nFhZGwYIFiYyMTPH4TTfdREBAAAcOHODw4cN07NiR0NBQQkJCqFevHp999hnwv6J/RYoUoUiRIgQH\nB1OkSBEt4Kd83ogRMHgwXHON05GovOLkj3YScBEIBRoAC0RkkzHmr1THPQ/cCtwInAU+AiYAHd0Y\na54RESpWrMiMGTPo27cvAFu3buXixYtJ+9J169aN+vXrc/DgQQIDA9myZUuK0hIiQlRUlEeXh1cq\nN61YAXv2QLduTkei8pIjLSgRCQI6AK8YY2KMMauAeUBa/9zCgJ+MMaeMMZeBr4DabgvWDbp165Zi\n9/Hw8HC6d+8O2N2+169fT/fu3SlYsCABAQHUq1eP++67L8U5dCGq8hfG2Fl7r70GgYFOR6PSZAxk\no3RQak518VUD4owxyYv9bCbtxPMxcKeIlHIltq7AQjfE6Da33XYb586d4++//yYhIYGvv/6axx9/\nHLCto8aNG/P0008zc+bMFDWfktMEpfzFokVw7Bi4fkWUJ3rvPVi1KsencSpBFQaiUj0WBQSncexO\n4ABwGDgD1ABG5Go0Q4faJeipb0OHZv749I7NpMRW1C+//EKNGjUoXbp0UtKZNWsWTZs25Y033qBS\npUo0aNCADRs2JL3WGENoaGhSddnixYvz999/5ygepTxRYutp2DAde/JYP/wA775r95/KIUf24hOR\nm4CVxpjCyR7rDzQzxrRNdeyXQEGgJxANvAQ8aIy5LY3zmuQF85o3b07z5s09ei+2ihUr8vHHH1Ol\nShWaNm1K48aNefDBB+ncuTP58+cnIiKC8uXLJx0fGRnJgAED+PXXXzl48CD79++nUqVKxMXF6RhU\nOjz556+y5vvv7cSIzZt1Ya4nWvrppyzt2xc6d4Zy5Rg2bFiO9uJzKkEFAZFA7cRuPhEJBw4bY4ak\nOnYLMMQYM991vyhwGrjeGBOZ6liv2yw2MUG1aNGCu+66i40bN3L06FECAwPTTFAA27Zto27dupw6\ndYqzZ89SqVIlYmNjCdDf2DR58s9fZV5CAtx8sx17atfO6WhUmtauhb174TE7KdsrN4s1xkQDc4Dh\nIhIkIncAbYDP0zh8PfCEiBQRkfxAX2wii0zjWK82bdo0Fi9eTKFChVI8PmjQILZt20Z8fDznzp1j\n0qRJVKlShZCQEMB28ekXsPJ1c+faVlPbtlc/Vjnk1luTklNucPJP7r5AEHAC+BJ4yhjzl4jcKSJn\nkx03ELgE7AKOA/cDPrMtZPJuuYoVK9KgQYMrnouOjqZ9+/aEhIRQpUoVDh48yLxk/bsiQkhISIp1\nUO+99577PoRSeSw+3racRozQHcv9idaDUj5Pf/7e78svYdIkWLlSE5Q3yWkXnyYo5fP05+/dYmOh\nVi344ANo0cLpaFQKp07B9den+7RXjkEppVRmffQRVKyoycnj7NwJN94I6azNzA26kkAp5bHOn7f1\nnhb61NJ8HxAZCQ89BG+8AeXK5dnbaAtKKeWxxoyxLaf69Z2ORCW5fBkefhhat4bevfP0rXQMSvk8\n/fl7pxMn7NjT+vW2i095AGOgZ0/bgpoz56qFuHI6BqVdfEopjzRihN1vT5OTBzl+HP75B2bMcEuV\nSL9oQYWFhbF//34HIlKeoEKFCkRERDgdhsqC3bvhttvgr78gNNTpaFR26TTzZNJLUEop79K5M9Sp\nAy+/7HQkKic0QSWjCUop77d+vd1rb+dOuPZap6NROaHroJRSPsMYGDQIXn9dk5NHiI119O01QSml\nPMb338PRo3aimHJYbKydSp4LdZ2yS2fxKaU8wuXLMHCgLcaqxQgdZgw88wzkzw+tWjkWhv4zUEp5\nhMmT7ZTyBx5wOhLFmDGwZo3dndfBvxZ0koRSynGRkVCjBixZArVrOx2Nn5szB559FlavzvE2RjqL\nLxlNUEp5p+ees0MekyY5HYmfi4uDJk3sDyIX9pfSBJWMJiilvM+OHfY7cft2XZTrERISbOniXKDT\nzJVSXm3gQDu1XJOTh8il5JQbdJKEUsoxv/xiW1CzZzsdifJEnpMqlVJ+JS4O+veHd9+FAgWcjsZP\nJSTYKeUeShOUUsoRkyZBiRJ2WyPlAGPg6adhyhSnI0mXTpJQSrnd8eO2WviyZbbmk3LAm2/C11/b\nH0KRInnyFloPSinldQYNgu7dNTk55osv4IMP4Lff8iw55QZNUEopt1q9Gn7+2dZ6Ug5YvBgGDLD/\nLV3a6WgypGNQSim3iY+3W7y9845H/+Huu4yBN96wXXtesGWHjkEppdxmyhSYPt0Oe0i2RyZUjuTi\nQtyr0Z0kktEEpZTnOnXKjjn9+ivUret0NModNEElowlKKc/Vpw8ULAjjxjkdiXIXncWnlPJ4GzbY\nunc6McLNYmNtuQwv7U/VSRJKqTwVH2/Xg44aBcWKOR2NH0lIgCefhPffdzqSbNMEpZTKU5MmQaFC\n9rtSuYkxdir5gQPQu7fT0WSbdvEppfLMoUMwbJgtzOqlvUze6e237WyU5cvtXwdeShOUUirPPPcc\n9O1rq+UqN/nkEzuff9UqCAlxOpoc0QSllMoT8+bB1q3w5ZdOR+JH4uNtyfYff4QyZZyOJsd0mrlS\nKtedO2c3KggPh7vucjoa5RRdB5WMJiilPEP//hAZCZ9+6nQkykm6Dkop5VH++MN2623b5nQkytvp\nNHOlVK6Ji4N//9tuBnv99U5H4wdOnLDjTj5KE5RSKtdMmADBwfDEE05H4gdOnICmTWHhQqcjyTPa\nxaeUyhW7d8PIkbBmja55ynNnzsB990GnTvDQQ05Hk2cca0GJSIiIzBWR8yKyT0Qey+DYBiKyTETO\nichREennzliVUhlLSLAbFrz8MlSp4nQ0Pu7CBWjd2raehg51Opo85WQLahJwEQgFGgALRGSTMSbF\ndpIich3wA/Ac8A1QACjr5liVUhn44AO4dAmefdbpSHzcpUvQvj1UrQpjx/p8U9WRaeYiEgScBmoZ\nY/a4HvsMOGSMGZLq2JFAWWNM90ycV6eZK+Vm+/dDw4awYgXUrOl0ND7u7FkYMwZeecXuUu7hcjrN\n3KkuvmpAXGJyctkMpFWD+DbgtIisEpHjIvKdiJRzS5RKqQwZA//6l92XVJOTGxQpYrv1vCA55Qan\nPmVhICrVY1FAcBrHlgXqA/cAW4F3gRnAnWmdeGiyPtnmzZvTvHnzHAerlErbtGl2Qe4LLzgdifIE\nS5cuZenSpbl2Pqe6+G4CVhpjCid7rD/QzBjTNtWxm4DfjTG9XPeLA6eAosaYc6mO1S4+pdzk8GG4\n6SZYtEhLuKu0eWsX307gGhGpnOyxekBaa8//BFJnHQP49uigUh7MGHjqKbtTuSanPPTNN3D5stNR\nOMaRBGWMiQbmAMNFJEhE7gDaAJ+ncfgnQHsRqSsi+YFXsa2vs+6LWCmV3Gef2ckRQ4Zc/ViVTe+8\nYydDRKUeDfEfTu4k0RcIAk4AXwJPGWP+EpE7RSQp+RhjlgBDgIXAMaAS0MWBeJVS2MQ0cCB8/jkE\nBjodjY8aNw4+/ND2n4aGOh2NY3Q3c6VUpiUkwD33wL33wqBBTkfjo6ZMgbfegmXLoEIFp6PJEd3N\nXCnlNuPH27WiOmsvj8yfD6NGwdKlXp+ccoO2oJRSmbJ9u91dZ+1aqFz56serbLhwAY4fh0qVnI4k\nV2jBwmQ0QSmVN2JjoXFjuyi3Tx+no1HewlunmSulvMgbb0CJErbWk1LuomNQSqkMrVtnx+03bfL5\nvUmVh9EWlFIqXRcu2OKDEyZAqVJOR+NjFi6Enj2djsKjaQtKKZWu55+HW2+FRx91OhIf8/33NjnN\nn+90JB5NE5RSKk3ffANLlsDGjU5H4mPmz4devWySuuUWp6PxaDqLTyl1hQMH4OabYcECaNTI6Wh8\nyLx5tvSwn1xYncWnlMpV8fHw+OO2xpMffIe6jzHw9dd+k5xyg7aglFIpjBhhNzL45RcI0D9hVQ7o\nVkdKqVzz228wcSL8/rsmJ+W8TP8TFJESIjJORHaLyEUROSgiC0TkgZwGISIVRCRBRBrk9FxKqeyJ\nioKuXeGDD6BMGaejUSqTCUpEKgAbgZbAS0AdbAn2hcDkXIhDuLIooVLKTYyxu0Q88AC0bXv141Um\nLF0KMTFOR+HVMtuCmoxNIA2NMbONMbuMMX8bYyZiK+EiIuVEZK6InHXdZotI0t9hIlJWRL4VkX9E\n5IKIbBeRxNUVe13/3eBqSS3OrQ+olLq6yZPh779h9GinI/ER06fDY4/Z4lkq2646BiUiIcB9wBBj\nzBV/DhhjEss9fgdEA81d9ycCc4HEif6TgUCgGXAOqJ7sNLcA64B7sSXe/bfGsVJu9vvv8Prrdvyp\nUCGno/EBH31kL+ivv0KNGk5H49UyM0miCrYLbkd6B4hIS2y3XyVjzEHXY12A3SLSwhizGCgPfGOM\n2ep6WfI/LU66/htpjDmRxc+glMqmqCi7S8TEiVC1qtPR+IDx420zdOlSvaC5IDNdfJmZIlgDOJKY\nnACMMfuAI0At10PjgFdF5DcRGaETIpRyljF2t53779etjHLF7Nk2QS1frskpl2QmQe3Cjj/VzOCY\njCY5GABjzDQgDJgGVAV+E5HXMh2pUipXTZgAEREwZozTkfiI1q1h5UqthJuLMrVQV0QWAnWBasaY\n6FTPFQUaAT8AlY0xB1yPV8Imt3uMMUvSOOeLwLPGmLIiUgo4DNxqjFmf7Q+jC3WVypT16+336erV\nWh1X5R13bXX0NLaVtEFEOopINRGpLiL/ATYbY37FTm74UkQaiMjNwBfAhsTkJCLvich9IlJRRG4C\n7ge2uc5/AogB7nOttyqS3Q+klMrY6dPQqZOduafJSXmyTCUoY0wE0AD4BXgL2AwsAh4EEmtstsVO\ndljieu4I0D7Ve43HJqWfgGPAk67zxwP9gN7YltS32f5ESql0JSRAt27Qpg08/LDT0Xix+HiIjr76\ncSpHdC8+pfzI0KGweDEsWgT58zsdjZe6fNlm+cqVYdQop6PxaLoXn1IqU77/3i7R2bBBk1O2RUdD\nx472Ar6mc7zymm4HqZQf2LXLTimfNQtKlnQ6Gi91+jS0bAmhoXZKecGCTkfk8zRBKeXjzp+HDh1g\n2DBo3NjpaLzUqVPQrJmtgPvJJ3CNdj65g45BKeXDjLFbwhUqBNOmgWR7NMDPxcbaVlOnTnoRsyCn\nY1CaoJTyYWPGwJdf2vWjus+ecjedJKGUStOSJfDOO7B2rSYn5Z10DEopHxQRAV26wOef68472aI9\nMR5BE5RSPubcObsQd9AgO+lMZdHMmXZWiXKcdvEp5UMSEuDxx+HWW+HZZ52OxguNGwf/93+wYIHT\nkSg0QSnlU155xS7XmTVLJ5tlSUICvPSSTUyrVkH58k5HpNAEpZTPmD4dvvrKTooIDHQ6Gi8SGwtP\nPmnLs69cCcWLOx2RctExKKV8wLp18Pzz8N13dqMDlQX58kGjRvDLL5qcPIyug1LKyx0+bMecJk2y\nkyOU8hTuqgellPJA0dHQrh3066fJSfkeTVBpCAgIIF++fAQEBFxxy5cvHz179nQ6RH766ScCAgKI\n1po0fis+3s7Yq1kTXnzR6Wi8iPayeA3HEpSIhIjIXBE5LyL7ROSxqxyfX0R2iMiBvI7t2LFjHD16\nlGPHjjF16lREhOPHjyc9Pm7cuGydNy4uLtdiNMYkNp9z7ZzKu7z4op2x99FHOmMv0+bOhfvu0yTl\nJZxsQU0CLgKhwOPAZBGpmcHxL2Kr8Oa5EiVKJN2KFSsGQGhoaNJjwcHBAAwYMIBq1aoRFBREpUqV\neOWVV1IkocGDB9OoUSOmTp1KpUqVKFiwIAkJCZw7d44uXbpQuHBhypQpw5gxY2jZsiVPP/100msv\nXbrEgAEDKFu2LIULF6Zx48YsWbIEgL///ptWrVoBEBwcTL58+VK8Vvm+99+HhQthzhydsZcpxsDo\n0bYv9K23NKN7CUemmYtIENABqGWMiQFWicg8oBswJI3jKwJdgP7AVHfGmpFixYrxxRdfULJkSbZs\n2UKfPn249tprGTx4cNIxO3bsYN68ecydOzep27Bfv36sW7eOBQsWEBoayquvvsr69eupWrVq0uu6\ndOnCqVOnmDVrFiVLluS7776jVatWbNq0iWrVqjF9+nS6du3Kvn37KFSoEEFBQU5cAuWA77+3hVxX\nrYKQEKej8QJxcTYxrVoFq1dDuXJOR6Qyyxjj9htwE3Ah1WMDgO/SOX4+0AZoBhzI4Lwmt33zzTcm\nICAgU8e+9957pk6dOkn3Bw0aZAoVKmTOnDmT9FhkZKS55pprzLx585Iei4qKMsHBweY///mPMcaY\nbdu2mXz58pkTJ06kOP/9999vBgwYYIwx5scffzQBAQHmwoUL2f5syvts2GBMaKgxa9c6HYmXiIkx\n5v777S0qyulo/I7rOznbucKphbqFgahUj0UBwakPFJH2QD5jzDwRaeaO4DJrxowZvP/+++zdu5fz\n588TFxdHgQIFUhxTsWJFihYtmnR/165dJCQk0KhRo6THihQpQo0aNZLu//HHHyQkJFC5cuUUY0yX\nL1+moFbx9FsHDkDbtvDBB7ZunsqEAgWga1fo3FmLDHohp35i54EiqR4rApxL/oCrK/Bt4IHEh652\n4qFDhyb9f/PmzWnevHkOwkzfsmXLeOKJJxg1ahR33303RYsW5euvv2bEiBEpjrv22mtT3E9MOJJB\nH3hCQgKBgYFs2rTpiudSn0/5h6goaNUKBg6E9u2djsaLiNipjsotli5dytKlS3PtfE4lqJ3ANSJS\n2Rizx/VYPWBbquOqAhWAFWK/0QOBoiJyBLjNGHPFjL7kCSovrVq1iipVqvDCCy8kPbZv376rvq5a\ntWoEBASwbt06HnroIQDOnj3Ljh07uPnmmwFo0KABsbGxnDx5kltvvTXN8wS6Rsbj4+Nz+lGUh7t0\nyW6ufddd8NxzTkejVPpSNwqGDRuWo/M5MovPGBMNzAGGi0iQiNyBHWP6PNWhW4By2DGrekBv7Ey+\nesBB90V8pWrVqrFv3z5mzZrF3r17GT9+PHPmzLnq60JCQnj88cfp378/y5YtY9u2bfTq1Yt8+fIl\ntapuvPFGOnToQNeuXfn222+JiIhgw4YNvPPOOyxw7bIcFhYGwPz58zl16pSuh/JR8fHQrZudDPHe\nezr5LEPGwJkzTkehcpGT08z7AkHACeBL4CljzF8icqeInAUwxiQYY04k3oBIIMEYc9IkH5xxQMeO\nHenXrx99+/alfv36/Pbbb5luvU2YMIFbbrmF1q1bc++993LHHXdQu3btFONL06dPp0uXLgwYMIAa\nNWrQtm1b1q5dS3nXLssVK1bk5ZdfZsCAAZQsWZKBAwfmxcdUDjLGtphOnoQvvrBbxql0XL4MffpA\n795OR6Jyke7F5wEuXrxI2bJlGTFiBP/5z3+cDkd5iJEjbdmMZcsg2TwbldqpU/Dww/YiffklBF8x\n10o5RPfi80Lr16/n66+/Zu/evfzxxx907dqVuLg4Onbs6HRoykNMnQrTpsEPP2hyytCWLXZK4+23\n210iNDn5FJ136QBjDG+//Ta7du0iMDCQ+vXrs3LlSkK1ToICvv0WXn8dli+HUqWcjsaD7d8PLVrA\n2LE6U89HaRefUh5k+XLo2NG2nBo2dDoaL7BvH1Ss6HQUKh057eLTBKWUh9i8Ge691w6j3HOP09Eo\nlXM6BuVHevToQRst+uOTduyABx6AiRM1OSmVSFtQXuTcuXMYYyhSxG7Ccdddd1GnTh3Gjx/vcGQq\nJ/btg2bNYMQI6N7d6Wg81OrVdgLEjTc6HYnKAm1B+YjY2NirHhMcHJyUnJRvOHLEtpheekmTU7o+\n/NBuQnj0qNORKDfTBJWO5cuX07hxY4KDgylWrBiNGzdm+/bthIeHExwczPfff0/16tUpVKgQLVq0\nSLHN0d69e2nXrh2lSpWicOHCNGzYMGkHiEQVK1Zk2LBh9OrVK2l3CYDhw4cTFhZGwYIFKVWqFE8+\n+WTSa5IvtrgsAAAe70lEQVR38fXo0YNly5YxceLEpEq/ERERVK1alTFjxqR4r127dhEQEJDm3n7K\nOSdP2uT0r39B375OR+OBLl60F2fcOFi5Elq2dDoi5WaaoNIQHx9Pu3btaNq0KVu2bGHdunU899xz\n5HMt5b906RLDhw8nPDycNWvWEB8fT4cOHZJef/78eVq1asWiRYv4888/6dixIw8//DA7d+5M8T5j\nx46lZs2a/P7774waNYo5c+YwevRopkyZwu7du1mwYAG3pLNt9bhx42jcuDE9evTg+PHjHD16lPLl\ny9OrVy+mTZuW4thp06ZRv359brrpply+Uiq7zpyxhV3bt4dBg5yOxgMdOmT7Pc+cgTVroFo1pyNS\nTshJrQ5Pu5FL9aAiIyNNQECAWb58+RXPffrppyYgIMCsXr066bH9+/ebfPnymUWLFqV7zttuu82M\nHDky6X5YWJhp06ZNimPGjBljatSoYeLi4tI8x5NPPmkeeuihpPvNmzc3/fr1S3HMsWPHTGBgoFnr\nKhgUHx9vypQpYyZNmpTBJ1budP68MbffbsyzzxqTkOB0NB5q+nRj3npLL5CXI4f1oLQFlYaQkBC6\nd+/Ovffey4MPPsjYsWM5dOhQ0vMBAQEp6jmVL1+e0qVLs337dgCio6N58cUXqV27NsWLFyc4OJjf\nf/+dAwdSbr6euHt5okceeYSYmBjCwsLo3bs333zzDZcvX85S7DfccAOtW7dOakX98MMPREZG0qVL\nlyydR+WN6Gh46CGoXt2uL9XNX9Px2GN2YE4vkF/TBJWOadOmsW7dOpo1a8a8efOoXr06v/zyS6Ze\nO2DAAGbPns3IkSNZvnw5mzdvplGjRlckm9S1ncqWLcvOnTv58MMPKVq0KAMHDqRhw4bExMRkKfbe\nvXszc+ZMLl68yCeffEKHDh1SFE1UzkhMTmXK2K2MAvS3T6kM6a9IBurUqcMLL7zAkiVLaNasGeHh\n4YAtKLh+/fqk4w4cOMCRI0eoVasWYGtFPfHEE7Rr144bb7yR0qVLs2fPnjTfI7XAwEAeeOABRo8e\nzbp169i2bRurVq1K99i06kHdf//9FClShMmTJzN//nx69eqV1Y+ucllMjJ2IVrIkfPqp7kyegpaK\nUenQBJWGiIgIBg8ezOrVqzlw4ABLlizhzz//TEpA+fLl4/nnn2fNmjVs2rSJ7t27U6dOHVq0aAHY\nWlFz585l48aNbNmyhW7dunHp0qWrvm94eDgff/wxW7duJSIigmnTphEYGEjVqlXTPD4sLIx169ax\nf/9+/vnnn6RqvQEBAfTo0YPBgwdTtmxZ7rrrrly6Mio7EpNTaCiEh2tySuHrr6FGDVsyWKlUNEGl\nISgoiJ07d/Loo49SvXp1evToQbdu3XjppZcAKFiwIC+//DJPPPEEjRs3RkSYPXt20uvHjBlDiRIl\naNq0Ka1bt6Zx48Y0adIkxXukVfK9WLFifPzxxzRt2pQ6deowd+5c5s6dS4UKFdKMc+DAgQQGBlKr\nVi1KlCjBwYP/q+HYs2dPLl++TM+ePXPjkqhsunjRztQrXhw++wyu0e2ZrUuXoF8/GDzY7o6rXdAq\nDbqTRBaFh4fTr18/zp49m6fvk1Nr166lSZMm7N27l7Jlyzodjl9KTE7BwTB9uianJPv3wyOP2MG4\nTz6BYsWcjkjlEd1JQqVw+fJlDh06xGuvvUaHDh00OTnk4kVbQ69wYbv5qyYnl8uXbYmMzp1hzhxN\nTipDmqB8zIwZMwgLCyMyMpLRo0c7HY5fio6GNm3g2mttyyl/fqcj8iCBgfD779C/v04hV1elXXxK\n5aJz5+DBB6FCBVsRV1tOyp9pF58HGzp0qNMhKDc6c8ZuF1ejhp1KrslJqZzxuQSVxY0XlMoVp07Z\noZXbboMpU3QRLhcvwjPPwOefOx2J8mI+18UXGWkICXE6EuVPjh2zu5K3aQMjR+rQCn/9BZ062abk\nhx/qRAg/pl18qVy44HQEyp8cPAhNm9pJaaNG+XlyMgY++giaNLFrnGbO1OSkckQTVB7SMSjftmeP\nrQjRpw+88orT0XiAV1+F8eNh+XJbx8mvs7XKDZqglMqGTZtsy+nFF2HAAKej8RB9+sDateDaEkyp\nnPK5MahlywxNmzodifJly5bZjRAmTYKOHZ2ORinPldMxKJ+bCKstKJWXvvvO9l7NmAF33+10NEr5\nNu3iy0M6BuVbpk2Dp56ChQv9ODklJNhxpr59nY5E+QFtQSl1FcbAu+/C5Mm2e69aNacjcsihQ9Cj\nB5w/r+ublFtoCyoPaQvK+yUkwAsv2FIZK1f6cXKaORMaNrTTFlesgCpVnI5I+QGfa0GdP+90BMpX\nXLoEvXrB3r125nTx4k5H5JDwcHjzTViwAG6+2elolB/RFlQe0haU9zp9Gu67z1bDXbTIj5MT2CmL\nf/yhyUm5nSYopVLZtw9uvx0aNLAVyQsVcjoihwUF2ZtSbqYJKg9pC8r7rF8Pd9wBTz8NY8ZAvnxO\nR+Rmp087HYFSSTRBKeUybx60amVn6/Xr53Q0bnbypN1QsFs3pyNRKonPJShPmiShLSjv8f77do3T\nggXQtq3T0bjZN99A3bpQtizMmuV0NEol8blZfGfOOB2B8iZxcTBwIPz4I6xaBRUrOh2RGx05As8+\nC1u3wuzZduBNKQ/ic3vx1a1r2LzZ6UiUNzhzxvZqxcXZyRB+N1Nv7lw7O+/ll6FgQaejUT4op3vx\n+VyCKl3acPiw05EoT7drly0weM89djJE/vxOR6SU7/HagoUiEiIic0XkvIjsE5HH0jluoIhsEZGz\nIrJHRAZmdN5Tp+zWNJ5Ax6A806JFcOed8NxzMGGCJielPJWTkyQmAReBUOBxYLKI1Ezn2G5AMeAB\n4BkReTS9k15zjc7kU+mbNAm6doWvvrKTIvzC4sW20q1SXsaRLj4RCQJOA7WMMXtcj30GHDLGDLnK\na8cBGGOeS+M5U66cYflyCAvL/biV94qNtS2mpUth/nyoXNnpiNzg5Em7keDixXbufOvWTkek/Iy3\ndvFVA+ISk5PLZqB2Jl7bBNiW3pPXXw///JPD6JRPOXHCblu0fz+sXu0HySk+HiZOhNq17cyPbds0\nOSmv5NQ088JAVKrHooDgjF4kIsMAAT5J75gzZ4YydqzdbLl58+Y0b948p7Fm29ChQ3UcymHr1tmq\nt926wfDhfrIzRP/+sHmzbTndeKPT0Sg/snTpUpYuXZpr53MqQZ0HiqR6rAhwLr0XiMgz2LGqO40x\nsekdd+utQ2nVCrp0yZU4lRebOtXOoJ461c8W344YAcHBINnuWVEqW1I3CoYNG5aj8zmVoHYC14hI\n5WTdfPVIp+tORHoCLwJNjDFHMzpxaCgcP56rsWabtp6ccfGi3arot99s6aLq1Z2OyM2KpP7bTynv\n5MgYlDEmGpgDDBeRIBG5A2gDXFGmU0S6AiOBlsaY/Vc7d9my6DooP3bwIDRtahfhrlnj48lp1So7\nsKaUj3JymnlfIAg4AXwJPGWM+UtE7hSRs8mOGwEUB9aLyDnXeqhJ6Z20bFlbmdoTaAvKvRYvhltu\ngUcftTtDBGc4ounFIiKgUye7DYYmKOXDHNuLzxhzGmifxuMrSTY+ZYyplJXzelKCUu4RHw8jR9qZ\n1F9+CS1aOB1RHjl71la2/fBDO2f+k0+0TpPyaT63WawnJShtQeW9Y8fswtuEBPj9dyhd2umI8sil\nS1CvHjRrBn/+CWXKOB2RUnnO5/bii4kxFCliB8oDfK6YiEpu0SJ44gno3Rtee80PppAfOwYlSzod\nhVKZ5q0LdfNMwYJQrJhnzOTTFlTeiI+H11+3a5vCw2HYMD9ITqDJSfkdn0tQYGv67N3rdBQqLxw9\nancgX7nSVoq45x6nI8plBw7Y7dWVUr6ZoKpWhd27nY5CW1C5bcECaNAAmjeHn3/2sQbFP//Yyon1\n60NkpG0mKuXnfG6SBNgEtWuX01Go3BIdbfc8XbAAZs6065x8RnQ0jBtnW00dO9rqtqVKOR2VUh7B\nZ1tQnpCgtAWVcxs3QsOGduHtpk0+lpzA1v/YuNEuup08WZOTUsloC0p5pIQE+L//g3ffhffes1PJ\nfdKAAbpnnlLp8Llp5sYYzp+HG26AqChbwFB5l4MH7fTx+Hj47DMfqe0VF2enGmoyUn5Ep5mnoXBh\n21PiCRMlVOYZA198Ybv0WraEJUt8IDnFx9sPVbMmLF/udDRKeRWfTFAAdevaBfdO0jGozDt6FNq1\ng3fegR9/hCFDvHxtU1ycTUy1a8MHH9jtiZo1czoqpbyKTyeoTZucjkJdTWKrqV49+zPbsMFOJfdq\nO3bYgdCPPoIJE2zL6a67nI5KKa/jk2NQAAsX2kH2xYsdDkql6+hReOopu6j6009t155PuHzZZtrb\nb3c6EqUcpWNQ6bj9dli/HmLTrb2rnGKM3XX8ppugTh37Xe4zyQkgMFCTk1K5wGcTVLFiUKmSXWLi\nFB2DutL+/fDQQ/DWW3bh7RtvQIECTkeVDRER8N//2mmGSqk84bMJCuDOO23Jb+W8uDgYPdq2lBo3\ntqUxbr7Z6aiyYe1aWxHx5pttS8lni08p5TyfHYMCmDHDVladO9fBoBQbNsC//w0hITBlip0/4HVO\nnrTTDI8eheefhx49fLhkr1K5I6djUD6doA4dsntvHj+utaGccO4cvPoqfPWVnT7erZsXr1M1xs68\nuf9+L5//rpT76CSJDJQtC6GhdrKEE/x1DMoYu6lr7dp2D72tW+3OEF6TnBISrnxMBFq31uSklBv5\ndIICaN9eu/jcaetWOywzapRd3/Tpp3D99U5HlQmxsfDNN3a90v/9n9PRKKXw8S4+sOMfXbvatZNe\n8xe8F4qKgqFD7fTx116z65u8Yh/Eo0ftLg8ffgiVK0PfvvavmsBApyNTyutpF99VNGwIMTHw119O\nR+KbEhJs2fUaNeD8edi2DZ55xkuS04EDUKuWTVI//mh3fOjUSZOTUh7C5xOUCHToYAfq3c3Xx6CW\nLoVbbrEljb77DqZOtWN+XqN8eTuTZsoUu2JYKeVRfD5BAfTuDR9/rLtK5JYdO6BtWzvTeuBAWL3a\nJiqPFBNjB8PSKxB27bXujUcplWl+kaBuvNHuKvH99+59X19rQZ04YYdomjSxt7/+gs6dPXAKf0KC\nrdXRqxeUKWMHxs6fdzoqpVQWedpXS5556inbFaWy7vx5GDnSDtfkz29bUAMHQsGCTkeWhkWLbNdd\n//424C1b4Icf7II4pZRX8ZsE1bEj/P03rFvnvvf09hZUTIzdnqhyZTv5Yc0aW379uuucjiwDN95o\nJzxs3GjLqZcp43RESqls8psEVaAADB5sp0KrjF26BBMnQpUqsGoV/PorTJ9u73uEvXvh/fftiuDU\nbrjBJimllNfz+XVQyV26BNWq2S/bO+5wY2BeIjbWThkfMcJ+xw8f7kFlMHbtsgtpv/kGDh60a5VG\nj4bChZ2OTCmVDt2LL5mrJSiwG8i+/bZdwOsVa3XcIDraznJ8912bwIcP97ByRk88Ab/8YtcLPPKI\n3aZef3hKeTxNUMlkJkEZAy1bQqtWdhw9Lw0dOtSjx6GiouzEkXHjbAmMwYM9dLr40aNQooTug6eU\nl9GdJLJIBCZPhjfftBO8/NHhwzYZVa4M27fbiW9z5zqUnPbvtz+Q1q1h0KC0jylVSpOTUn7I71pQ\nicLDbVffunX+M4yxdq2dhffTT3Z/wv79oWJFBwKJiLD1N375Bc6ehXvvtWV277sPihZ1ICClVF7Q\nLr5kspKgjIF//cvWivr2W9/9Az02FmbPtonp+HF49lno2dPhPHDkiB0MbNnSzsbwuJW+SqncoAkq\nmawkKLBf3q1aQVgYfPBB7n9POjkGtXu3nfjw6adQvbotAvvQQ25IxFFR8NtvsGKFreu+cKHvZn+l\nVIZ0DCoH8ueHOXPsAt5evbx/r76LF+0U+hYt7Cy82FhYvNhu6tquXR7niUGD7G4NZcrYvtN8+exC\nWR/6A0gp5V5+3YJKdOECPPqonW799dfetSN3QoJtrMyYYZcINWxoN8dt2zYPqkZERtr9jYKCrnxu\n+nTbFG3Y0K6KVkr5Pe3iSya7CQogPh5efdV+z37yiS2s6qmMseu4ZsywpdWvvx4ee8zeKlTIpTc5\netRuF/Tnn7B5M6xfb3eLnTcPmjfPpTdRSvkyr+3iE5EQEZkrIudFZJ+IPJbBsW+LyCkROSkib+dF\nPPny2TLl778P3bvbiQQnTuTsnLk5/nTxot1irm9fm4S6dLGVIn75xeaPQYOymZzi49N+fPx4O7Pi\n1Cl44AGYPx/OnNHkpJRyGyeX408CLgKhQANggYhsMsakqH0rIn2ANkBiRblfRWSPMebDvAjqwQeh\nWTNbtrxmTTs29d//2qU47mSMHRtbutROC1+82NbUe+ghm6hq1sxaCfulX39N84AA2Lkz5W3AALso\nKrU338y1z+INli5dSnNNvmnSa5MxvT55x5EWlIgEAR2AV4wxMcaYVcA8oFsahz8BjDbGHDXGHAVG\nA0+me/Lrr7fjIB062Klr2SilGxwMY8falkl0tK3a0K6dna597lzmz5OVFlRMjN0tfOJE21VXurRd\nFrRmjf0oe/bAypXw0ks2nqTkFBcHx47ZPr85c+zurmlY+tVXti5SVJQt5vT223aL8vQWx/qZpUuX\nOh2Cx9JrkzG9PnnHqRZUNSDOGLMn2WObgaZpHFvb9Vzy42qne+bt2+3uBIm36Oi0j1u+HIYNs7tf\nlyxpt9IJCbFNk6Y2jLJlbZffm2/ayRMffghPPgmNGtldFxo1grp1oVy5zNVGMsbmh/377d6nO3fa\nVtLGjbB7l+GmatHcXjuKjrWjGP1IFKWvjbK1LW6++cqTzZoFTz9tu91CQmywFSrYbJbWTrh16+pW\n7kopr+JUgioMRKV6LAoIzsSxUa7H0laihL01apRxBLVr29bD8eP/u+3da7c8b5oyTwYHQ68is+i1\nuhcJBQtyaWtBojcX4NykgvwQ8CDPx7xJ8eL2bQsXtrd6kUtos/VVChQoRHwCxF+ORy5fYlX+uwiv\n+gZVq0LVqnbf0379oN62GeT/T284URT+KGpX0hYtasd/0kpQ990HW7faFqOuM1JK+SBHZvGJyE3A\nSmNM4WSP9QeaGWPapjr2DHCPMWaD634DYIkx5oq9EETEd6YkKqWUD8jJLD6nWlA7gWtEpHKybr56\nwLY0jt3mem6D6/5N6RyXowuhlFLKszgyScIYEw3MAYaLSJCI3IGdqfd5God/BvQXkdIiUhroD3zi\nvmiVUko5wcmtjvoCQcAJ4EvgKWPMXyJyp4icTTzIGPMBMB/YAvwJzDfGTHUiYKWUUu7jUztJKKWU\n8h1+vVmsUkopz+VVCcrTtkfyNJm9PiIyUES2iMhZEdkjIgPdHau7ZeXfjuv4/CKyQ0QOuCtGJ2Xx\nd6uBiCwTkXMiclRE+rkzVnfLwu9VoIhMEZFjru+e70TEzXvQuJeI9BWR9SJyUUSmXeXY/7r+vZwW\nkY9EJP/Vzu9VCYqU2yM9DkwWkZqpD0q1PVJd4EER+bc7A3VIpq6PSzegGPAA8IyIPOqeEB2TlWsD\n8CJwzB2BeYjM/m5dB/wATAZCgCrAz26M0wmZ/bfzPHArcCNQGrtmc4K7gnTIYWAE8HFGB4nIfdjf\nqbuAMKAyMOyqZzfGeMUNO6HiElA52WOfAaPSOHYV0DvZ/Z7Ab05/Bk+5Pmm8dhwwzunP4CnXBqiI\nXcpwH3DA6fg96foAI4Fwp2P20GszCXgr2f1WwF9OfwY3XacRwLQMnv8SeCPZ/RbA0aud15taUOlt\nj5TWtkdZ2x7JN2Tl+qTWhHTWlvmIrF6b8cBg7F/N/iAr1+c24LSIrBKR465urHJuidIZWbk2HwN3\nikgp136jXYGFbojRG6T1nVxCREIyepE3Jai82x7JN2Tl+iQRkWGA4NtryzJ9bUSkPZDPGDPPHYF5\niKz82ymL3cC5H1AOiABm5GVwDsvKtdkJHMB2e50BamBbFirt72ThKt9P3pSgzgNFUj1WBEhrf/HU\nxxZxPebLsnJ9ABCRZ7B96q2MMV5e8D5Dmbo2rr9638Z++YL9BfIHWfm3EwPMNcb8YYy5jB1HuF1E\nMvyi8WJZuTZTgALYsblrgbnAj3kanfdI6zvZkMH3E3hXgkraHinZY1fbHilRutsj+ZCsXB9EpCd2\n0LKFsWVMfFlmr01VoAKwQkSOArOB0iJyRETKuydUR2Tl386f2C+W5Ay+m8yzcm3qAp8aY6Jcf/BN\nAG4RkeJuiNPTpfWdfNwYczrDVzk9uJbFgbjp2MG2IOAO4DRQM43j+rguSGnXbSvwL6fj96Dr0xU4\nClR3OmZPujbYP9hKJLu1Bw5hZ2+J05/B6evjOu4u4B/sl3F+YCywzOn4PeTaTANmYVsH+YEhwEGn\n48/ja5MPKAiMwk4eKYDtIk993H3AEaAmtoW5CBh51fM7/QGzeDFCsM3m89i+706ux+8EzqY69i3X\nL9Ip4E2nY/ek6wPsxc5MOottYp8FJjkdvydcm1SvaYYfzOLL6vXB/gF4yPX79R1Qxun4PeHaAMWB\nL4DjQCSwHLjZ6fjz+Nq8DiQA8clur2HHJ88BZZMd+zx26cYZ4CMg/9XOr1sdKaWU8kjeNAallFLK\nj2iCUkop5ZE0QSmllPJImqCUUkp5JE1QSimlPJImKKWUUh5JE5RSSimPpAlKKTcQkWYikuCN296I\nyBIRGe90HMr/aIJSPkVEbhKROBFZkY3Xvi4iW/IiLhdvXRXfHlt+BABXVdn+Dsaj/IQmKOVr/gVM\nBG4UkerZeL23JpFsyUzZbWPMGWPMBXfEo1RymqCUzxCRgkAXYCrwDdA7jWNKiciXInJKRC6IyB+u\n7rfu2H3Faru64uJF5AnXaxJEpEOq86RoRYjIf0Vks4icF5FDIjJVRIpmMf4OrnNEi8g/rq61UNdz\nr4vIFhHpJSL7XcfMdZVgT3z9zSLyk4icFJEoEVkhIreleo8EEXlaRGaLyHlgpIhcIyLjReSwiFx0\nnX9UstckdfGJyBLsju/vJrtOQa73S32NWorI5cTPoFRWaYJSvuQRIMIYsxW7aecTIpIv8UlXvafl\nQHmgLXAjMNz19FfAaOBv4AagFDAzC+8dDzwH1AIeAxphK/NmiojcgC389wm20F0T4PNUh4Vhd6J/\nCLgbWx7k42TPB2N3lL7D9f4bgQVpjHu9BizAfv6JwLPY6/EoUAXohL0OaemA3Sh2GFASKGWMiXbF\n3jPVsT2AecaYkxl+eKXScY3TASiVi3phv6AxxiwTkQtAG+xO1GC/3EsAt5j/1aHZl/hiV4siLjtf\nqMaY5MnogIi8BHwLdM/kKUpjfx9nG2MOuh7bnuqYgkA3Y8xhV7x9sLWrKhtj9hhjliQ/WESeAzoC\n92NLRiT6yhgzLdlxFYCdxphVrocOAWvS+ZynRSQeOG+MOZHsqanAahEpZYw5KiLFgHbAw5n8/Epd\nQVtQyieISBVsyyF5+fHp2DGpRDcBf5qrFUnL3vu3EJGfReSgiJwF5gCBIlIyk6fYjK2Rs01EvhGR\np0Tk+lTHHE5MTi5rsaUOarpiCBWRD0TkbxE5gy2jEoptMSb3e6r7nwL1RWSniLwvIq1EJEsFCI0x\nv2PrriUm5K7YkhNaUVZlmyYo5St6Y/89HxSRWBGJBV4CWopIGdcx2a36mlbF2KTJBa5qu99ji2R2\nBBrwv+6uwEy9gTEJxph7gZbYZNUL2CUidbIQ52dAQ2xXY2NsBdPDacSQYsKDMWYjdlxpMPZzhgM/\nZ+F9E32E7dbD9d9PjNbzUTmgCUp5Pdc40xPAIOyXcvLbn/zvS/MPoG4Ga5EuYyuEpnYSOyaV+H43\nJL8P3IxNWP2NMWuNMbuBMmSD6/UjjDGNsBVIOyV7ukyyZAtwKzahJHYF3gFMMMb8aIz5C5uIkseZ\n0fteMMbMNsb0BVoDd7tapWlJ7zp94YqxL1Af2zJTKts0QSlf8CBwHfCRMWZ78ht2okMv13HTgRPA\ntyJyp4iEichDItLM9XwEUEFE6ovIdSKS2PJYDPQVkYYiUh87kSEm2fvvwv4u/dd1zsewrZjU0m3B\nicitIvKyayZeORFpC5TFtsoSXQTCRaSeiDQGJgPfG2P2up7fCTwuIjVFpBG2u/PSVa5d4gzEziJS\nw5WUugJR2LGotEQATUSkdPJZhMaYs9jZk6OxZeD3XO29lcqIJijlC3oCi9MZW5oFlBeRe1yzzZph\nu73mYcdMhvK/tU+zgYXYsaATQGfX4wOAvcAS4GvshICkCQLGmC3YhPRfbELp6XpNahl1d0VhW0Dz\nsYnmXWC4MSb5mNo+7GzD+cCvwG5SzpzrARQGNmCT8cfYZHK1GM4BL2DHtDYAdYEHjDEX03lNYknv\nPSS7Di4fY7sUP0apHNKS70p5ARF5HXjYGFPX6VgyIiKdsC270skSnFLZotPMlVI5JiKFsONdg4EP\nNTmp3KBdfEqp3PAisAM4BbzhcCzKR2gXn1JKKY+kLSillFIeSROUUkopj6QJSimllEfSBKWUUsoj\naYJSSinlkf4fSRC3hLflMqQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71401a320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0, 1, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"Actual sparsity\")\n",
    "plt.ylabel(\"Cost\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def kl_divergence(p, q):\n",
    "    \"\"\"Kullback Leibler divergence\"\"\"\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 1000  # sparse codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "#activation = tf.nn.softplus # soft variant of ReLU\n",
    "activation = tf.nn.sigmoid\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_outputs), name=\"biases2\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "outputs = tf.matmul(hidden1, weights2) + biases2\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # batch mean\n",
    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
    "loss = mse + sparsity_weight * sparsity_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0817271 \tSparsity loss: 0.384313 \tTotal loss: 0.15859\n",
      "1 Train MSE: 0.0547101 \tSparsity loss: 0.133792 \tTotal loss: 0.0814684\n",
      "2 Train MSE: 0.0491168 \tSparsity loss: 0.102076 \tTotal loss: 0.0695321\n",
      "3 Train MSE: 0.0427555 \tSparsity loss: 0.0580622 \tTotal loss: 0.0543679\n",
      "4 Train MSE: 0.0388306 \tSparsity loss: 0.258078 \tTotal loss: 0.0904461\n",
      "5 Train MSE: 0.0350311 \tSparsity loss: 0.0247787 \tTotal loss: 0.0399869\n",
      "6 Train MSE: 0.0312519 \tSparsity loss: 0.0519023 \tTotal loss: 0.0416324\n",
      "7 Train MSE: 0.0282995 \tSparsity loss: 0.0559849 \tTotal loss: 0.0394964\n",
      "8 Train MSE: 0.0268548 \tSparsity loss: 0.0824516 \tTotal loss: 0.0433452\n",
      "9 Train MSE: 0.0252737 \tSparsity loss: 0.101543 \tTotal loss: 0.0455822\n",
      "10 Train MSE: 0.022887 \tSparsity loss: 0.14663 \tTotal loss: 0.052213\n",
      "11 Train MSE: 0.0230027 \tSparsity loss: 0.177287 \tTotal loss: 0.0584601\n",
      "12 Train MSE: 0.0208771 \tSparsity loss: 0.0856517 \tTotal loss: 0.0380074\n",
      "13 Train MSE: 0.0193335 \tSparsity loss: 0.0664709 \tTotal loss: 0.0326276\n",
      "14 Train MSE: 0.0208669 \tSparsity loss: 0.0702151 \tTotal loss: 0.0349099\n",
      "15 Train MSE: 0.0174612 \tSparsity loss: 0.105143 \tTotal loss: 0.0384897\n",
      "16 Train MSE: 0.0174152 \tSparsity loss: 0.111816 \tTotal loss: 0.0397784\n",
      "17 Train MSE: 0.016073 \tSparsity loss: 0.0863786 \tTotal loss: 0.0333487\n",
      "18 Train MSE: 0.0154538 \tSparsity loss: 0.0626113 \tTotal loss: 0.027976\n",
      "19 Train MSE: 0.0192317 \tSparsity loss: 0.376623 \tTotal loss: 0.0945563\n",
      "20 Train MSE: 0.0151243 \tSparsity loss: 0.0612712 \tTotal loss: 0.0273786\n",
      "21 Train MSE: 0.014892 \tSparsity loss: 0.0431137 \tTotal loss: 0.0235148\n",
      "22 Train MSE: 0.0149928 \tSparsity loss: 0.241904 \tTotal loss: 0.0633736\n",
      "23 Train MSE: 0.014827 \tSparsity loss: 0.146745 \tTotal loss: 0.044176\n",
      "24 Train MSE: 0.0172172 \tSparsity loss: 0.198305 \tTotal loss: 0.0568781\n",
      "25 Train MSE: 0.014371 \tSparsity loss: 0.118335 \tTotal loss: 0.038038\n",
      "26 Train MSE: 0.0144909 \tSparsity loss: 0.0903507 \tTotal loss: 0.0325611\n",
      "27 Train MSE: 0.014833 \tSparsity loss: 0.10467 \tTotal loss: 0.035767\n",
      "28 Train MSE: 0.0160102 \tSparsity loss: 0.0471489 \tTotal loss: 0.02544\n",
      "29 Train MSE: 0.0148225 \tSparsity loss: 0.364606 \tTotal loss: 0.0877437\n",
      "30 Train MSE: 0.015598 \tSparsity loss: 0.0576926 \tTotal loss: 0.0271365\n",
      "31 Train MSE: 0.0135493 \tSparsity loss: 0.0724452 \tTotal loss: 0.0280383\n",
      "32 Train MSE: 0.013253 \tSparsity loss: 0.306267 \tTotal loss: 0.0745063\n",
      "33 Train MSE: 0.0135294 \tSparsity loss: 0.0979742 \tTotal loss: 0.0331243\n",
      "34 Train MSE: 0.0115316 \tSparsity loss: 0.0975675 \tTotal loss: 0.0310451\n",
      "35 Train MSE: 0.0132746 \tSparsity loss: 0.130185 \tTotal loss: 0.0393116\n",
      "36 Train MSE: 0.0133852 \tSparsity loss: 0.0606884 \tTotal loss: 0.0255229\n",
      "37 Train MSE: 0.0131865 \tSparsity loss: 0.0839338 \tTotal loss: 0.0299732\n",
      "38 Train MSE: 0.0132433 \tSparsity loss: 0.186769 \tTotal loss: 0.0505971\n",
      "39 Train MSE: 0.0125048 \tSparsity loss: 0.168512 \tTotal loss: 0.0462072\n",
      "40 Train MSE: 0.0132689 \tSparsity loss: 0.175295 \tTotal loss: 0.0483278\n",
      "41 Train MSE: 0.0134131 \tSparsity loss: 0.147331 \tTotal loss: 0.0428794\n",
      "42 Train MSE: 0.0132286 \tSparsity loss: 0.107903 \tTotal loss: 0.0348093\n",
      "43 Train MSE: 0.0126851 \tSparsity loss: 0.129837 \tTotal loss: 0.0386526\n",
      "44 Train MSE: 0.0140411 \tSparsity loss: 0.38007 \tTotal loss: 0.0900552\n",
      "45 Train MSE: 0.0119832 \tSparsity loss: 0.208153 \tTotal loss: 0.0536138\n",
      "46 Train MSE: 0.0120448 \tSparsity loss: 0.0933219 \tTotal loss: 0.0307092\n",
      "47 Train MSE: 0.011037 \tSparsity loss: 0.130909 \tTotal loss: 0.0372187\n",
      "48 Train MSE: 0.0117794 \tSparsity loss: 0.123891 \tTotal loss: 0.0365576\n",
      "49 Train MSE: 0.0115448 \tSparsity loss: 0.300135 \tTotal loss: 0.0715718\n",
      "50 Train MSE: 0.0119193 \tSparsity loss: 0.146806 \tTotal loss: 0.0412805\n",
      "51 Train MSE: 0.0112673 \tSparsity loss: 0.136534 \tTotal loss: 0.0385742\n",
      "52 Train MSE: 0.0129241 \tSparsity loss: 0.215849 \tTotal loss: 0.056094\n",
      "53 Train MSE: 0.010884 \tSparsity loss: 0.137653 \tTotal loss: 0.0384146\n",
      "54 Train MSE: 0.0125442 \tSparsity loss: 0.371092 \tTotal loss: 0.0867626\n",
      "55 Train MSE: 0.0115945 \tSparsity loss: 0.375916 \tTotal loss: 0.0867776\n",
      "56 Train MSE: 0.0112874 \tSparsity loss: 0.236811 \tTotal loss: 0.0586495\n",
      "57 Train MSE: 0.0110727 \tSparsity loss: 0.103596 \tTotal loss: 0.031792\n",
      "58 Train MSE: 0.0108197 \tSparsity loss: 0.0945379 \tTotal loss: 0.0297273\n",
      "59 Train MSE: 0.0114168 \tSparsity loss: 0.283274 \tTotal loss: 0.0680716\n",
      "60 Train MSE: 0.0109551 \tSparsity loss: 0.212905 \tTotal loss: 0.0535362\n",
      "61 Train MSE: 0.0114536 \tSparsity loss: 0.135698 \tTotal loss: 0.0385931\n",
      "62 Train MSE: 0.011533 \tSparsity loss: 0.129325 \tTotal loss: 0.0373979\n",
      "63 Train MSE: 0.0111304 \tSparsity loss: 0.28615 \tTotal loss: 0.0683605\n",
      "64 Train MSE: 0.0114537 \tSparsity loss: 0.126432 \tTotal loss: 0.0367401\n",
      "65 Train MSE: 0.0169832 \tSparsity loss: 0.985183 \tTotal loss: 0.21402\n",
      "66 Train MSE: 0.0223841 \tSparsity loss: 0.415693 \tTotal loss: 0.105523\n",
      "67 Train MSE: 0.0140507 \tSparsity loss: 0.156901 \tTotal loss: 0.045431\n",
      "68 Train MSE: 0.0198567 \tSparsity loss: 0.550292 \tTotal loss: 0.129915\n",
      "69 Train MSE: 0.0112844 \tSparsity loss: 0.880939 \tTotal loss: 0.187472\n",
      "70 Train MSE: 0.0205504 \tSparsity loss: 0.282671 \tTotal loss: 0.0770847\n",
      "71 Train MSE: 0.0143246 \tSparsity loss: 0.606177 \tTotal loss: 0.13556\n",
      "72 Train MSE: 0.0163604 \tSparsity loss: 0.373482 \tTotal loss: 0.0910568\n",
      "73 Train MSE: 0.0178618 \tSparsity loss: 0.370052 \tTotal loss: 0.0918722\n",
      "74 Train MSE: 0.0214178 \tSparsity loss: 0.444671 \tTotal loss: 0.110352\n",
      "75 Train MSE: 0.0181602 \tSparsity loss: 0.487524 \tTotal loss: 0.115665\n",
      "76 Train MSE: 0.0170524 \tSparsity loss: 0.230372 \tTotal loss: 0.0631268\n",
      "77 Train MSE: 0.0155673 \tSparsity loss: 0.305983 \tTotal loss: 0.0767638\n",
      "78 Train MSE: 0.0299082 \tSparsity loss: 0.13183 \tTotal loss: 0.0562743\n",
      "79 Train MSE: 0.0168766 \tSparsity loss: 0.284595 \tTotal loss: 0.0737956\n",
      "80 Train MSE: 0.0261243 \tSparsity loss: 0.47939 \tTotal loss: 0.122002\n",
      "81 Train MSE: 0.0184599 \tSparsity loss: 0.311156 \tTotal loss: 0.080691\n",
      "82 Train MSE: 0.0206999 \tSparsity loss: 0.34335 \tTotal loss: 0.0893699\n",
      "83 Train MSE: 0.0157594 \tSparsity loss: 0.136414 \tTotal loss: 0.0430423\n",
      "84 Train MSE: 0.0171685 \tSparsity loss: 1.04854 \tTotal loss: 0.226876\n",
      "85 Train MSE: 0.0266792 \tSparsity loss: 0.565319 \tTotal loss: 0.139743\n",
      "86 Train MSE: 0.0124258 \tSparsity loss: 0.0797283 \tTotal loss: 0.0283714\n",
      "87 Train MSE: 0.0151309 \tSparsity loss: 1.56184 \tTotal loss: 0.327499\n",
      "88 Train MSE: 0.0174013 \tSparsity loss: 1.14313 \tTotal loss: 0.246027\n",
      "89 Train MSE: 0.0328306 \tSparsity loss: 1.28411 \tTotal loss: 0.289652\n",
      "90 Train MSE: 0.0230594 \tSparsity loss: 0.148135 \tTotal loss: 0.0526864\n",
      "91 Train MSE: 0.0133196 \tSparsity loss: 0.229191 \tTotal loss: 0.0591579\n",
      "92 Train MSE: 0.0177904 \tSparsity loss: 0.579763 \tTotal loss: 0.133743\n",
      "93 Train MSE: 0.0142399 \tSparsity loss: 0.371078 \tTotal loss: 0.0884555\n",
      "94 Train MSE: 0.0131689 \tSparsity loss: 0.136192 \tTotal loss: 0.0404072\n",
      "95 Train MSE: 0.0146155 \tSparsity loss: 0.176726 \tTotal loss: 0.0499607\n",
      "96 Train MSE: 0.0232258 \tSparsity loss: 0.219851 \tTotal loss: 0.067196\n",
      "97 Train MSE: 0.0370769 \tSparsity loss: 0.513892 \tTotal loss: 0.139855\n",
      "98 Train MSE: 0.0186342 \tSparsity loss: 0.355253 \tTotal loss: 0.0896847\n",
      "99 Train MSE: 0.0159218 \tSparsity loss: 1.31571 \tTotal loss: 0.279064\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_val, sparsity_loss_val, loss_val = sess.run([mse, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHQtJREFUeJzt3VuMnWXZxvGnpdPpbDszDZ3uWzqlpa1UCza0ign2gBjZ\nRbEmyIEGTSQmmkiiBhITNTF6pj0xYDSgaAzRpBJiRMsuLQREKlMVrA4zbZRWSne0M9PZFfDoO/i+\n57r83odZa5V75v87vHOvd72z1kzvrqwr9zPn7bffTgAARDD3Yt8AAABVMbQAAGEwtAAAYTC0AABh\nMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYcxr4HOxegONMOdi38BsMDg4mP09z5mjX3q1dcf1Vn38\nf6src+fm/z93j1f39tZbb1XuLXm8uq+S53Km+/rOm6dHw5tvvjmt65b8jvT19clmPmkBAMJgaAEA\nwmBoAQDCaOR3WgBmsFp896S+z1Hf+6RU9r3NdLl7UPer7ss9Xn1HdMkll8je6Z7IUfK92oULF2Tv\ndL9XK/ke0eGTFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM0oMA6kql4VySTfWWpOZcr0vvVTU5OSnr\nVTdiuESguq+WlhbZe+bMmUrPlZLeaOG2XFRNQNZT0XaTOt4HAAA1xdACAITB0AIAhMHQAgCEQRAD\nQDEVFhgfH5e9JV/0l6xxUkpWSbkwiFph5K6rwg0l96vuYWRkRPaWhCOmpqYq95YEZUpeR3XdWgQ8\n+KQFAAiDoQUACIOhBQAIg6EFAAiDoQUACIP0IIBiY2NjlXsbeXBgyWGLbjWTer7m5mbZ29ramtWa\nmpqy2sTEhHy8Sgqqx6ek03juZ3AJREX9DKqW0vRXapEeBADMKgwtAEAYDC0AQBgMLQBAGAQxABRT\nq3tKzoxSq5Jcr6Oez92DMn/+fFlfsGBBVuvs7JS9KligwhHu51X361Ywqedqb2+XvSrMocIoKemV\nXK5X3Zs7p0vV3XVdXeGTFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM0oMAiqmUn1vx45JzVbm1RirJ\n5g6iLEklqtSbS/Sp66qUX8kaKJeAdElBRa3Zcqu3VNrRrYdS72VJYtP9jpS8P3zSAgCEwdACAITB\n0AIAhMHQAgCEMauCGM8991xW2717t+xdvnx5VlPrTlJK6dOf/nRW6+npkb2uDkSiwgZuFY8KNrhw\nhvqi/sSJE7L3L3/5S1b729/+JnvVyia1iiolHZp4//vfL3sXL16c1UrWS1166aWV7+vUqVNZzQUY\nzp8/n9Xca97W1pbV3Gom9dqUrHxy52m5n1nhkxYAIAyGFgAgDIYWACAMhhYAIAyGFgAgjDlurUYd\nNOyJnA0bNmS1gYGBujzXwoULZX379u11eb56WbNmTVa7++67Ze+qVavqfDeV6HgSampwcHBaf88u\ncabWMP3pT3+Svb/5zW+y2osvvih7Fy1alNVUws7dm1q3lJJOD6rUnHu8Sg+eOXNG9h47diyrqZRz\nSjpV2NXVJXtvvfXWrLZp0ybZq9J/7nWcmJjIau7gTZVW7Ovrk3/LfNICAITB0AIAhMHQAgCEwdAC\nAIQxq9Y4/frXv85q/f39snfz5s1Z7aWXXpK9f/jDH7Laww8/LHt/97vfZbXLLrtM9h4+fFjWq3Kr\nWJYuXZrV/vWvf1W+rgpnpJTS1772tcrXQGxuHU/VXre2p2SdT29vb1b78Ic/LHvV76wLg6gwh/v7\nGBkZyWrDw8NZzYUV1L8p7vywBQsWZLXR0VHZ+/zzz2c1FYxIKaWNGzdmtfXr18tetcbJrahSoQvX\nW/L7xCctAEAYDC0AQBgMLQBAGAwtAEAYDC0AQBizKj2oUjKq5mzZskXWb7vttqz23e9+V/YeOXIk\nq7n04NDQUOV7U9zKFJUedPegDuC74oorpnVfiE+l/FzqTaX03OGFKl2m1q+llNLKlSuzmlrXlFJK\nnZ2dWU2l/FJK6X3ve19Wc39L6n5LDshU6b/u7m7Zq1ZBuRVXKhXt0sQdHR1ZzaX5VN29NopLh5as\nE+STFgAgDIYWACAMhhYAIAyGFgAgjFkVxGgktXIlpbIQQ0lIpIRaO3Xy5EnZe80112S166+/vub3\nhFhUkMKFDVTdfSGvwhxqdVBKOnTR3t4ue9Xfo/sbVWdkud4LFy5ktZKfVwUb2traZK9aBfXLX/5S\n9qq/5xtuuEH2Xn311VnNhTbUz1YS2nBrnAhiAABmJIYWACAMhhYAIAyGFgAgDIYWACAM0oMzmDsg\n7mMf+1hWc+mm73//+1mtpaVlejeGGakkAeZ6Jycns5pLp6mEm1sPNTU1VblXpffcAYpqhZE6GLLk\n8SqRmJI+MPKxxx6TvWpt1Yc+9CHZu2TJkqym3oeU9L8T7nVU75tb9eWeT+GTFgAgDIYWACAMhhYA\nIAyGFgAgDIIYM9gDDzwg66+99lpWc+cQrV69upa3hBlCBSlcYMJ9Ua+oYIIKUaSkVwK5gIcKR5QE\nCNxaI/V8Kqzg1hepe1Bn7qWU0n333ZfV3Jl7H/nIR7La1q1bZa/6ed3rWPJeqmu4QEoJPmkBAMJg\naAEAwmBoAQDCYGgBAMJgaAEAwiA9OEMMDg5mtbvuuqvy45999llZVyteAJUiK1njVJI0dAcwqpSe\nu4eSJJu6N3e4pDoUUaUd3X2p5+rv75e9Tz/9dFZzqcbrrrsuq7m/ZbU2yq11q/r4lPR76dKHHAIJ\nAJiRGFoAgDAYWgCAMBhaAIAwCGLMEI888khWc+tvdu3aldXWrl1b83vCzKW+qHfhCvUle3Nzc+Xr\nut/jkgCBej4XrhgfH89qag2Uez51jl13d7d8vAomHDt2TPZ2dHRktZ07d8re97znPVlNnROWkn5/\n3NqpsbGxrObe93rhkxYAIAyGFgAgDIYWACAMhhYAIAyCGMG4L6X37NmT1dyX3d/5zneymvviFZgu\n9UV9yRlZJWc4ue0ZanPE5OSk7D1x4kRWc9sz1PP19PRkNfe3ODAwkNUOHTokey+99NKsdtNNN8ne\nFStWyLqighjnz5+Xveq9dO+Pes1dUKYkzMEnLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYpAeD+fGP\nfyzr+/fvz2qf+tSnZC8rm1APJWdGOep8KlVLKaX58+dnNZdkU9dQK4lS0qm3rq4u2atWK6l1SceP\nH5eP/8UvfpHVnnnmGdn70Y9+NKtt27ZN9qpUo0sEqtfRvZfq9VWPL+1174W8buVOAAAuMoYWACAM\nhhYAIAyGFgAgDIIY72L9/f1Z7Ytf/KLsVV8Uf+tb36r5PQGOC1yo1T0uMKF63YoxdZ6WOsvKcdft\n7OysfA11D6dOncpqDz30kHz873//+6y2atUq2fuJT3wiq6kgSEr6TDB1rynp0IUKo7h6yfvuVmeV\n4JMWACAMhhYAIAyGFgAgDIYWACAMhhYAIAzSg+8CboXJbbfdltXcSpvbb789q7GuCY3kVv+U9Jak\n09RBki6d1traWum53DXc/aqVTWoN0xNPPCEfr9KON998s+xdt25dVnOrmdRr49KS6t8U99ooKqmY\nUtmBkSW/O3zSAgCEwdACAITB0AIAhMHQAgCEQRCjwdRqkxtuuEH2/v3vf89qGzdulL3f/OY3p3dj\nQAO5L95VWMAFMVSv+6JfBSZKzv9y11Urm/bs2ZPVDh48KB+/fv36rLZjxw7Zq9YwTUxMyF51bpUL\nfLW0tGQ1dR5XSmXnnanXseScLodPWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwSA822OnTp7PaU089\nVfnxDz74oKz39PS801sCirlEn6ISYyWHQDY3N1e+B5d6a2pqympu/ZA6UNUl79Tf7r59+7KaSi+m\nlNKuXbuy2urVq2WvSgqWrGZyyT2VNFRrr1LSa6PcyieXKpwuPmkBAMJgaAEAwmBoAQDCYGgBAMIg\niFEnZ8+elfXt27dXvsbPfvazrLZ169Z3fE9Aragv2V0oQAUmVODCcauK1HXVSiJ3DRcGUb2vvPKK\n7H344YcrXffGG2+Uj7/pppuyWkdHh+xVr687T0uFTFxIRd2vOo8rJf2+uVCOqpcEeBw+aQEAwmBo\nAQDCYGgBAMJgaAEAwmBoAQDCID1YJ/fff7+sDw0NVb7Gtddem9Vqkb4Bpksl2dyaIKVkjZM6/ND1\nujSdul+3WkmtbFKrmVJK6fDhw1mtvb09q1155ZXy8WpFlUtLlqxFUklB92+HSgq69KCqu/eyXv9W\n8UkLABAGQwsAEAZDCwAQBkMLABAGQYwaGBgYyGrf+MY3Gn8jQINMTk5mNXUuU0o6oOFCGyW9Kojh\nekuCI0ePHs1qTzzxhOwdHBzMatdcc01Wc2dkqbVTbh2W+nldCEL9bC7Qoq5Rspqp0fikBQAIg6EF\nAAiDoQUACIOhBQAIg6EFAAiD9GAN7N+/P6udO3eu8uM3btwo6+5AO+BiU0lBlywrWe+kruEScmpl\nk1qLlJJezeTWIqkDH48fPy571YGNmzZtymo9PT3y8eoehoeHZe90X0eXSqwXdb+1+B3hkxYAIAyG\nFgAgDIYWACAMhhYAIAyCGA32gQ98IKvt3btX9hLEQCRqzZDjvnh3oQulq6ur8nVVAMCdW6XWHfX2\n9speFbr45Cc/mdWWLl0qH6/uwa1bUj+DW53V1NSU1cbHxytftyQYUfKal/Q6fNICAITB0AIAhMHQ\nAgCEwdACAITB0AIAhDGnJCUyTQ17IsxqF/+UullgYGAg+3suWRNU8u+OSyWq55uampK98+blQWl3\n3dHR0cr3ptJ7Krmnnj8lnZYsOfTSKVmhpOquVyUbSxKf7mdQz9fX1ydvgk9aAIAwGFoAgDAYWgCA\nMBhaAIAwGhnEAABgWvikBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUA\nCIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD\noQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACGNeA5/r7QY+\nF2avORf7BmaDoaGhaf09v/22fvicOdXfPnWNksfXS8k9vPXWW9O6rnsudV3Xq17HefP0aLhw4ULl\n6yol99vX1yeb+aQFAAiDoQUACIOhBQAIo5HfaQFASsl/t/Hmm29mtUsuuUT2qrr7jmju3Pz/5+57\nNdXrrqvuoeR7H9Xrvk9SSn6Gku8R3c873e8Ma/FdJp+0AABhMLQAAGEwtAAAYTC0AABhMLQAAGGQ\nHgRQTKW9XDKsZHOFSuOVXFel5tx1HZXeGx8fl70q/afuYWpqSj5eXbe1tVX2lmyuKElAqnotUphK\nLTaW8EkLABAGQwsAEAZDCwAQBkMLABAGQYwa+PnPf57VRkdHZe+BAwey2g9/+MPKz/X1r39d1nfu\n3JnVrrvuusrXBUqoL+TVCqZacCEKFRZwoQAVeDhz5ozs/cc//pHV+vv7Ze+RI0eymvrbdz/DqlWr\nstqOHTtk7/bt27PaggULZK96bdz7U3I8igu6VOWeqyTMwSctAEAYDC0AQBgMLQBAGAwtAEAYDC0A\nQBikBwt84QtfkPX77rtvWtctSeR8+9vflvU9e/Zktaefflr2Lly4sPLzAYpKotXkgL+CvwV1XbVW\nKaWUTp48mdWefPJJ2fvCCy9ktX379slelRRsbm7Oat3d3fLx6nVcsmSJ7F20aFFWW7x4sextb2/P\nam1tbbJXJRtd0rBkJVdJbwk+aQEAwmBoAQDCYGgBAMJgaAEAwiCIYajQxXQDFymltHXr1qx26623\nyt6BgYGs9pOf/ET2vvzyy1ntV7/6lez97Gc/+99uEfh/1e1LdhHEcCuQVP3UqVOyd//+/Vlt7969\nsvfVV1/NaldeeaXs7e3tlfX/y611U+dsqedPKaUrrrgiq61cuVL2qjBIS0uL7FWrlUZGRmSv4t6f\npqamrOYCHi5Ao/BJCwAQBkMLABAGQwsAEAZDCwAQBkMLABDGrE8P/vOf/5T1H/3oR5WvsW3btqz2\n6KOPyt7W1tasNn/+fNmrkjavvPKK7H3mmWeymlpdA9SCSoy5NU6q7lJkKnHm/j7Onz+f1VSKNqWU\n/vrXv2a1yclJ2Xvttddmtcsvv1z2rl69Oqv19PRkNZcIVIdInjhxQvaqn9eZNy//p/3cuXOyVx0k\n6ZKG6jVTz+W435GSa/BJCwAQBkMLABAGQwsAEAZDCwAQxqwPYriwgvrCUAUuUkrpsccey2rqPJtS\nDzzwQFb74x//WPnxt9xyy7TvAVBUkMKtcSoJbaiVQi6AoIIFai1SSin19fVltaVLl8retWvXZjW3\nxkkFFsbHx7OaC4ioINjRo0dl77Jly7Lahg0bZK86e8sFT9QKJReMUGu23Aom9ftQEtZx+KQFAAiD\noQUACIOhBQAIg6EFAAiDoQUACGPWpwevuuoqWVepQrdOxq08mS61SsolgIBGKkmGqUSfSqGllNLE\nxETle1DP59J0ixcvzmpuhZtK6bm/cfU6nD59OqsNDw/Lx6uDXtXjU0rp0KFDWW3Tpk2yVyUj1Yqs\nlPSBj2q1U0r630CV+EypMBFofh9kb+VOAAAuMoYWACAMhhYAIAyGFgAgjFkfxHAWLlzYsOd68MEH\nZf3gwYOVr3H99ddnNbW6BqgFt7Kpaq8LFKkVSGoNVEr6b7S5uVn2qiCFO7dK1V1ARN2Devzx48fl\n41esWJHVOjs7Ze+aNWuymlsXp8IRbjWTOuPPrWZS4QoX8Kh6X+66Dp+0AABhMLQAAGEwtAAAYTC0\nAABhMLQAAGGQHmywF198Mat9/vOfl70qseQOrtu9e3dWK0n1ACVU2sul/NSBkSWHQLpEYFdXV+V7\neOONN7La6tWrZa9KMLr7VQdUvv7665WePyX9N64OoUwppfXr12c1t25JXXdsbEz2qgSi+7dDrVsq\nOdhR/S646zp80gIAhMHQAgCEwdACAITB0AIAhEEQo8GeffbZrFZyhtCdd94p6+pLWqCRSlb/uJVC\nHR0dWa3kvDp1dldKOqCxfPly2Xv27Nms5s64GhoaymovvPBCVhscHJSP37x5c1bbsmWL7H3ve9+b\n1Xp7e2Wv+jdFhUZcrzs7UL3HbqWXCtW4wEXJWjA+aQEAwmBoAQDCYGgBAMJgaAEAwmBoAQDCID1Y\nJ3fccYesP/TQQ5Wv8eUvfzmrffWrX33H9wTUk0uGqfSgW/2jruGShqOjo1nNpQfVuqPu7m7Ze+bM\nmazm0n8nT57MampV0bZt2+Tjd+7cmdXWrVsne9W6JbcWSSX3XCJQ9ZYkQd2aLZUIdGu23OGQCp+0\nAABhMLQAAGEwtAAAYTC0AABhEMSogZGRkaz229/+Vvaqs3rcKpZ77rknq7kvU4GLrWQVT0kQw31J\nr9YtuV4VFnChDfX3fOTIEdmr1jup12HDhg3y8cuWLctqLgSh/u1wq5na2tqymgpypKTXOLlwhep1\n96sCNJOTk7LXBTQUPmkBAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA9WAO7du3Kaq+//nrlx3/pS1+S\n9Z6ennd8T0CjlaziqcXBgSpJqxJ2Kemkofsbffzxx7Pac889J3uPHTuW1ZYsWZLV3EGvahXVwoUL\nZa9KCroUpkoPqlpK+nV096sSly7RrO6tJCXo8EkLABAGQwsAEAZDCwAQBkMLABAGQYwCBw4ckPWn\nnnqq8jU+/vGPZ7W77rrrnd4SEJILYqgv710Qo6mpKau5s7fGxsay2r59+2Tv3r17s9qf//xn2bt8\n+fKsdtVVV2W1q6++Wj5+zZo1Wc2tZlLrktxro84PU69XSjr84oIYJe+PO+tLYY0TAGBGYmgBAMJg\naAEAwmBoAQDCIIhhqC9u7777btnrzohR1BeynJGF2cZtz1ABDXdekzrzyYUNDh8+nNVeeukl2dvf\n35/VVLAhJX1O1ubNm7Pa4sWL5eNVAME9l+rt7u6Wver5XNhBvebuPC31b5V7L9X75oIyJWex8UkL\nABAGQwsAEAZDCwAQBkMLABAGQwsAEAbpQePee+/NauqcHeeOO+6QdVY2YaZyZzu5NT9Ve106TSUF\n33jjDdn75JNPVqqllFJLS0tWu/HGG2Xvzp07s5o6D0udQ5WSvt+uri7ZqxKBHR0dslel/9xqJpXo\nU69BSjpVPTw8LHtVWrHk7C2HT1oAgDAYWgCAMBhaAIAwGFoAgDAIYhj33HPPtB7/ve99T9ZZ2YSZ\nqhZnZKnQhVuTpsIGg4ODsvf555/Pav/+979l76ZNm7JaX1+f7F25cmVWU2uY3BlZKsTg1lapuntt\nVEjFnW+lAhMlwZGTJ0/KXhXmcGucOE8LADAjMbQAAGEwtAAAYTC0AABhMLQAAGGQHqyTkZERWS9Z\naVNCJalcIkeliNyKF0Wtckkppd27d1e+huLuVyU53WF/aAyVCHTpQfU7794/lR50STb1fKdPn5a9\nR48ezWqdnZ2yt62tLaudO3dO9r788stZTaUHh4aG5ONV+s8lDZcvX57V3N+MWvmkfi5Xd6/5oUOH\nspr7t2PLli1Zza2oKklV80kLABAGQwsAEAZDCwAQBkMLABAGQYw6UV+a1tOdd96Z1ZYtWyZ7X3vt\ntaz2gx/8oOb3VCvqtfzc5z53Ee4E/8OFLqpy5yepIIZb/aPWBPX29speFVRSYYWU9BlVBw4ckL2P\nPvpoVnNrjRQVUnHrltTfgXsfVJjDveZqRZU6E8zd24oVK2TvdH9HHD5pAQDCYGgBAMJgaAEAwmBo\nAQDCYGgBAMIgPWjcfvvtWe3++++/CHdSzb333luX66rkVsmBbZ/5zGdkfceOHZWv8cEPfrByLy6e\nkrVhKiXoel3qTR2KuGrVKtm7ffv2rDYwMCB71e+8OvwwpZTOnj2b1VSC0T1eJSBPnDghe1999dWs\nNjo6KnvVGjmXCFTrrNy6JfWzXX755ZV7a7F+jU9aAIAwGFoAgDAYWgCAMBhaAIAw5rgvOeugYU9U\nLz/96U9lXZ2JU+LgwYOyPt3VSl/5yldkfd26dZWvcfPNN2c1t/7mXaI+u2PwvwwNDVX+ey45e0sF\nNMbHx2WvChC4M6PUeVZuNZM6e8tdd3h4OKupwMPx48fl49U5XS5csWTJkqzmVj6pMElPT4/s7e7u\nzmrub1z1Llq0SPa2trZWui9n7dq18peET1oAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMEgPYqYhPdgA\nJelBlQh0K5/UaqaxsTHZq1YCqcMeU9Ipv4mJCdmrEnkueacOjGxvb89q6lDGlPSBrO6+1M82f/58\n2auSle7f+qmpKVlX1HvpkqAqKejSjuoal112GelBAEBsDC0AQBgMLQBAGAwtAEAYnKcFoJj6Qt2F\nK1TdhQLUl/cqVJCSDmK4c7rUGiZ3tpNay+Z+NnWelgp9uLCDCl3Mnas/S6h7cK9NSRhEvZcuMOHu\nrep1nZJAIJ+0AABhMLQAAGEwtAAAYTC0AABhMLQAAGGQHgRQTCXZXMKuZPWPSpG5ZJlK3rnrKq63\nJGlY9VBDdSBiSno1k0vdqXtwr41KCroEY8khne8GfNICAITB0AIAhMHQAgCEwdACAIRBEANATZSs\n7anFdauGIFLSwYKSx7vAg7o3FTxx66UUd1/qrDEXmFChmJJ7qBf3OpYEP/ikBQAIg6EFAAiDoQUA\nCIOhBQAIg6EFAAiD9CCAYo1c/ePWQ6l7cEnDkrVTJYdWVj0U0T1ePZe7ZskBjOPj45Ufr+7NvTYl\nCVH1++DuoSTZyCctAEAYDC0AQBgMLQBAGAwtAEAYc9wXhAAAvNvwSQsAEAZDCwAQBkMLABAGQwsA\nEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAG\nQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQxn8Ai+FDB8A7dBQAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc713e6af60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer(mode=\"FAN_AVG\",\n",
    "                                                             uniform=True)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "weights1 = tf.Variable(initializer([n_inputs, n_hidden1]))\n",
    "weights2 = tf.Variable(initializer([n_hidden1, n_hidden2]))\n",
    "weights3_mean = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights3_log_sigma = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights4 = tf.Variable(initializer([n_hidden3, n_hidden4]))\n",
    "weights5 = tf.Variable(initializer([n_hidden4, n_hidden5]))\n",
    "weights6 = tf.Variable(initializer([n_hidden5, n_inputs]))\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros([n_hidden1], dtype=tf.float32))\n",
    "biases2 = tf.Variable(tf.zeros([n_hidden2], dtype=tf.float32))\n",
    "biases3_mean = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases3_log_sigma = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases4 = tf.Variable(tf.zeros([n_hidden4], dtype=tf.float32))\n",
    "biases5 = tf.Variable(tf.zeros([n_hidden5], dtype=tf.float32))\n",
    "biases6 = tf.Variable(tf.zeros([n_inputs], dtype=tf.float32))\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "\n",
    "hidden3_mean = tf.matmul(hidden2, weights3_mean) + biases3_mean\n",
    "hidden3_log_sigma = tf.matmul(hidden2, weights3_log_sigma) + biases3_log_sigma\n",
    "noise = tf.random_normal(tf.shape(hidden3_log_sigma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.sqrt(tf.exp(hidden3_log_sigma)) * noise\n",
    "\n",
    "hidden4 = activation(tf.matmul(hidden3, weights4) + biases4)\n",
    "hidden5 = activation(tf.matmul(hidden4, weights5) + biases5)\n",
    "logits = tf.matmul(hidden5, weights6) + biases6\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_log_sigma) + tf.square(hidden3_mean) - 1 - hidden3_log_sigma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "with tf.contrib.framework.arg_scope([fully_connected],\n",
    "                                    activation_fn=tf.nn.elu,\n",
    "                                    weights_initializer=initializer):\n",
    "    X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "    hidden1 = fully_connected(X, n_hidden1)\n",
    "    hidden2 = fully_connected(hidden1, n_hidden2)\n",
    "    hidden3_mean = fully_connected(hidden2, n_hidden3, activation_fn=None)\n",
    "    hidden3_gamma = fully_connected(hidden2, n_hidden3, activation_fn=None)\n",
    "    noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n",
    "    hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n",
    "    hidden4 = fully_connected(hidden3, n_hidden4)\n",
    "    hidden5 = fully_connected(hidden4, n_hidden5)\n",
    "    logits = fully_connected(hidden5, n_outputs, activation_fn=None)\n",
    "    outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train cost: 18606.2 \tReconstruction loss: 14819.0 \tLatent loss: 3787.25\n",
      "1 Train cost: 16776.8 \tReconstruction loss: 13075.5 \tLatent loss: 3701.32\n",
      "2 Train cost: 16465.2 \tReconstruction loss: 12710.3 \tLatent loss: 3754.91\n",
      "3 Train cost: 16599.6 \tReconstruction loss: 12777.0 \tLatent loss: 3822.65\n",
      "4 Train cost: 16195.2 \tReconstruction loss: 12320.3 \tLatent loss: 3874.92\n",
      "5 Train cost: 16021.1 \tReconstruction loss: 12250.5 \tLatent loss: 3770.53\n",
      "6 Train cost: 15870.0 \tReconstruction loss: 12095.8 \tLatent loss: 3774.19\n",
      "7 Train cost: 16136.7 \tReconstruction loss: 12267.2 \tLatent loss: 3869.49\n",
      "8 %Train cost: 15875.1 \tReconstruction loss: 12092.0 \tLatent loss: 3783.13\n",
      "9 Train cost: 15580.6 \tReconstruction loss: 11930.5 \tLatent loss: 3650.03\n",
      "10 Train cost: 16030.2 \tReconstruction loss: 12270.3 \tLatent loss: 3759.87\n",
      "11 Train cost: 15378.9 \tReconstruction loss: 11545.0 \tLatent loss: 3833.88\n",
      "12 Train cost: 15337.2 \tReconstruction loss: 11546.4 \tLatent loss: 3790.84\n",
      "13 Train cost: 15994.3 \tReconstruction loss: 12163.5 \tLatent loss: 3830.81\n",
      "14 Train cost: 14971.5 \tReconstruction loss: 11317.0 \tLatent loss: 3654.44\n",
      "15 Train cost: 15843.0 \tReconstruction loss: 12050.9 \tLatent loss: 3792.15\n",
      "16 Train cost: 15069.3 \tReconstruction loss: 11359.0 \tLatent loss: 3710.32\n",
      "17 Train cost: 15422.4 \tReconstruction loss: 11637.2 \tLatent loss: 3785.22\n",
      "18 Train cost: 15438.7 \tReconstruction loss: 11687.6 \tLatent loss: 3751.15\n",
      "19 Train cost: 14984.6 \tReconstruction loss: 11216.3 \tLatent loss: 3768.25\n",
      "20 Train cost: 14704.2 \tReconstruction loss: 11028.1 \tLatent loss: 3676.13\n",
      "21 Train cost: 15276.4 \tReconstruction loss: 11517.0 \tLatent loss: 3759.33\n",
      "22 Train cost: 15510.2 \tReconstruction loss: 11725.9 \tLatent loss: 3784.34\n",
      "23 Train cost: 15190.2 \tReconstruction loss: 11402.1 \tLatent loss: 3788.12\n",
      "24 Train cost: 14963.1 \tReconstruction loss: 11238.4 \tLatent loss: 3724.7\n",
      "25 Train cost: 15016.2 \tReconstruction loss: 11323.5 \tLatent loss: 3692.74\n",
      "26 Train cost: 15273.6 \tReconstruction loss: 11513.8 \tLatent loss: 3759.86\n",
      "27 Train cost: 14804.0 \tReconstruction loss: 11065.3 \tLatent loss: 3738.63\n",
      "28 Train cost: 14421.7 \tReconstruction loss: 10794.7 \tLatent loss: 3626.98\n",
      "29 Train cost: 15030.3 \tReconstruction loss: 11319.5 \tLatent loss: 3710.85\n",
      "30 Train cost: 15015.6 \tReconstruction loss: 11235.7 \tLatent loss: 3779.92\n",
      "31 Train cost: 15164.7 \tReconstruction loss: 11399.4 \tLatent loss: 3765.35\n",
      "32 Train cost: 14998.0 \tReconstruction loss: 11148.6 \tLatent loss: 3849.44\n",
      "33 Train cost: 15035.4 \tReconstruction loss: 11261.1 \tLatent loss: 3774.32\n",
      "34 Train cost: 14661.0 \tReconstruction loss: 10954.7 \tLatent loss: 3706.24\n",
      "35 Train cost: 14787.9 \tReconstruction loss: 11092.8 \tLatent loss: 3695.13\n",
      "36 Train cost: 14925.3 \tReconstruction loss: 11259.0 \tLatent loss: 3666.24\n",
      "37 Train cost: 15087.2 \tReconstruction loss: 11296.1 \tLatent loss: 3791.1\n",
      "38 Train cost: 14526.3 \tReconstruction loss: 10798.2 \tLatent loss: 3728.12\n",
      "39 Train cost: 14927.5 \tReconstruction loss: 11250.3 \tLatent loss: 3677.24\n",
      "40 Train cost: 14467.2 \tReconstruction loss: 10821.2 \tLatent loss: 3646.0\n",
      "41 Train cost: 14638.6 \tReconstruction loss: 11002.9 \tLatent loss: 3635.68\n",
      "42 Train cost: 14746.2 \tReconstruction loss: 11073.8 \tLatent loss: 3672.47\n",
      "43 Train cost: 14830.3 \tReconstruction loss: 11105.0 \tLatent loss: 3725.31\n",
      "44 Train cost: 14188.7 \tReconstruction loss: 10546.7 \tLatent loss: 3641.98\n",
      "45 Train cost: 14416.7 \tReconstruction loss: 10829.1 \tLatent loss: 3587.54\n",
      "46 Train cost: 14966.1 \tReconstruction loss: 11291.3 \tLatent loss: 3674.81\n",
      "47 Train cost: 14484.0 \tReconstruction loss: 10847.7 \tLatent loss: 3636.37\n",
      "48 Train cost: 14867.3 \tReconstruction loss: 11230.3 \tLatent loss: 3637.02\n",
      "49 Train cost: 14913.4 \tReconstruction loss: 11177.9 \tLatent loss: 3735.57\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        cost_val, reconstruction_loss_val, latent_loss_val = sess.run([cost, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Encode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = hidden3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Decode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's plot the reconstructions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAHGCAYAAABaaN0mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGFZJREFUeJzt3XuwVmX5N/C1OQpCKip5QpMUAkvTEgxsbOxgZxKz0mxo\n1NTJDmPTSUdDGyebmnHin0wnB1O0cazJxj+y1MJBMRISJxrHUFNRUSATBNwcNrx/vO/7m9993wv3\n3hfP8+y9n+fz+e9ac6393DCz9pf1XNxrde3evbsCgP4aNtALAGBoEiAAhAgQAEIECAAhAgSAEAEC\nQIgAASBEgAAQIkAACBEgAISMaOFneWZKZ+ka6AXQVK7nzlJ7PbsDASBEgAAQIkAACBEgAIQIEABC\nBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAA\nhAgQAEIECAAhAgSAEAECQIgAASBkxEAvAOD/2717d1N+bldXV1N+bqdzBwJAiAABIESAABDSljOQ\nv/71r0m9YMGCpD788MOLc8aMGZPU8+bNK3omTJjwpjWwd3p6eopjq1atSuo//OEPSf3nP/+5OOep\np55K6re97W1Fz8c+9rGknjNnTtFz5JFHJvXIkSOTetiw8t/gfZnj9GUm01vPYJjruAMBIESAABAi\nQAAI6WrW/7uu0bIPmjp1alKvXr26IT93v/32S+pTTjmlIT+3UfLveS+//PKkzr/PbbKB/4KWZmrK\n9bx9+/bi2O9///uk/trXvpbU69evL87Jf6/VzQvyuWc+36iqqtp///3f9Jy8rqqqevXVV5N61KhR\nRU9+rR5wwAFFTz6T+cxnPpPUY8eOLc5potrr2R0IACECBIAQAQJAiAABIKQth+hPPPFEUq9cuTKp\njzvuuOKcf/7zn0m9bNmyoicf5j333HNFz9FHH53U//73v998sTVGjCj3dx566KFJvWbNml5/znXX\nXZfU3/ve9/q9lr1giN7emnI91/0+2rJlS1J//etfT+qHH364OOfll19O6roBef5Z27ZtK3ry4Xs+\n5K9bb3791n32jh07krpuA+XBBx+c1MuXL0/qQw45pNf1NpAhOgCNI0AACBEgAIS05QykWbq7u5P6\n2WefLXryGcgzzzzT78+p23iUz0Dyz6mqckPV7373u6Sue1hcE5mBtLfW/eLIfkft3Lkzqbdu3Vqc\nk88U6uYQ+Wwln5vUHfvHP/6R1PmDHquqnE3kD3asqqp68MEHk3rjxo1FT/7Q16VLlyZ1/juhqsxA\nABgiBAgAIQIEgBABAkCIIfoQkW9snDVrVtEzY8aMpM7f1Fb35NAmMkRvbx1xPee/H3urq6ocZK9b\nt67o+cpXvpLUjzzySNGTP333Zz/7WVKPGzeuZsVNY4gOQOMIEABCBAgAIeVT+xhw+QanqqqqM888\nM6l37dpV9OTfkbZ45gFtJ59nRDbq1W1ifOGFF5K6bp5xwQUXJHWL30DYJ+5AAAgRIACECBAAQgQI\nACGG6IPQLbfcUhzLnwp64IEHFj1HHXVUs5YE9FH+H1zyp2JXVVU9//zzSV23Mfjd7353UjfxSbth\n7kAACBEgAIQIEABCPExxEHj66aeTevr06UXPjh07kvrJJ58seo499tjGLmzvDL4vbGkk1/MevPrq\nq0l90kkn9dqzaNGioueTn/xkUg8bNqD/3vcwRQAaR4AAECJAAAixD2QQuOeee5I6n3dUVVWdffbZ\nST158uSmrgnoXd1DTefPn5/U//nPf4qed7zjHUl9+umnFz0DPPPok8G/QgAGJQECQIgAASBEgAAQ\nYojeYnUD8vxha6NHjy56rrvuuqQePnx4YxcG9CrfeJ1vAq6qqlq4cGFS1z0E8dJLL03qofr2UHcg\nAIQIEABCBAgAIWYgLXbzzTcXx5YsWZLU5557btFj4yC0Xj7z2LZtW1KfccYZxTlbt25N6kmTJhU9\nn/3sZ5N6KGwarDM0Vw3AgBMgAIQIEABCBAgAId5I2GQrV65M6pNPPrnoGT9+fFIvX7686BmCQ3Rv\nJGxvHXE9578fH3rooaT+0Ic+VJyTD8RvvfXWoicfotdtNhxkvJEQgMYRIACECBAAQmwkbLA33ngj\nqc8555yk7unpKc754he/mNRDcN4Bben1119P6osuuqjXc+bMmZPUc+fOLXqGwMyjT9yBABAiQAAI\nESAAhAgQAEJsJNwLu3btKo7lG4sWL16c1NOmTSvOyZ/GO2HChL1f3MBrjykhe9IR1/OFF16Y1Pmm\nwLon7a5YsSKp2/l6dgcCQIgAASBEgAAQYgayFzZs2FAcmzhx4pueU/egxJNOOqlhaxpEzEDaW9td\nz0888URxbObMmUmdz0nuv//+Xs9pk02DZiAANI4AASBEgAAQ4mGK/bBx48akPuWUU3o9Z9GiRUl9\n4oknNnRNQEx3d3dSf/7zny96tm3bltSf+9znknrGjBnFOW0y8+gTdyAAhAgQAEIECAAhAgSAEEP0\nfli4cGFSP/PMM72ec+qppyZ1Jw3YYLCo2zB9xx13JPXzzz9f9OQPQvzRj36U1MOGdfa/wTv7Tw9A\nmAABIESAABBiBrIHq1evLo5dffXVrV8I0G/5zGPp0qVFz5VXXpnUPT09Rc/FF1+c1IcffngDVtc+\n3IEAECJAAAgRIACECBAAQgzR92DJkiXFsU2bNvV63rRp05J6zJgxDVsTUKrbJLh27dqkrnvS7vr1\n65O67sm6V1xxRVJ3+sbBnL8NAEIECAAhAgSAEDOQvTBr1qzi2H333ZfUZiDQejt27EjqY489tujZ\nsmVLUs+dO7foMfN4c/52AAgRIACECBAAQgQIACFddZtwmqRlH8Sg4NWL7W1IXc/d3d3FsXxAPnLk\nyKLHG0T/R+1fhDsQAEIECAAhAgSAkFbOQABoI+5AAAgRIACECBAAQgQIACECBIAQAQJAiAABIESA\nABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCI\nAAEgRIAAEDKihZ+1u4WfxcDrGugF0FSu585Sez27AwEgRIAAECJAAAgRIACECBAAQgQIACECBIAQ\nAQJAiAABIESAABAiQAAIaeWzsIaU22+/vTi2ZcuWpF6xYkXRc9NNN73pz73qqquKY6effnpSf+AD\nH+jDCoGqqqrdu8vHcm3atCmpV65cWfS8+OKLSf3KK68UPfk1v2bNmqSePXt2cc6sWbOS+uijjy56\nhg8fXhwbityBABAiQAAIESAAhAgQAEK66gZQTTKoX0Dz1a9+NalvvPHGln329OnTk/qhhx4qevbb\nb79WLadRvFCqvQ3Y9bxr166kzgfdVVVVd999d1Jff/31Rc9LL72U1Dt37ix6enp6knr06NFJPXny\n5OKcGTNmJPWVV15Z9EyYMCGph8BQ3QulAGgcAQJAiAABIKQjZyD5vKOqYjOPE088sTh21llnJfXq\n1auT+le/+lWvP7duM+IFF1zQz9UNODOQ9jZg1/O2bduS+oYbbih68pnH+vXri5599tknqffff/+i\nZ9y4cUl9wAEHJPWGDRvefLFVVX3hC18ojn3zm99M6re85S1FT1fXoLqEzEAAaBwBAkCIAAEgRIAA\nENIRT+N9/vnnk/qXv/xlr+ecfPLJxbF77703qceOHVv0jBo1KqnzjUhPPfVUcc7DDz+c1H0ZzEGn\nyofoeV1VVTV+/Piknjp1atEzf/78pD7mmGOKnvwa7+7uTupVq1YV51x99dVJvWTJkqLn/PPPT+q6\nIfpQ4A4EgBABAkCIAAEgpCNmIPlMoW7zZD7zuP/++4uefFNRX9xyyy1J/eijj/Z6zpw5c/r9OdCu\n8us13wA4d+7c4px3vetdST1p0qSiZ8qUKUk9cuTIXteSf/ZRRx1V9OQPe9y+fXvRk89Ghyp3IACE\nCBAAQgQIACEdMQM56aSTkrpun0W+f2PMmDEN+ex8z0nd96HAnuUPFRwxIv21NXHixOKcfB9I3YMS\n82u+Tj5/2bRpU1LXPYT1hRdeSOojjjii6Nl33317/eyhwB0IACECBIAQAQJAiAABIKQjhui5/fbb\nr2k/+7bbbkvqxx9/vNdzPvKRjyT129/+9oauCdpJPlSv+w8v+UMQhw8fHvqs/OGJ+ZsEH3zwweKc\nfLPhueeeW/Tkv4MG2dsH+8wdCAAhAgSAEAECQEhHzkAa5bHHHiuOXXzxxUmdv+zm0EMPLc5ZsGBB\nUvfloW7A/1V3veQbAOtmDHlP3YupfvKTnyR1/lK5fFNjVVXV2WefndT5y6P2dN5Q5A4EgBABAkCI\nAAEgRIAAENIek5wB8sgjjxTH6gZx/9sll1xSHMvfjAbsWT4Qr3vDaC5/S2BVlZsEv//97xc9ixYt\nSur8adrHHntscc4Pf/jDpB49enTRM1Q3DubcgQAQIkAACBEgAISYgfRDviHozjvv7PWcyy67LKm/\n+93vNnRN0On6Mk/o6ekpjt18881Jnb89tKrKmcchhxyS1PnDU6uqfNtgu8w76rgDASBEgAAQIkAA\nCBEgAIR09WUTToO07IMaYfPmzcWxfNPQunXrip63vvWtSb1q1aqknjBhQgNWNyS07+SQqhrk13O+\ncXDNmjVFz8yZM5P6lVdeKXpGjRqV1HfddVdSf+ITnyjOib79cJCrvZ7dgQAQIkAACBEgAITYSLgH\n+VvFqqp+5pH7xje+kdQdNPOAAVE3x81nmJdeemnRs2HDhqSum12ceuqpSf3Rj340qYcN6+x/g3f2\nnx6AMAECQIgAASDEDOT/WbFiRVIvXry413Pmzp1bHPvWt77VqCUBfVD3Erdvf/vbSf3ggw8WPfnM\n45hjjil6rr322qTu9JlHzt8GACECBIAQAQJAiAABIKQjh+hvvPFGcezyyy9P6vxNZHXe8573FMfy\nh68BjZU/KPHee+8tev74xz8m9YgR5a+6I488MqmvueaaoueEE05I6r4M0SMPqB2qby10BwJAiAAB\nIESAABDSkTOQX/ziF8WxBx54oNfzzj///KS2aRBa7/XXX0/q22+/vejZtGlTUk+aNKno+elPf5rU\np512WtEzevTofq+vLzOQwTTzqFtvX9fnDgSAEAECQIgAASBEgAAQ0hXZ9BLUsg/qzZgxY4pjfdk4\nuHHjxqQeN25cw9bUhgbPlJBmaN0vjux31MqVK5O67u2h+dsG58yZU/TccMMNSb3PPvsUPb1tHKz7\n/ZlvdGzlE3zzz657UnG+qXLkyJFFT80QvfZ6dgcCQIgAASBEgAAQ0pEbCaM2b96c1I36bjPfrJS/\nKa2qqqqnpyep677bzOUPjVywYEFgdeV6rrjiiqKn7ntUaIb87aGvvvpq0ZN/h183r9y5c2dS59dY\nVZUzjnzGEN2El59Xdz3nmyHzOW3d3Hbt2rVJ/eijjxY9+UMkZ82aVfQcdthhSb2nP5M7EABCBAgA\nIQIEgBABAkCIIXo/HH744U35uZdccklS5wOsqqqql19+Oal//vOfN2UtfVH393DhhRcOwEroRNOm\nTUvqsWPHFj2vvfZaUt99991FT77ZMP+5VVUOsv/73/8m9cEHH1yckw+lDzrooKLnT3/6U1IvWbKk\n6Mn/c83jjz+e1HX/2Sa/NvMnF1dV+dbUW2+9tdefsyfuQAAIESAAhAgQAEI68mGKdd/XL1y4cABW\n0lj5Q9LqviPNffnLX07q973vfb2eM3v27OLY5MmT80MeptjeBux67u7uTuof//jHRU/+oMR8E3Cd\nut+F+Wa9fCNh3Qa7/fffP6nrNgnmP7fuWs035+YbH+tmpXWbIXMf/vCHk/r6668vemo2XnqYIgCN\nI0AACBEgAIR05AykTv5/ofvygqk6+f/VjuzX+M53vlMcO+aYY3o979Of/nRST5w4sd+f3UBmIO1t\n0FzPdd/75/sfFi9eXPTcc889Sf3YY48VPevWrUvqfJ5Rtwcln0Xms4uqqqodO3Ykdb43o6qq6pRT\nTknqiy66KKnr9qAsW7YsqfN9K1VVvlxrypQpRU8NMxAAGkeAABAiQAAIESAAhBii0yyG6O2t7a7n\nvvwuzHuivz/78tbCvKcv5zSRIToAjSNAAAgRIACEeKEUQBWbS3Q6dyAAhAgQAEIECAAhAgSAEAEC\nQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACEtPKNhF7lBe3D\n9Yw7EABiBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAi\nQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAACBEgAISMaOFn7W7hZzHwugZ6ATSV67mz\n1F7P7kAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACECBIAQAQJA\niAABIESAABDSyhdKsQcbNmxI6okTJxY9d911V1KfddZZTV0TELNs2bKkvuqqq4qeH/zgB0k9e/bs\noqera/C/k80dCAAhAgSAEAECQIgZyCDw5JNPJvWwYWWuH3HEEa1aDtBHu3fvLo7ddNNNSf23v/2t\n6FmzZk1SD4V5Rx13IACECBAAQgQIACECBIAQQ/RBIN94NH78+KJn5syZrVoO0Efbt28vjt13331J\nvW3btqJn6tSpTVtTK7kDASBEgAAQIkAACDEDabG1a9cWx+bPn5/Ul112WauWA+yF/EGodcfGjh1b\n9Bx33HFNW1MruQMBIESAABAiQAAIMQNpseeee644tmXLlqQ+77zzWrUcoB/yhycuXLiw6Nm5c2dS\nz5kzp+gZNWpUYxc2QNyBABAiQAAIESAAhAgQAEK66t6o1SQt+6DB7IMf/GBxLH872apVq4qeITh0\nG5qvWKOvOvJ63rp1a1K/853vLHrWrVuX1P/617+KnsMOO6yxC2u+2uvZHQgAIQIEgBABAkCIjYRN\n9tprryX1X/7yl6Ln+OOPT+ohOO+AjvDCCy8k9SuvvFL0vPe9703qgw46qKlrGkjuQAAIESAAhAgQ\nAEIECAAhhuhN9ve//73XnkmTJrVgJUB/5Rutly5dmtTd3d3FOZ/61KeSesSI9v016w4EgBABAkCI\nAAEgpH2/nBskHn300V57rrnmmhasBOivHTt2JPWCBQuSeteuXcU5M2bMSOphw9r33+nt+ycDoKkE\nCAAhAgSAEAECQIg3EjbYM888k9QzZ85M6unTpxfnPPDAA0ndJhuPvJGwvXXE9Zw/TTt/su7IkSOL\nc/I3Eo4fP77xC2s9byQEoHEECAAhAgSAkLb4sn0wyecZGzZsSOoTTjihOKdNZh4wpNXNgx977LGk\nzjcOzpo1qzhn7NixjV3YIOYOBIAQAQJAiAABIMSX7w22fPnypO7qSv/79HnnndfK5QB9tHPnzuLY\ntddem9T5nGTevHnFOe388MRc5/xJAWgoAQJAiAABIESAABDiYYp7YfPmzcWxqVOnJvXEiROTOt+Y\n1MY8TLG9td31nD84saqqatKkSUmdb/pdsWJFcc7kyZMbu7DBwcMUAWgcAQJAiAABIMRGwr3wm9/8\npji2du3apD7nnHNatRygH/L5769//eui54033kjq0047LanzGUmncQcCQIgAASBEgAAQIkAACDFE\n3wtPP/10rz0HHnhgC1YC9NeOHTuS+oYbbih6Ro4cmdRnnXVWUnf620TdgQAQIkAACBEgAIR09hd4\ne+m2227rtefMM89swUqA/nrppZfetK6qqtp3332T+uMf/3hS528c7TTuQAAIESAAhAgQAEIECAAh\nhuj9sHr16qR+8cUXB2glQH/UvXn1xhtvTOpt27YVPfnTdm0MTrkDASBEgAAQIkAACDED6Yff/va3\nSd3T01P0vP/970/qKVOmNHVNQO+2bt1aHLvjjjt6Pe+MM85I6rFjxzZsTe3AHQgAIQIEgBABAkCI\nGcge5C+bqaqquvPOO3s9b968eUk9bJiMhoH27LPPFsfWrl2b1MOHDy96zj333KR2Paf8bQAQIkAA\nCBEgAIQIEABCDNH3oG5YdsghhyT1iSeeWPR86UtfatqagJjRo0cXx6ZOnZrU06ZNK3qOP/74pO70\nNxDm3IEAECJAAAgRIACEdNW9aKVJWvZBDAq+LG5vrufOUns9uwMBIESAABAiQAAIESAAhAgQAEIE\nCAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgZEQLP8sb6qB9uJ5x\nBwJAjAABIESAABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgA\nIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACH/B5czMtAfmBYwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71432d710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Generate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure generated_digits_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEGCAYAAAAwpAFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXd4VFX++P+aPuk9BFJIqAFEQEAQZEGkWRZYG6BYQF3s\n+9jL2r7YFRYXXWwfREUEdRVQUJAivYZepIQUkkBIgSQzSabe+/tjfuc4g+iuMDMJ7n09j8+jMZl7\n5txzzrue91unqioaGhoaGhrNDX1TD0BDQ0NDQ+NMaAJKQ0NDQ6NZogkoDQ0NDY1miSagNDQ0NDSa\nJZqA0tDQ0NBolmgCSkNDQ0OjWaIJKA0NDQ2NZokmoDQ0NDQ0miWagNLQ0NDQaJZoAkpDQ0NDo1mi\nCSgNDQ0NjWaJsakH4IdWFFBDQ0Pjj4fubP9Qs6A0NDQ0NJolmoDS0NDQ0GiWaAJKQ0NDQ6NZogko\nDY0govVX09AIHs0pSUIjTIhDVKc769ilxq+gzWnz5nQFQntfzRtNQP3B8Xq9uN1uGhoa2Lp1KwC7\nd+/mwIED1NbW0rFjRwAyMjLo2rUr0dHRZGVlARATE4PBYAjYxNqG1jjfUBQFgLq6Oo4ePYrL5cJs\nNgPQunVrYmJi0Os1Z1JzRHsrGhoaGhrNkvPWghKmuqqqKIqCoihSuzeZTE05tGaFmJ+KigqioqIA\nOHr0KEVFRURHR3PgwAEAbDYbdXV1mM1mqV1edNFFZGRkkJKSgsViCfjMUFtSYtwejweHwwFARUUF\njY2NJCcnExcXB/jetclkalaWneZGaj6oqsqpU6cAmDt3LsuWLaOqqorGxkYAevbsyd/+9jfatWsn\n171mTf2M1+tl/fr1TJs2jW7dugFw5513kpaWhsFgCPnzdc0oqPtfD8Tj8VBQUADAkSNHqKioYOXK\nlURHRwPw3HPPkZKS0uwOBq/Xi81mA+CRRx6hsLCQ9957j7Zt2wLBPcj836uiKDQ2NlJZWQnAqVOn\nMJvNJCYmEhsbC4DRaERRFFwuF0ePHgVgw4YNJCQkcPHFF5OZmSl/L1SI5wNUVlayf/9+li9fzoYN\nGwA4fvw4Op2ODh06cP/99wPQo0cPUlJSfuGKDOaYABoaGjh06BANDQ20bNkSgJYtW2IymQIONJ1O\nh6qq8h/wzVmwx6YoCl6vl8bGRmpra+Vz4uPjsVqtzW7ti7lwu904HA4iIiLkARcqgaCqKg0NDXz4\n4YcATJs2jaqqKrxerxyPwWAgIyODBx54gPHjxwMQHR3dZPMn1o3L5ZJnRXR0NEajMWyKt6qq1NfX\nA/DUU08xc+ZM+c4AHnzwQZ599tnfM56znsxmb0H5b3RVVfF6vXg8HjmB27dvp7S0lOPHj3Pw4EHA\nd7COHTuWRx55JCxS/tcQh5vdbmf37t189tln7N27F4D9+/fjcDh45513eP311wGCOlb/DWYwGIiK\nisJqtQKQmZn5q8+yWq106NABgOrqar777jsMBgPp6elBG9vpqKpKY2MjeXl5vPzyywDk5eXhcDhQ\nFCXgMImOjqahoYGysjIAunfvHrBGIDiCXhxuX3/9NQAvvPACx48fJyEhgdtvvx2AMWPGkJCQgF6v\nlxo5+A5cVVWlME9OTj6nw0VRFE6dOsWGDRt4//33AVi7di2NjY243W75ezqdDqPRyO23387bb78t\nx9KUqKqK0+lk7dq1APzzn/8kNzeXv//979IKDuVzFy1axCuvvAJAVVUVqqqi1+vlu/F6vZSXl/PW\nW29RVVUFwGOPPRbgMQjlGEV8WJwVDocDh8PB3LlzOXHiBACTJ0+WezfU4/F4PFRWVjJ9+nQAPv30\nUxobG+XaAp/VGa5ztdkLKJvNJjehTqfDbrdjMBjkYrLZbBiNRtLS0igqKgJ8mva7775Lv379uPTS\nS+Xfhguv18uxY8f47LPPANi2bRvFxcVERUXJhR8XF0dkZGRID35/dDpdwKI603yIQ14calFRUVIr\nD8VBJzZlSUkJ9957L6tXr5YHvTjghaUHcMkll5CWlobZbJbuSpvNRkxMDBEREUGzWFRVZe/evdx3\n331s2bIFAJfLhcViITc3l3bt2gEQERHBqVOnyM/Pl7934sQJLrnkEnJycqR1mpqaelbj8Hg8AGzc\nuJHbb7+dsrIy6e6EX75Tr9eLy+Xio48+YsqUKQBERkae1bN/L8LNLv4BcDqdrF69mmnTprF7927A\nN7ddunTBbDaHZE+KNdDQ0MA999zDv//97wDlwWQyYbFYpLfFarUSFRWFTqfj0KFDgM+V3KpVq5Ac\nwqqqynd45MgRfvrpJ+Bn5dTpdPLTTz/x3XffMXHiRAC51kOFWGcOh4PKykrmzZvH8uXLAd+61+v1\nmM1mLrjgAgB69eoVtvNUc7ZqaGhoaDRLmr0F1dDQIH2x6enpmM1mvF4vubm5gE+LjYiIYNmyZdIM\nzs/Pp6ioiFtvvZWnn34agPHjx4clmO7xeFi8eDGffPKJtPLatWvHnXfeSUpKiowXfPXVV1itVoYN\nG9YkbpjTY49C83W73dJ9qqoqV1xxBV26dAlJDEXExG688Uby8vLwer1yLuLi4hg7dizDhg2jZ8+e\ngE+TtNlslJWVceTIEcBnsbjdbhITE6UbLSkp6azGKzT/f/zjH7z44os0NDRIzbZr1648++yz9OvX\nT1olOp2OmpoavF6v/D0xf0IzF5/7e7Vx/+D+XXfdRXFxMeBzFwLccMMNDB8+nNzcXJxOJwATJ04k\nLy8Pl8slrYHu3bv/7ucKN9h/83vi2QcOHGDu3LksXLhQrnHwzZFweYJP+3744YeJiIgI+ppSVZW6\nujoAxo0bx7Jly/B6vfL/m0wmMjIyuPLKK+W8xMfHs3v3boqKiuR45s+fz8SJE6WVFSwUReHIkSO8\n9957gM86yc7O5sYbb5Rr6siRI3z88cekpqZyyy23yHGHCq/Xy/HjxwEoKyvj8OHD2O12GXMuKirC\n5XKRmJgoz9IWLVqEzYJq9gIqNTWVFi1aAIEBaLF4UlJSsNlsDB06VGbh/PjjjxQWFlJVVcXcuXMB\n2LVrFzfddBM9evQIyQsXh1txcTHz5s2jU6dOchMMGDCAuLg4FEXhk08+AWDv3r1kZWWF9WWLQ0LE\n8fzvRu3atYuCggK8Xi/Dhw8HIDY2lt69e4fExeB2u2V85+DBg/Lw6N+/PwBPPvkkOTk5Ae/K6/VS\nX19PXl4e27ZtA36+v9W9e3d5f+tshdPkyZMBePnll1EUhejoaK6//noAXnrpJZKSkmTcAHzJJiUl\nJeTn58v458mTJ0lKSqJNmzbSNXm2rqKNGzcCcOzYMUwmE0888QSPPPIIABaLRX5Psfbuu+8+JkyY\ngKqqfPTRR4BP2P5eBehMv++v0IgMS4fDwQ8//AD4Aufl5eV4vV45rtTUVNq2bSuFOMDTTz9NUlJS\n0JUyVVUpKyvj5ptvBmDdunVyLOJcuPzyy3nsscfo2LGjVGZ1Oh1ZWVmsX7+evLw8ABYuXMgVV1xB\nu3btgrY3VVWlqKiIv//979LN//TTT9OmTRsMBoMU9I8//jgbN27k0UcfDblrD3xCUig/JSUlZGVl\nUVlZyZ49ewCfC11VVVJSUujbty8Q2kSp02n2Aur0hazT6aSgEjQ0NBAbGytf8rhx46iurmbfvn0y\nEPvBBx8wa9YsHA6HnGixcIOBGE9dXR25ublce+21MjtPLEi73c5XX30F+A63Sy65hISEhKCN4T+N\nr6GhAYAtW7bw9ddfs2HDBgoLCwHkAZKcnCzn8dprr8VgMKDX64OeYejxeKSQMZvNtGrVismTJzNs\n2DD5M/HuhUDYvHkzH3/8MatXr5aHcufOnenatSspKSlSmP3esXo8Hh5++GGZgKAoCh06dGDq1Klc\ndtllwM/v0Ov1Sgth+fLl/PDDD9TU1MiAtsFgoFWrVsTHx9O6deuzGo+YIxGfiIyMpH///jz22GO/\nuWbr6urkOhQCU1GU3yUMTh+rf9UR/3/X6XQ4nU4Zqzh58iTgm6cLL7wQ8GXT1tTUMG3aNLKzswHf\n1YVgCydFUdiwYQN33313wPc2mUzEx8dzxRVXAPDwww/Tpk0bLBaLXD8ej4eMjAw6duzIjh07AF8M\nu7y8XMYag0F1dTXPPvssBoOB1157DYC0tDT0ej2Kosh3vX79eqKjo3nooYdCrrgqisKJEyfYvHmz\nHM+xY8dYtGiRtKoURcFisTB27FhpFIQznt/sBdR/wul0smLFCvbu3cs111wD+LLUMjMzadmypVyI\nBw4cYPny5bzyyit8/PHHwNkHr8+E2HTt2rVj+PDh5OTkyENNuDhqa2vZv3+//Jt77703bO49f5fR\nZ599xvr16ykvL5f/PyoqivT0dOLi4sjIyACgtraWsrIyMjIypAsiWOOtqalh586dgC84m5ycTI8e\nPQIOYKfTyYYNG6TVcPjwYcB3CF5yySWALw22c+fOMtD9e/F6vcyZM4eZM2fKYPGFF17I8uXLiYuL\nk99XHNAul4sff/wRgHnz5smszJiYGABycnLo06cPOTk5cjxnMy6dTiffQ4sWLUhLSwN+WabKPyV4\nxowZgO8dCWvyXPm1KiJiPoTLsVevXrRr146RI0fSu3dv+XvvvfceNTU1XHzxxQBBde2JuViwYAEP\nPPAAlZWV8meJiYncdttt3HLLLdJdFRkZKbV/8XvikM7Ly2Pfvn2Ar7pEWlpaUMYpEjRuuOEGYmNj\nmTFjhrymID7f4/Hw//7f/5PjmjBhQlisp8bGRn766SeZ/JCSksK7777Lrl27pFJoNpvp0KEDY8eO\nbZKMaC1JQkNDQ0OjWXLeW1AnT57kiy++oKKiQmrVF1xwgUzBFRZUZmYmdXV1xMXFhUQ7EdpQREQE\nGRkZv0jI0Ol08s4D+LS5zMzMoLvOfu3zdDod8fHxAIwYMYK4uDicTqeM+fTt25eoqChMJpO0JMrL\ny9mzZw+HDh1i8ODBgM9SONcxi1RbkUTicDgoLy9n37590lLLz8/njjvu4NChQ/Id6nQ6YmJiuOSS\nS/jHP/4BQHZ2Nnq9/ndf1BUadHFxMdOmTaOxsVG6g5cuXUpCQkLA54nY06FDh1i6dCngu0wsLnUK\nSys5OZmsrCzi4uLOyVev0+mk1WGxWNi5cyebNm2iVatW8mdOp5PS0lK+++47AHk3TK/XS7dWKN0x\nkZGR3HrrrQA89NBDci6Eu3j//v18/vnnuFwurrvuuqCOR1VV8vPzAXjiiSew2+1ER0fLePXTTz/N\n6NGjA1zFZ7pEXVdXx6pVq1i2bJn87H79+pGUlHTOY/R6vfzrX/8CfO/m5ZdfpmXLlgFzoCgKx44d\nY82aNYDPzfb888+HpVKL0+mkVatW8rsWFxeTl5dHbW2tfL5w74UzVu7PeSugxAKrrKzk4MGD1NTU\nyIPjkksuQa/Xo9frZVbPjh075CXUcPh2T8+SU1WV3bt3y83brl07GUQPFr/1vXQ6nRTMo0ePZuTI\nkQEHq39sz796wsaNG1FVVWbSCVfWuY4zJiZGuqGqq6spLy/n4YcfDihr5HA45O+Cz/X2+uuv07Nn\nT3n4n+27FH9XUVFBaWkper2eUaNGAT73kL8LDXxxsN27d/PCCy9IwZqRkYHBYKCyslL+LDc3l6Sk\npKBUcxAuvqysLHbs2MErr7wi7/rFxsYSHR1NTEyMzNhzu93odDrS09NlnDUULmSxRkwmEzk5OQHP\n8c/se/TRRzl69Cg9e/aUrsBgXaIuLy/nhRdeAHyu6MTERB544AGZ+Xamu3tiX4rKKgBLlizhm2++\noby8XBZOHjp0aFDuj9XU1MgkkoyMDOlK8z8b6uvreeSRR+TPZsyYEZZLwl6vl+rq6oB7WUVFRXJ9\nifcUExPD1VdfHdR4/e/hvBVQgqSkJMxmMy6XSwqjU6dO4XQ6sdvtcrNUVlZSWlpKv379Qpq2aTAY\nsFqtOJ1OeYjq9XrcbjcvvfSSXIijRo0KSartbyE2rIiJnV59QfwjqK+vZ9++fWRnZ591AsKZ0Ol0\nREdH069fP8CX0VhXV0dNTU1AjMVisXDNNdfI6hKitFAw56y8vByPx4PFYpG1xgSicglAYWEhU6dO\n5ejRo1x00UUA9OnTh59++olDhw7JA61Tp05Bu4QqMs3efvttVq1ahcViYf369YAvftC+fXscDodM\nUMjPz8dsNjNu3DiZfBOK8lkej0d6KM4kAMUVgO3bt2OxWJg2bZoskxOs8bzzzjtSIbVarfzzn/9k\nxIgRv7m3xftsaGjgnXfeAXzx2OPHjxMdHS2FbUZGRlAOZIvFIpWwgoICFi5cSM+ePWWSjdPp5Isv\nvuDHH3+Ult/QoUPDciaIuaitrWXRokUAfP3111JYiTEMHTqUjh07NllFkvNWQIkJy8nJIS8vj/nz\n59OjRw/AF+TU6/WyjAn4XIEFBQUcPXpUBgBDIaiEK62hoSGgRtqCBQsoKCiQgcbRo0c3ab004bIS\nY/B3R4l6eF988QVer5err746KC4PgbgDJTLfnE6n1G79XaU9evTgnnvukW6tULyvyy+/nJYtW1JU\nVCSTHyZMmEBMTAyqqsrD/8UXX2T79u20bdtWZho2NjZSUFCAzWaTSQwDBgwI2mYWc5GSkiJdZFdd\ndRXwcybkyZMn5VUKRVGIi4vjxhtvDLrG659e39jYSGRk5BnXr8fjkYKjsbGRzp0707lz56AecIqi\nMGfOHOx2OwAdO3bkoosu+lWXqn+ptNraWl588UVmz54tx6jX62nbti1t2rQBCFodz8jISFkNYs6c\nOfzf//0fL730khxnTk4OhYWFOJ1OXnrpJSC8pakiIiJwOBzynMrNzaWkpASPxyPHeNtttzVpuTgt\nSUJDQ0NDo1ly3lpQAr1eT0xMjPQ9++N/I768vByHw8GiRYuYOnVqSMdkMpmIiIiQGmdtbS07d+6k\nsbFRujqExt0UCD+8f7xJzJXT6ZTp34cOHeLPf/4zV111VVAv54kS/sK14Ha70ev1WCwW6erIzMwk\nPT2d/Px8ea8mFFXBo6Ojuemmm5gyZYp0n40bN44bbriBwsJCFi9eDPiC3Iqi0LFjR2nJLV++nN27\ndxMTE8OECRMAXwWMUFjG4jP9tVlVVcnLy5Op7qqqkp2dTdu2bUNSpUG4O71er6yecXoiiaIorFix\nQv73+PHjg27NCc/If1sgWKzxo0eP8t577zFnzhxZnQZ8lkROTg4jR44ECJobWa/Xy8StXr16yVqJ\nYjyHDh3ixhtvpE2bNowePfo/fo9g09DQwJEjR+Saqq+vx+12B1wf8L8y0BSc9wLqt/CvwHvw4EEZ\ndwl1wE/ET0SwMyIiguHDhzNr1ix5JyMc1YlPx//uh4g3CXee2+2mvLycbdu2yXs11113XdCFkwii\nr1ixQsYHRfzn/vvvl8Vz165dK4tmikue/fv3D/otdp1Ox4MPPsjBgwelMFq7di1r1qwJEOCxsbEy\nOePbb78FYM+ePURHR3PrrbfKAyac7hCv18trr70mBYfFYuHNN98MSZDdfy9FRESc8RAXmapCwYmP\nj2fChAlBd1sZDAaSk5Ola6qgoID333+f8ePHyztGQnj6K1zTp09n1apVMlYNPqWnZcuWXH755QEV\na4KF+Cxx5kRERMg1tXTpUjweD99++21YqzOAb34SEhJo27atfPYPP/yA3W5Hp9PJdjbhSNj4Lf7Q\nAkpVVXn5rqGhAavVytixY8Pe0FCv12O324mIiJBaWrg0JdE3yF/bFOn3DQ0NMg60d+9eVqxYQXJy\nsrQGsrKygrZxxPOdTqe8FCmE0S233MI999wTsHmzs7OZPXs2mzZtYvv27YCvHl4oKqtHRUXx4Ycf\nyurzn376KT/99BNOp1MeLCL2tH37dlkGJioqigEDBnDbbbeFfSOrqkpJSQkHDhyQQrFv37706NEj\nZGtLrAVx+J+p6sSMGTNkbGjIkCFByfo8Hb1ezzPPPCNLU9XX1/PGG2/wj3/8Qz7PbDZjMpmwWq1U\nVFQAvkouwoIRaygyMpI777yT0aNHh01pFErhjBkzGDx4sFRaw4mqqkRERGCxWGSK+8GDB1EUhZYt\nW0oB1dT8oQWUoii88cYbgO9gTElJ4aGHHgp7RoqqqixatIjGxkaZyhpqASWsk+rqaux2O0lJSdK9\n6PF45B2VXbt2AT5tt0ePHkyaNCkkGWACg8FAREQECQkJcmNedtllWCyWgKwwi8WCy+WivLxc3u9x\nuVwhmzeTySTdxDfccAM2m00Kd/BZK0eOHGHt2rXSPdS+fXsmTpwoe0L9Gr91P+1s8Xq9PPXUU9jt\ndnl94LnnnguZ8vVrFSX8aWho4NNPP5X//4knngjJXtPpdNx8883ymsZrr73G/v37aWxslPcMxZUJ\n/6sU4ud6vV4Ko2HDhnHdddcFKD6h3psiPd7pdDJ58uQmKxbtcrlYt24d33//PeC7j2g0Gvnggw+a\n3HISaEkSGhoaGhrNkj+0BVVXVydTXhVFoX///kGtv/ffoigKhw4dwmKxyAKUodCqBV6vV7rFZs+e\nTUVFBd26daNLly4A7Ny5kxUrVlBaWio1yQceeIBhw4aFLD7nH+TPyMggPT2d0tJSAL799ltSU1NJ\nSkqS7qGvvvqKBQsWcOrUqYAmj6HUbv1T3K1Wa4Bb1O12c/z4caxWq0y5HzhwIO3bt/+PYwrmmIUL\ntKioiCVLlqCqqlzT4Wwkd6ZxFRYWYrPZZHWS0++VBROLxSLjfoMHD2bJkiV88skn0qUvCufq9Xpp\nVXo8Hun6E+167rrrLlq0aPG7K5GcLU6nk1mzZgG++1YitT3cKIrCvn37mD17tozlGQwGRo4cKbsZ\nNAf+sAJKVVXmzp0r/b0Wi4W//vWvYY8/ge9ws1gsJCYmhrQisOin5Ha7ZbygsLCQXbt2sXLlSvlM\nkTHXq1cvnn/+ecB3mIRjg+p0OpKTkxkwYACffvopAFu3bmXHjh3ExcXJ8jUFBQU4HA6ysrJkTCyc\nbofTq+YbDAZSUlJIT0+XMZ/BgweHpceYQLSUABg5cqR0yUyaNAlAunDDiX8H27lz55KcnMzf/vY3\nILjdAs6EcI3Fx8czZswYrr32Wqng1NXVUVFRgd1ul+vm5MmTGI1GPB6PVBRzcnLCWiXhrbfekmOc\nOXNmk12A9Xq9zJ8/X2angi+zeObMmU167+l0/rACqq6ujg8++ECmehuNRqxWa5MsCLvdTmVlZUAj\nt2BaUP6t2kWmosh8e/TRR3nvvfc4efKkLO/St29fbrnlFjp16hR2ga3T6TCZTPz5z3+WB8MXX3zB\n9u3bsdls8n0JgfDggw/K2/hN8e78hXpycjJjxoyR48jKygrrmDweDzNnzgR8ioeiKERFRTF+/Hg5\nxnDin3q+d+9ejhw5QkZGhrROwmnNiXUl4qcJCQm/qOjun8XqX1UlXIgLxiJm+Kc//SlszxaIOTh5\n8iT79+8PqHjzxBNPhCSp5Vz4QwooVVXZuHGjbMQFvr5BotJEuLFarTQ0NGCxWGT/pU6dOgVtA58e\nwBZdhsGXgCB6GjVl5Qp/9Ho9CQkJsj1Kv379+P777zl48KDsQ5OUlMTo0aMZMGBAk1i9p6PT6bBa\nrWRlZQV0aT29/UUoqa+vZ/Xq1YBvDkWlgmBW+fi9CA+Fy+UiOTmZmJiYkKRrnw1nSoOHplF0wPf+\nCgsL5X5sCotXzIHVaqWurg6r1cqQIUMAmDRpUpPNza/RvEajoaGhoaHx//OHtKDAVxw2JSVFdrV9\n8cUXmzR1slevXthstrBrl02txf4a4jIz+GonijjKmX6vOSCKo0ZEREgt0+l0YjKZZLsP8XuhQDQm\nrKmpAXzxuOHDh/P44483iWtPIOIXUVFRWK1WrrrqqiarfN3cWbZsGUajkYEDBwJNu7YTExOlNS72\nYXPZa/78YQXU0KFD6dChgyw0mp6e3mQvwGq10rVrV+68886gt9j4o9AcN8fpGAwGoqKipIvPaDTK\n7C//ZJxQoNPpaNmyJQsWLAB8lzwnTJgQ0BqkKRBV3HNzc7n77rtJT09vdm6i5oCqqphMJrp168bQ\noUOBpl3zwmXd3NGd3reoCWk2A9HQ0Pj9+CcfaGj4cdaSWBNQGhoaGhqh5KwFlKbuaGhoaGg0SzQB\npaGhoaHRLNEElIaGhoZGs0QTUBoaGhoazZI/bJq5hoaGhsZ/RiTKiTJp0PRVNwSaBaWhoaGh0SzR\nLKhmQDjruWk0f0QhYf8rINra0AgmotDv/PnzZRHpiooKKisrufjii0lOTgage/fuxMXFNdk4tXtQ\nQca/wrN/h05x4IjSME6nk6NHj1JQUCA7tPbt25eWLVv+4UvFKIqC0+mUhXOLi4spLi7GZrPJSs/9\n+vWjQ4cORERE/M8dzmKdiHXh9Xoxm80BbRDcbjdRUVHo9Xo5P/9r8/RHwOVySQEBEB0dHdJ2F+L8\nqampYe7cuRw9elRWM4+NjWXbtm2UlJRQX18PwJw5c7jgggvOdW2d9R//oSwo8aLXrVvH559/zt69\ne2X5+NzcXMaNG0f37t1lFeFgbmghlBobG8nPzyc6OprWrVsDyAVwevvpEydOMGvWLBwOBwAlJSXc\nd999QRvTb6GqKjabjT179gCwefNmEhMTGTRoEBkZGQHjDjaKolBVVcXUqVMB3/tyOBwB7VA++OAD\nunXrxp133kmfPn1COp7fg8Ph4N133wV8jRavuOIKJk2aJAVrMH321dXVgO/d2O12dDqdrOV46tQp\nsrOzadWqFSkpKYCvbb0Q6Jqw+pnm6qHwer0UFBQwc+ZMWWPxscceo127diEbq38187Zt25Kbmyt7\n1BmNRtq2bcvUqVOpqqoCfG1UunTp0mRz1/Q7PgioqsqJEyf4y1/+Avg6xrrdbgwGg6yNtnv3blau\nXMkrr7zClVdeCQT3wBN9jBYuXMjGjRvp0qULEydO/MXv+S+Qqqoqtm7disfjAXxCNBytJZxOJ1u3\nbuW1114jLy9P/kxRFNq1aycbCXbs2DEkC1Ov18s6dgDJyckkJibSqlUr+bPDhw+zevVqtm3bxuzZ\ns4HwNVWdr8HiAAAgAElEQVT8NSorK5k4cSLr168HfOsuISGB6667Tio9wRBQwto+ePAgADt27KC0\ntJSUlBS5niMjI6mpqaG0tJSjR48CPuXoiiuuoHv37rLOWnM7lMOFqqrs27eP66+/nqKiIsA3F/Hx\n8dxxxx08/fTTQOibKv4WTqeThQsXsnjxYk6dOgX4vCht2rQJmRXl3zV60KBBQOCazc3NZevWrXzy\nyScALFq0iNGjRzdJaxDQkiQ0NDQ0NJop570FpaoqhYWFXH755bIdNkBcXBwjRoyQwb7du3fT0NDA\ngQMHZDVhUYk6GGMoKCgAfN1h8/PzsVqt/Kf4XnFxMSdPnpTaSdu2bUOW1qmqqjTbn3vuOZYsWYKi\nKMTGxgI+K6a8vJyysjKmTZsG+Cpmh0KT0+l0JCUlcddddwE+V9nll19OamqqbIe9ePFijhw5QnFx\nMVOmTAHgo48+CrubT7zDlStXcv311+N2u6XbODo6GqPRiMPhCOp7U1WVhoYGtmzZAkBRUREGg4Gs\nrCwuvvhiwNfeoqysjIKCArZv3w7AmjVr+OCDD7j11lt54YUXAMLe7FFRFOmyNhgMv3i+iK8ZDIaQ\nrHURY1mwYAFjxoyR3gnwrbuKigpee+01Fi1aBMAPP/xAUlJS2CxNVVVxOp2AzzKeM2cOJSUlcp3V\n1NSEJbVbtLvx7+ytqioul4vCwkI5b8eOHcNmszWZBXXeC6iamhoGDBgg+z8B3Hbbbdx///3Ex8fL\nBbtr1y7mzZsXcAgGq9y8qqocOXIEgH379lFbW4uiKAGdV0/H7XazcOFCPB6PPHT79esXlPGcaXy1\ntbUMHz4cgPz8fJKTkxk2bBg33XQT4POHT5s2jXXr1smDUbhJg41Op8NoNMpY15gxY4iLiyMuLk7O\n2ZAhQ1i/fj1Op1NmEYU7oUdVVXn4X3fddbjdboYMGSI7kK5fvx6Hw0FsbGzQDzidTscFF1wA+Nq7\nd+jQgSuuuEK2jzGbzVJAbtiwAfDtBdEW/p577gH4RdvzUOJwOHj77bfZu3cvAO3bt+fSSy8lPT2d\nY8eOAT6hlZ6eTosWLQL6EAVLUfz8888BuOWWW/B4POj1ehm3GzRoEMXFxezcuZN9+/YBMHjwYNat\nW0dMTEzIhZQQAHV1dYAvFFFXV4fL5ZL7rKGhIaRj8Of0edfpdJhMJtq0acOPP/4I+BT9pmwDf94K\nKKGFXHnllVRVVZGens68efMAX2qkyWRCp9PJAy8jI4Pk5GRcLpf8mb/2cC54vV6WL18O+BIfRHO5\n3zrcKysr2b9/PzqdTjZVzMnJCckmURSFt956S8Y09Ho9I0eO5O6776Zly5aAL+gurKna2lrAJ6BC\n1TNGr9cTHx8P+HooORwOFEWRCkVERAQdOnQgLi6OO+64Awh/kkR+fj5XXXWV/O9p06YxduxY8vPz\nAZ9/vkuXLiQnJwf1vQkBnpqaCkDXrl0ZMWIEOTk5AYe60WikTZs28h2K+WtoaJAxjczMzJAevEJp\nOHHiBAMHDqSoqEj2iBo0aBButxtFUaRAaNu2Lddeey0pKSnyXRsMhqDsxZMnT3LbbbcB4PF40Ol0\nPPvsszz++OOAz5psbGzk+eef5/333wfg6NGjTJs2jSeffDKk8ShVVfF4PAGKa3V1NfX19SiKEtC8\ns6kQF3W3bt0qY+rx8fFht8L9OS8FlKIoTJ8+HYDt27eTmJjIsmXLyMnJAX4O+vmnde/YsYP9+/dz\n6tQpmUwRrI2rqqoMVLtcLiwWC7m5uQHZe/5jBzhy5AhOpxO9Xk+PHj0AQmZG19TU8OWXX8psneHD\nh/PII4+QmpoqhaiiKKSkpAS4Zerr64mOjg7ZASeeo6oqDoeD/Px8eZAdPHiQmJgY7rrrLjp06ACE\nN+Dv8Xi48cYbpbX95JNPMmHCBBwOBzNmzABgz549PPnkk0HfwKqqYjQapeC5+uqr5fWD090/er1e\nKhRCWERGRsrGmKGcM0VRWLJkCYCcK7PZzKhRowAYNmwYtbW1LFmyhF27dgE+q6pNmzYBafPBUhKH\nDx8uG0fqdDpuvvlmnnrqqYD3ExkZyS233MLcuXMBn6L48ccfc+ONN9KuXbugjUfgb/WLzstCGMXG\nxmKxWNDpdNL7c+mllzZpYsv+/fvZtWuXPKcSExObtJqEliShoaGhodEsOS8tKJvNxhdffAH4fPHv\nvvsuOTk5AZLe6/XidDrZtGkT4HPPFBYWMnbsWHlnJVgoikJlZSXgc1dER0dz6aWX/kLzED5oQMaf\n9Ho9l112mfzbYCPiT2lpaVKzve+++0hKSgoYn9FoxGQyYTabpavP7XaH5Q6JqqqUlpbyz3/+U/rg\nc3JyuPvuu2ndunVILy7+Grt27eLAgQPSzXbvvfeiqirffPMN8+fPB3xrr2/fviGJP+n1emnx6vV6\n6SISCDfyrl272LFjh/y5Xq+nb9++ctyhwuv1Mnv2bJ544gnAl+KekpLCc889Jz0UXq+Xffv2kZ+f\nL9f9n/70JxISEoKWoCRYt26dtL4BWrZsyfTp03/hFhZxFnHvSJwTYs0HG/8EBPFehUUnEkoMBoP0\nEjRlvMfj8fDNN98EuCEvvPDCJhsPnKcCqry8XMYvLr74YgYMGAD87D6rra1lxowZbNq0iZ9++gnw\nBR8jIiJITk4OicmamZkJIDdJeXm5PFDEZlQURV4m3r9/P+Dz8Q4ePBgIjRAQi2348OHcfPPNAFI4\n+ZfT8Xg88v6YcDeIWAKE9rKjx+Phk08+Ydu2bTIxoCmFk6IovPfee5jNZplEYjKZqKio4JVXXpHx\nz2effTZkcQv/+GltbS1GoxGPxyOfffz4cVasWMHevXtlxYnIyEgsFgvPPfdcSOMpLpeLNWvWMHny\nZClEb7jhBp555hliY2MDMtJWr16NzWaTbq2kpKSgCydVVVm9ejVer1eul7vuuovIyMhfPEdRFF54\n4YUAV2B2dnbIYq3+nJ6UsHnzZnkeCAW3vr7+F8pjuHC5XOzatSsgK7VNmzZN6nI87wSUqqryoif4\nfKQVFRU4HA4WL14M+A6OmpoadDqdzAATh22LFi3OGBs6F0wmE926dQN8aauNjY1MmTJFBtN79epF\nWloabrdb+uKjoqKIj49nyJAhQbfo/PF6vcTHxzNy5EiSkpKAQEtNHCY7d+5k+/btqKoqs9RCGX/y\nf/b69euZP38+Ho9HKh4iA9O/lE+4UFWVTp060adPH3r27An4qny8//77VFdXy1jF3XffHbKxeb1e\nGf8qKytj27ZtHD58WFpLJSUlNDY2YjAY5JylpqZy0UUXyezIYCMO9dmzZ/PWW2+h0+lk5ZM77rgD\nq9WKx+ORVvDhw4c5cOAALpdLauKdOnUKydgiIyMxm80yuC/21Olp1MeOHeOHH36QyqzRaOTaa68N\n+Vr3VwbF/FRWVuLxeAJSz7du3UpycjKRkZFhE1L+mc7l5eWoqiotOU1AnQVWq1Vql0VFRTz33HMU\nFxfLLLX6+npMJhPp6enSfXb48GEqKytDkpGi1+tltteMGTMoLy9n48aNskqDMO31er1MhIiJiUFR\nlJC4iE7H5XKRmZn5C2vEf2NMnToVt9tN9+7dZSaU2WwO6diE9jhlyhROnDhBRESEFOqvv/46iqIw\natQoevfuLccTLkaNGkVsbKx0/Xz++ecsXLgwIEEnlNlNqqrKRAeRBHHkyBGOHz8OILO/4GfBYTab\nKSgokO80mAh3HcBLL72EzWajc+fO9O3bF4CqqipsNpsUpgB2ux2r1YpOp5MuLJEUEEx0Oh2jR4/m\njTfekOWh3nzzTdq1a8fw4cOlQlpfX89TTz1FfX29HENMTAxXXXVVyIWBf0sLcU/MarVisVjweDxy\n3D/++CMXXnghbdu2DVuNxeLiYgBmzZpFUlISXbp0kQLK5XIFLdv5bNCSJDQ0NDQ0miXnpQUVExMj\ntemCggKKiooCLrx26dKFSZMmMWTIEKlNTp48maqqKkpKSoI+Hp1OR3p6OuCrX3fq1CkaGxvlsxVF\nkVqIiBecPHkSo9HIwYMHf1H9PBgIjc3r9f6iSK34/16vl3//+98AbNu2DYvFwttvv01CQoL8XqFC\nURRZI81msxEZGUlaWpqs/PHdd99x4sQJ3n33XXk3ZNGiRWG5J6LX68nKyuKaa66R62Xz5s1UVFTQ\nokULWc0hVIhkGuERUBSFTp06ERsbK2OXFRUV7N69m1OnTslYp8PhQFVVDh8+LGOiwXqHbrdbutCT\nkpKIiorCaDQyefJkOR6j0Uh6ejpt2rQBfPuwsLCQbt26cf311wd8v2CvraysLIYMGSIv6h47doxJ\nkybx2muvcemllwK+9fPjjz/K5CTwpb1nZ2eHtZKEcN0aDAa5P8V5NnDgQNLS0sJW8FdVVV5++WXA\nd+8vKyuL1NRUGYc+cOAALVq0CLgWEM742HknoEQWjnC9ZGVlkZiYSOfOnWUR2NzcXBn0FJUADh06\nhF6vD9nFRREE7tu3LzU1NZw4cUJmColLqP6xM6/Xi9vtZsGCBYwbNw7wxaqC9fKFgHK73Zw4cYKE\nhATpElJVFbfbTV5eHp999pkcY+vWrcnNzQ3LxlAURbqucnJy6NixI5MmTZLz43A4+OSTT1i0aJGs\n0jFw4EAWL15Mly5dQjo2nU6HwWAgKipKJgEUFxcTGxvLpZdeGvKKFqqqsmXLFj788EPAp5Ddeuut\ndOvWTbqSDQYDn3/+OStXrpSut6qqKk6cOMHf//53WWBXFB4913dqMBi4+uqrAd+l7ri4OCwWCwcO\nHACgrq6Ozp07M2LECOmSzc/P5/jx46SkpMgDzz8WE8x1Zjab+eCDD6S7c+nSpdTX1/Pxxx+zatUq\nwPcOT6/uEi7Xnvh3RVFkfMx/LELx6tWrF1FRUWFLDvJ4PLJqhMvlorq6mlatWsl52bRpE3v27MFi\nschwyQUXXBC28Z13Agp8vn9Rx23v3r1ceeWVpKamBtyyF4ewqPBQVFREbm4uvXv3DskBLIKzSUlJ\n9OrVC6PRKBegTqejU6dOAaV8Zs+ezYYNGygrK5NlaZYtW0Z8fHxQx3fs2DEWLFhATEwMHTt2BJCJ\nI1u2bJGHv06n489//nPYbo3r9XqpRPTq1Yu//vWvREVFBXz3fv36sWvXLiZMmAD4rOVRo0aRl5cn\nEwNChRjH22+/Dfgs3pSUFHr27Cmt4FDEU8B3cG3dulUerAaDgYyMDMaPHy+/t16vZ8SIEWRmZrJu\n3TrAFyerrKxk586d3HvvvQDMnDlTVkE/l5iG0WiUisHzzz8vFS7RN0gk45hMJpkEMH/+fI4fP05u\nbq70MIikl1DMW2RkpLSgCgoK+Oyzz+T4wHcAV1RUUFdXJ63OoqIinE6nPDtCjU6nkzEo4fkxGAwy\nGSc9PT2smasOh0Mqio2NjdhsNqqqqqQyazabKS8vZ9WqVRw6dAjwxSCDfU79GuetgBLVF0R/J//7\nDiKlu6KiQt4Yd7lc5OTkBGgHwUJRFMrLywHfBuzTpw+ZmZkyyN2iRQsSExMDFt7gwYMZM2YMu3fv\nlokBK1euZNSoUUEt6VNVVUVZWRmZmZlSiCYkJOD1emnTpk1ANlO/fv3CGgwVdeWGDh1KRETEL96L\n0WjkoosuYsWKFYDvDk1JSQlvvPEGL774IhA6N6Sqquzfv1/et4uNjWXgwIGy4CgQsiKjooyRqNnm\ndrv5v//7P2pra3nggQcA3zts3bo16enpsoZjeXk5ixcvxuVySatq+fLlDBkyJKCVydmOSSgv/utT\nWJgCcacNYMWKFSiKwvjx42XQPdQWi/j8du3a8fTTT+NwOKRSWFlZyRtvvEFlZaWcW4/HE1DmK9j4\nZxAKa0rs9+rqarxeLxaLhdGjRwOhU3p+Df8rAA0NDXi9XhoaGqQQbdmyJR6PB4vFIhuMnjhxgpiY\nmLCUHtOSJDQ0NDQ0miXnpQUFP2tKwiqAQG3a4/Ewffp0adlYrVYuu+yykJjyiqLI9HGDwUBycjJp\naWnyNr8IKPtrsK1ateK1115j3Lhx0sR+7733uPzyy4PivhJzkZqaStu2bfnTn/4kXTRWq1W2XRc+\ne0VRpBsmHPhr5I2NjbKi8+nao2jNAXDrrbfy0ksvsXTpUp5//nkgdKneXq+XTz/9VFrBffr04aqr\nrmLz5s1Sk+zYsWNILAKDwcCECRPkNYU1a9ZgMpmw2+2ypYzonmuxWOQYRo0aRU1NjUykAN/F8WHD\nhgV8/rnGgH7r77xeL9999x3gs94HDBhAnz59muTiqbjWISyotLQ0xo8fT35+vkyFt1gsHDlyhKys\nrJDfgxLuvWXLlgG+WJ6qqrRu3Vq6+MKdzp2SkiLvPS5cuBCn04nVapX3RsvLy0lNTWXQoEGyG0I4\nizaftwJKUFJSQk5OTsDh7/F4mDt3Lh9++KEUYMOHD2fMmDEh2ygiC0e4hiIjI+XB6n8HQixAvV5P\nRkYGHo9HbqD6+npsNhtxcXG/KDB7tuNOTU1l+PDhpKSkyEVnMBikgBJ3kfzHGS7E83bs2EF+fj5X\nXnmldLX4v0/hhiwsLMRgMNC6dWv5XkMloGpra9mxYwddu3YFoH///phMJkpLS2WmYajmS6fTkZCQ\nwKxZswDfQSbuiQnXi9vtloevWD9ms5mEhAQ8Ho9UmNLS0qQS4o/oyRRsVFWViRPx8fE8/vjjTVoN\nWyS8gO9gTUlJoV27djJ5ymazsX//fi699NKQ37NTVZXjx4+zcuVKAFnF/IUXXgio2hJOjEajXM8u\nl0tWgRduSIvFQlpaGn/5y1/kXoiPjw+bwnHeCiixKTdu3MihQ4cYNmyYnLQPPviAyZMnU19fL2/9\nT58+PWR1rgwGgzxYc3NzycvLo6ioSKbbiguvRqMxwPI7fPgw8HNfKpvNxqZNm/jLX/4SUOn5XLQq\ni8VCTEyMbD8CP2cTbdy4Uf6e0WgMmR/+TOh0Ovm96+rq+Prrr9m2bZtsQz1o0CD0ej0ul4s5c+YA\nviaG4gJ2KLU4VVXJz88nPj5eVgiJj4/n2LFjREREyMM/1Bq3mJ+WLVuSlpYWIIzEs71er8wW3b17\nN/v27cNut0tPQVlZWUANP4F/WaBgoiiKFEi9e/emS5cuTVqJAAhQCi0WS4DAVFX1jNl9oUBRFJk4\nIsbTv39/hg4d2qRzJCqPmM1m+f6E0GrXrh19+/alT58+Z4xBhprzVkCJzVVbW8uqVauYPXu2rLtX\nUlKCy+UiKSmJL7/8EvC51EK5CIT21bt3b44fP47dbufkyZOAL6iYnJxMQkKCTDEVC+Hiiy+W405L\nS6Nz586/+OyzGbfQ7vV6PUlJSQF13Orr66mqqmLnzp1yHnv27CkXZbgQWuO4cePYvn07S5YsYe3a\ntYCvIoder6e6ulq6tRRFITs7m0cffTSk2q5OpyMtLY3GxkaZueR2u6moqCAyMlLegwqn20ooOOJw\nEO/XYDBIN6SiKNhsNgwGg7QwFUX5RcNH/3s3oRinSJzo2rVr2LLj/ht0Oh1RUVE0NjZKS9RsNnPs\n2DFppYeavXv3yn0YFxfHuHHjmqxbrUCkj69evZqEhAQyMzPlOXTZZZfJBK+mEKJakoSGhoaGRrPk\nvLSg/DXArl278u2337Jnzx5558FgMJCZmcnSpUtp3769/JtQjkdo0xEREVx77bXYbDapsZpMJll3\nS/ye0Wikb9++TJ8+XSZJgM/cDkZsw9+tERERgcvlkl1WV6xYwbZt2zhw4IDUcPv06RN2bVfMRXp6\nOh9++CH79+9n9erVgK9yQ0NDAx07dpRu2k6dOjFmzJiQFUP1Jzk5mUGDBrFlyxbAd2csMjKS3r17\nS4sl3DE7f8T7Fe1dAB599FEURWH58uWyksQjjzxCYmJiQMwp1HtBuGm7d+/epM3uTken0+FyuaSH\nBXxxlyNHjoTNOujevbuM0WVnZzNq1KgmnSOdTifd2F988YUsxi3G1NTvT9eUm+w0ftdA/CsDf/PN\nN7z77rvybsPAgQN58sknSU1NbXL/d1PhXzXC5XJht9ulAP/hhx9Yvnw5hw4dkpd3X3rppYAClf/r\nKIpCXV2dvMi8cuVK4uLiSE5ODrhUebo/Phz9s34LUYxUuLDCdV9F4HK5qKqqAnx3x06/fN2UKIpC\nWVkZw4cPl0kARqORm266ibfeeivgon8o8Hq9HDt2TBaGTU1NpWXLls1mfkLIWX/B81ZAafx3+F8Q\n9C9/VFVVRWVlJW3btgV8Fy6bWltqjvj3yxL1C/17BzXHw8V/T4d7fIqiSOvEYDA0aQbf6Yjq/d99\n9x3PPPMM4Et+mTlzJu3bt2+S3mP/I2gCSkNDo+nx7/xrNBqbpQDXCDtnvQg0lVlDQ0NDo1miWVAa\nGhpBpanjcBrNjrNeCOdlFp+GhkbzRRNMGsFCc/FpaGhoaDRLNAGloaGhodEs0QSUhoaGhkazRBNQ\nGhoaGhrNEi1J4n8AUb1cXKAUpaL+m0rp/i1CNDQ0gouqqrjdbnlJOFQtUM7XPawJqBDgX70Bwl/P\nyuv1ymfW19djt9sxGAyyBI0oP5OYmCjL4JhMJlmHy5+mXtj+Fz/dbrcsR+Nfzbupx9icaYrDSbwb\nUW5Lq9DgQyiJQlFcs2YNCxYsIDExkccffxyAxMTEoL+vYH+eON/8P1f8d7CfpQmoICHaG5SVlTF/\n/nw2btzIsWPHAF8/n4SEBHr06EGfPn0AyMnJwWq1BnTaDdYLVlWVvXv3Ar6WFkajkbi4ONl2QbTb\nKC8vl1104+LisFgseL3egEKz/p8pxhguGhsb2bVrl+xAKmoJejwe2TxtxIgRUtCe7dgaGxvR6/Xy\nQBU1HcXcwe+zOpsKRVFwu93Y7XbZI8pkMlFdXc0FF1wQ9HV2JvwVCq/Xi9FoPK81+GAhvBhlZWUs\nWLAA8J0VMTExZGZmytY8cXFxIVW6fmsf+yvWv6VUq6pKfX09brebdevWAb76p6JzeDDRYlAaGhoa\nGs2SP7wF5e9u0Ov1IdNMhHaampqK3W7H5XLJzpmHDx9Gp9OxadMmNm3aBECPHj2wWq3o9Xq6dOkC\n+NpJJCQkBGWMubm5AGd0j9ntdk6cOIHb7ZYt3zt06IBerw/QgM5kyoca8b5qamp47LHHWLNmjWxv\n0aNHDyIiIjh16pRsWdCpUyeSkpJ+9xjFc4Q71Ol0UlFRAcC6deuw2+107dqV1NRUwNdM0mAwBLir\n/N24p1dkCeVa80cUQAU4fvw4W7ZsYfv27ZSUlAC+hnzjx4/H4XDIIreh0tCF9SSqdQsLXTS8CxWK\nouDxePB4PNjtdsDXSVp4Nfy/t8FgCLC2w7W2RXX8vXv3Sku9V69eZGZmkp2dLefH4XAQGRkZsnH9\nN5/7a78jxm2z2Vi1ahWFhYXSUr/kkkuwWq1BX1t/KAElfLsnTpzg9ddfZ968ebIHEvjK/3/66aeM\nGDECCG5sSHyW1WrlrrvuYuXKlbItg9vtJi0tLaBi8u7du1m6dCn79u2TB8zIkSOZOnXqOXc7NRgM\n8sAULhaz2Sw3rF6vp23btthsNmJiYgDf4mtoaCAiIkIuRJPJJHvDhCuOZrPZAHj88cfZtGkTOTk5\n3HfffYCvW7HD4WDp0qVSkCYkJEhX0u9BCGadTofX66Wuro6GhgYAkpKSqKioYO3atXTq1Ek+Jyoq\nSrpq4Gfh4HK5pIumvr4ei8VCWlqadKmG0l1jt9t5+eWXAfj+++9xu93yMATIzMxk0aJFnDhxQq7H\n1q1bh6yLqxgTwJw5c6isrOTll1+mVatWIXke+Pb92rVr2bJli+yA7PF4MJlMGAwG2rRpA/jiO2az\nmaioKNkVOSsrK6TtSMQ+dLlcVFZWkp2dLbvVpqeny70qzq7GxkapuDYFv1b6TlEUeU41NDSQlpZG\naWkp2dnZgC+ubTKZtBjUmfB4PGzatIkHH3wQgIKCAurq6gIaG4psmY8//li2OA7FJtXr9SQnJ3PN\nNdf8wp/r//K7detGUVER69atkwfj6tWrsdvt0mI4W84UY9DpdAExiMTEROx2u0ycUBSF+Ph48vPz\n5ThbtWqFyWQiKipKtmcP5WZ2OBxMnToV8M1Fz549efXVV2nZsiXgE7wulwuLxcJPP/0E+Kyqs9nM\n4t0LgZOYmCi/d2FhITabjQMHDkgr2OPxcOWVV8o4nfiZsEI///xzAP79739TV1fHI488wsSJEwFC\n0ghSVVVsNhsPPfSQjNFZLBb69+/P8OHDZZuL/Px8UlJSOHDgAKWlpQDcdttt57T2z2Q5qqpKQ0MD\nhYWFPPzwwwDk5eVhNBoZPXo0o0aNAoIrrMWzq6qqmDFjBtu2bZMHveiHdbry4vV68Xg8xMbGAjB1\n6lTGjRsXEoFweqKUxWIhISGB+Ph44GdF0u12yzXlHwMOF/4tZVwuF6qqBvTGEq1mhGJWX19PVFQU\nXbt2JS0tTX6/UIz7vBdQDQ0NPP7448yePVtOUExMDJdddhldunSRGvmyZcsoKyvj6NGj8tDJyckJ\nyaTqdLpf7YMjFkNERAR2ux273S7dcPX19SHTnBRFkZt2586dLFmyhMrKSrkQ+/fvT9euXTGbzbLD\n79GjR1EUhdzc3JBp3AKPx8OXX37JJ598AviE4yuvvEJ6enrAnNTU1PDDDz9Ii2X06NFnNWf+Lh5h\nPR49ehSAHTt2sHDhQk6ePCkFc2NjI/369SMtLS0g8zEqKoq4uDjGjx8P+ATUsWPH2LJlC3feeedZ\nzsavI6zgffv28eijj3Lw4EGuuuoqwCd4OnbsiMVikcqIx+OhtLSUZcuWUVhYCEDfvn1p0aLFWY9B\nHPIOh0OulcLCQoqLi5k5cyZ5eXkAOJ1ODAYDX331FSNHjgRCY006nU6OHz9OfX291PLFgSreLyAF\ngaIo0oKeMmUKY8aMCcm+0+l0AUIqOTn5F9aR1+vFbrfL8Zyr9+T34u8i3rdvH4sWLeKCCy6gf//+\ngP3GphQAACAASURBVO87WCwWLBaLXPdpaWk4nU5atGghBX0orCfQkiQ0NDQ0NJop560FJYJzF110\nEaWlpZjNZsaNGwf44hcZGRmAL1ALvpThefPmERMTE+CWCLc5LTTgL7/8kgULFuDxeKRGNWjQIKKj\no8/6s8X3Of17ifRj0b78nXfeYceOHRiNRtq3bw/Apk2bSEtLo3PnzlK7PnbsGC6XC5PJJLXQUKAo\nCocPH+b111+XVt5f//pXMjIyArRNl8vFs88+y+rVq0lJSQF8Kfxno/36a7aKolBYWMisWbMA2L59\nO8ePH8ftdktLOCkpSaYAn+l5wg1pNBrxer0kJCQEPTHA5XIxb948wLfGHQ4HN9xwA1OmTAH41U6/\nlZWVbNiwQWq7O3bsYODAgec0ls2bN/Pjjz+yb98+wOdSKyoqoqSkRHoEhKtZuJFDRYsWLXjuueeY\nNWuWTNAwmUwYjUY8Ho+87lFWVibPDUFFRcVZxTB/L+I6ib9VpaoqjY2NlJaWkpCQAPhiOeGKP4nY\n1549ewB4+OGHOXz4MMnJyYwdOxaA22+/XcbyxLhNJhNerzfAqgrVmM9LAeXxeOjXrx8AxcXFREVF\nMW3aNG666SbgZzPZ7XbLA6+4uJjGxkYqKyspLi4GkAG+cOD1eikuLubpp58GYMmSJdhsNsxmMxde\neCEAzz///Dkdav5+Yv/PEUkOwk3XpUsXdu/ejcFgkPGviIgI+TdCSCYkJHDq1CkpVENFRUUFEydO\npLS0lB49egBw3XXXyUUvnv/YY4/x5Zdfoqoq119/PeALNJ+NkuHvdxf3q/zdeQaDgdjYWAYMGADA\nvffeS3R09K9uRDGG0tJSVFUlLS0tqMqP2+3mzTff5F//+pcc4+jRo3n55ZcDBJPL5UJRFPkua2tr\nefPNN6murpZ7QShtZ4vIfCwuLpYudJEYERsbS05ODuBTcNxuN2azWcZYgnmQifmNjIxk8ODB9OnT\nR35H8L3jw4cPs2jRIgCWL19OQ0ODdGmBTyCEUvn6T3cKxToT7zCYLj7xPYWSdfoYPB4PO3bs4Mkn\nnwRg69atuN1uqqurWbVqFQDXXHONFEJinoxGoxS4oRam56WA+v7776U1YDQamTx5MjfffHPARArh\nJDSqsrIyamtr8Xq9AYs4FIgb4+LZixcvZs6cOeTn50ufvdBALrroIj766CPg7A9bgdBYz5QgYTab\n6dChAwBPPPEE48aNo66uTmq7cXFxZGZmYjKZ5MJ2Op1UV1cTGRl5zokbZ0IInk8//ZSDBw+SmJjI\n9OnTgZ+tAVVVef/99wGYOXMmAFdccQVPPPEEcPZVCsQcGY1GIiMjiY+Pp3v37gCUlJRQV1fHgw8+\nyKBBg4Cfq2/8GkLI1dXVoaqqfM/BQFEUpk2bxrRp02QSwGWXXcazzz5LZGSkXM8NDQ1UVFTgcrmk\nRr5p0ya2bNmC1+uVCsrNN998TuMxmUx07dqVSy+9VKb7nzp1CovFQufOnenVqxcAb775Jhs3bqSk\npETOj0gQCCYiAcFsNstD1Ov14nK5aNGihVQyKioqKC8vD1j3LpeL2trakMdY/deO+HdFUVBVlV27\ndgVkiwYLf6vwdM+KSGrJy8tj//79gE9ZE8pNZWUlAAsWLMBut6MoiszE7N27NwkJCbRu3Tr0lmdI\nPz0EqKrKq6++KrWQpKQkxo4dKzU08LkbGhsbcTqd0lrKz8/H5XIFZPAEe1zg26hTp05lxYoVcvMK\n7dJgMEgtPSYmhvbt2zNlyhSysrKAc9cu/5NwE//fbDbTvn37AM1RCDZFUaRWPH/+fMrLy7n//vuD\nrimpqio1+QULFpCcnMwDDzwgXY56vR5VVVm3bh3PPPMM4DtMsrOzefXVV+X9pLMV6P5/ZzQaAw4y\nRVFo0aIFQ4YMCchm+q3v8v3338t/t1qtMsh8Loj3s3jxYl599VXq6+ulK/H2229HVVUqKiooKysD\nkIHrtLQ06dbau3cvHo+HiIgImbQh0pzPFr1eT1RUFH/605+kxl9TU0O7du3o3LmzPOzNZrOs0iHc\nfHFxcSG94yPWqaIoeL1eTCaTVIS6du2Kqqps3rxZrr2amhpeeeUVpk6dGvLD9kzjra6uZvXq1VKo\nhyrZ4EyfKe6EifdlNBrR6/XExsZK71JeXh6bNm3C4XDIMy47O5u//e1vZGVlBbjKtSQJDQ0NDY3/\nGc47C0qkBAtpHRUVxb59++jQoUPAbWyTycSJEyfYvn07gLywe/nll0ttJVhWgX/a6mOPPcYXX3xB\nfX19gIViMBjkJUHwVZyYNGkSHTt2bLJimsJKgZ9dAKqqSu1769atWK3WkLg/FEWRFmZycjJ33nkn\no0aNklqs2+1mw4YNXHfdddI9lJyczNy5c8nOzg5pckv79u3JyMj4r9+L0+lkzZo1ADLtvFu3buc0\nBlVV5Zp94IEHqKurw2QyyRhdbGwsNpuNxMRE6fZLSkoiMzMTo9EoXYzbtm3DZrORkpLCXXfdBQRH\n09Xr9SQmJkrrpLKyEqfTSXR0tJw3cWm2srKS/Px8wKd9h8NSERewhZsdfOdCQkIC6enp0oVls9mY\nNWsWPXr8f+ydd5zU5bX/39Nne1+WpS0svRrB0BUFa1BRrMQrsZIY0yzR3JhLYu5N8oqJYtRcFdFE\nxaigEKJRQRCQJlU6LNvZXbaX2d3p35nfH/N7DjMICuzMUO7384+Au/N95vuUc55zPudzvsWdd94p\n3y0e8Pv9vPrqq+Tm5pKdnf21zz4dQpeah+MV3xoMBsxmMy6XS8KKHR0dJCUl0atXL1k/1dXVeDwe\nnE6nRKmKior4+OOPufzyyyWSEasz7Jw0UHl5ebLgU1JS2L9/P3a7XUJl6rANTzgGAgGSkpJ4/PHH\nTypscyoIr55XiuDh6gs2m03YOYqUUFtby8qVKxk9erSE/U5UOxUPKIKF0+mkvLwcCC3Ojo4ODh48\nKKw5m80Wlffmdrsl/Pr973+fCRMmkJCQIPmUxYsX89BDD+FwOGS+fvSjHzFy5MiobwZN06ipqaG0\ntBQIHfQJCQkRqvAnKkTUNI2ysjKpMcrOzmbkyJFdTnYHAgFh7NXW1mIymRgwYIAU/44aNQqLxYLF\nYpFDwmg0Cunjww8/BGD37t0kJCTw5z//Oar5H2VA1TvbvXs3ZrOZ0tJSqVErKiqio6MDs9nMokWL\nABg9ejRZWVkxMwLhpITExEQGDx7MoEGDgFBR9+bNmzGZTBw8eBAIHcpOp5O5c+dKAX/v3r1j6gAp\nx7WyspKysjKmTJkSQXQ5HsJzR+F/PxmcSBjW5/NFhPNUCFnTNDkD2tvbcbvdkrNTUDnHWItIn3MG\nymg08otf/ILf/va3QEi+ZMSIEQwbNkwmWcn7JCcnS3xead4NHTo06i/TaDSKF/Loo49y4YUX0tzc\nHOEp7tu3j/3794vn5vP52LJlCwsXLmTOnDlAiC7bFVXuU4FK0KrN4vf7cTqd7N+/X1Qampubcblc\nLF++XAgWPXv27PL4FINISdDk5+fj8XhoaGjg73//OwDPPfcc7e3t2O125s6dC8CDDz4YNe9bfW9V\nwFxcXCzK7n369JHcRbie27HP9vl8VFVV8corr8g7U1JDXb11apomt+2cnByGDRvGvHnz5J0pynK4\nQojCl19+yf/+7/8C0NTUxK233sr06dOjZtiDwSCaptHa2hpx43U6nVRUVIjjoYrQLRYLhw4dAqCk\npIS0tLRvNOBdLQFRxCDVRkb928SJE+nVq5fcrJYuXUpnZydNTU1SJP7YY4/FJBek9pu6GT///PM0\nNjZy9dVXn5TBDt+vJ6PccKzxCP99n8+Hx+MhGAyKY5yamsqRI0fw+/0RdHxV8KyQmprKxIkTRQot\nljjnDJTJZGLixInMnz9f/p6ZmRlRPR+OzZs3A6EE+9ixY2NySwlXjhgwYAD9+vWLWAxtbW3s2bOH\nZcuWSRuMI0eOEAgEKCoqEvn9G264QURJTwffJKWvxlNfX8+uXbvIzs6WA6axsZGKigqqqqpEUcHn\n8xEMBmloaBDiRLjo7OnC7/ezZs0avvjiCyDEXiwqKmLv3r0iputyubBYLNxzzz386Ec/AqJ3e4Oj\nBsrhcFBRUcG2bdsYPnw4EKItG41GXC6XHCZ5eXkSElUhxyNHjvDFF1+wdu1auUEnJCSQmpqKw+Ho\nUk2bSvBDqKXII488Qr9+/SLkqo73naqqqnjyySeFBNC3b1/++te/Rj2spmpoVL1hWVkZLpeLjo4O\nkb/xeDxykCrjP3jw4JPag12d53ClEBWaUlGV/Px8xo0bB8AHH3wgEZdwynxaWlpECPxUD2IVLj9W\nFsrr9bJy5UoAVq5cyZQpU8jKyjrpz1WphPCykJMZC4S+l9rbDocDo9FIfn4+F154IRBiWG7cuJGm\npiZh8prNZmw2G3a7Xfb97bffzp133klSUlLM0xM6SUKHDh06dJyVOOduUBC6Nal8E0R6SwoqDKGS\nfYFAgIkTJ8Y89q2K19TzIZQn69OnD7169RIdwNzcXMrKyjh06JDEdy+66KLT0kgLr0yHo+E7CHni\ndXV1eL1eSVq//vrrJCQk0Lt3b8lLtLW1UVxcTGJiouTEevXqRWJiIuPHj5c4dVffXyAQoLKykpdf\nflnESw0GAzU1NRG6hCaTiQsuuIBf/OIXUc8ZwlHl+4ULF9La2srw4cMll6PCTwkJCRGV8kq3TM1h\nY2OjFO+q952RkUHv3r27VM+iCkx37doFwPTp0+ndu/dx373KG0KolOLxxx+nsrKSadOmAfC3v/1N\n5jNaUGHF7OxsWT8TJ05k3759OBwOed7hw4cJBALY7XapMYtlKwn4qpBteN2jCuceOnRIClGVkktq\naqqovavveGxo61THoeoxIZRzVWQDtZ6vuuoqJkyYcFI3SvV56mf9fv833l7C6606Ojp48cUX+eST\nT4BQ6Hf06NHMnj1bhHw/+eQTli1bhsPhkAiDwWAgJSWFK664ggceeAAI0fWjGc34OpyTBgpO7qCs\nra2Vq6rVau0ys+pUEG6g1MKsrKyUA7i1tVWYMSrem56efkK5+296FoSSnA0NDTQ1NQmJRLFwrrrq\nKmn/cNttt0n/I3XlP3DgAC0tLQwcOFDyHLm5uVKQqQ7trl7pfT4fK1eupLKyUg5WFQYymUyyeRMS\nEhgxYkTMqvzVwbF9+3Y0TWPatGkR3XPhqFwOROZd1IHX2dlJZWUldXV18n7GjRvHAw888I1J76+D\n3+9nxYoVQkBQYrDhYVoVlmpsbOT5558HQoLIeXl5vPLKK1JgHAvGnBpHR0eHhKwhFDbKycmR0K3L\n5SIYDFJQUCDFwbFk8AUCgYhCXb/fLwW7EHLCVq9ezeLFi9myZYv8js1m49prr+Wyyy4DjtZqdTUH\npmma9BjbtGkTRqORiy66iMLCQiCU11Qh0q+Det/hjueppioqKir44IMPxOkxGAz4fD7GjBkj3ak3\nbNhAY2NjhKK6xWJh1qxZ/OY3v5E1Hc9WIOesgToZKBozhDxbxUSLJVTzNJfLJXmA9evXs3LlSrZs\n2RLB9rNareTl5clhorTeThXKEL722mvs2LFDEtIQakQ4YsQI+vbtG5E7cjgcLFiwQJL7lZWVJCYm\nio4cwMyZM8nJyZFW53D6txg1xurqaoqKiggGg9KLCkL07Pb2dsnbdO/e/StMzGhCOQUdHR3k5OSQ\nnZ19XAWOcAQCAdra2uTmt3LlSjZt2oTJZKJ///4A/OIXvzitBorhaGhowOFwSEG5ulV7vV4xjnV1\ndWzatIk33nhDGIQzZszg5z//Obm5uTE9RNR3czgc4uAcPHiQHj16UFtbK0W5qmj55ptvljUVK69b\nzY1q1Kj6ciUlJcnaUyScL7/8MkK+q3fv3jz00EMyxmi1jlCMXvWZBQUF2Gy2iNx0XV0dhYWFcviH\n33qUYfX7/SItpPbDqcyvwWAgPz+fiRMnSp6trq4Ol8vFkiVLpETio48+EuOkxvOjH/2IJ554IuIM\niCfOeQP1dZXMbW1tEXpdsaRxhzcmO3DgAOvXrxeCxt69e6mtraW1tTXiRlBQUMCMGTO49957AbpM\nv121ahUNDQ1CAQWkdqa5uVkOf6PRSHFxMTU1NfJ+pkyZwqBBgygsLJRQR2ZmZtQ2q9ps5eXl1NfX\n4/f7IwgIiqCg6kHGjBnDwIEDY6Y8oIzj+PHjpWYtPLwY3qYBQofEvn37+Oijj6S2buPGjSQnJzNt\n2jRhYnZVrioQCDB//nwWL14s87V+/Xqqq6vZsWMHGzZsAEIKEUajkZEjR4rSxtixY7Hb7XE5SBS9\nXbHhvF4vmqbRo0cPMeAmk4mLLrqIGTNmxLwpoNfrZe3atfzrX/8CQg5pfn4+gwcPlrU3f/589u7d\nK+oWEGJsvvDCCxQWFn4tAeV0YLFYJAQ6evRo7HY7wWBQmknu2LGDNWvW8Oabb4q2pFLnKCkpEWJJ\n9+7du9St1mAwkJGRwX/+538yY8YMAJYtW8aGDRsoKSmRc0rVl6ampvL73/8egLvvvjsm/cxOFjpJ\nQocOHTp0nJU4Z29Q4SQAv9//FSsfCARYu3ateMUZGRlxae3c0tLCm2++yfvvvx/Rbl6pTCvPLSsr\ni5/97GfceuutETebrjy7b9++BINBUcqAkEdfXFzMjh075Ofq6uooLS0VIgLAXXfdRbdu3bBarV/J\nxUQDKqQyf/58vvjiC+l4DKHbzKBBg7jkkkskPp+SkhKRA4o2lLc8YMAA1qxZw9KlS6UN+KBBg7Db\n7XR2dkr4bNmyZXz00Ue0t7fLd0lJSeHaa6/lrrvukmLQ0/VyFZSSR3l5uczXf/7nf0qnUxV6ueKK\nK5gzZw5jxoyRyEA8QzAqZKXUF7788ksGDx7Mvn37JIzd3NzMVVddRY8ePWKyphTUO3v11Vclt5Sf\nn09ubi4rVqwQYem6ujo8Hg9Wq1WKcl977TUyMzNjQpdWmoUQuk253W7ReYRQaDEQCLBhwwZp8qhE\ni8eOHStrSpXQdOXdGY1GkpOTRYmkT58+JCcn88Ybb8h6NpvNpKam8vjjj8e0G/Sp4Jw3UG1tbXLw\nq42qWFBr1qwRA9WnT5+4xOWTkpJoaWmhvr5eCBqqaNBms0muYu7cuVx11VVRKcxVm19dzevr60V0\nVYlmHj58WCSMFDtt7NixTJo0CUDqPmJ1gCgkJCRgMBjo1q2bMDHvvvtuJk6cSFZWVsRhG8sDV228\ngoICDh48SElJCZ9//jmAvKdgMCghrNbWVoLBIN27d5fusBdccAE33XRThLxPV8fscrlwu92YTCbJ\nN7W0tGAymejZsycvvfQSEGLNxbv76rGwWq3CFpwwYQLBYJD+/fvLnuvVqxd33HFHzJl7EHpHBw4c\nkPkK70EVPjeZmZn84Ac/kLb0sQqJHpuzVWrr4bj//vu58cYb+eMf/yih2379+nHppZdy2WWXSU4s\nWmMMr9fMycnhu9/9Lh0dHcJoNJvNzJkzhxtvvPGMry0Fw+mwxmKEUxpIeD8fp9MZkZA8fPgwt9xy\ni7CgAF544QW+973vxcxIhVO8a2pq+OUvfynJx0AgwMiRI5k4caLkKtLT06M+lsbGRhISEsQInGiM\nxys+jPUBohLVLS0tVFVVYbPZxEBFw0M8Vaj8UmdnJ8FgkL1797Jq1SogpEHodDppbm6Ww9ZoNFJY\nWMgDDzwgN61YjNvj8bBu3Trmzp0rdPYxY8bw2GOPMWzYsDPu0YYjnFWoWGadnZ3CXOvVqxcJCQkx\nZ30Fg0GqqqqYPXs2O3fuBI52EFDtVCAkdfT0008zePDguCuXfx2OLeiNtXOmoDREw9VSlDpElJ9/\n2h92zhoo+aX/T//1eDzCXvvd737H559/TjAYlCvtO++8E5M2GzrODxy7D9ShcWwfn1gfHCrhr2jk\nQFwkZU4Hx9YcKYOlnBHVzuF4bMhj32tXoWkatbW1LFmyBAgREDweD/n5+dxwww1ASL/wRM6bjpji\ntF/42bfqdejQoUOHDs6DG5SC8qAA3nvvPXbt2kVBQYEU340dO/aMtbWIF7oqsKlDx+ni2NuUwonU\nL45tPx7NzgLHPkPfE2cc/3dDfDp06NCh46yGHuLToUOHDh3nF3QDpUOHDh06zkroBkqHDh06dJyV\n0A2UDh06dOg4K6EbKB06dOjQcVZCN1A6dOjQoeOsxNmj93EeIlxaqL29nfr6evLz84HYdxbVoUOH\njpNBeFsZp9NJfX29iBIrDVGbzRbRXTpe0A1UHOD1elm9ejVLlizhwQcfBEKyK7HsT6Xj/yb0Yu1z\nC0oPT83ZmZBiUo60w+Hgf/7nf1i/fr10+rVarfzgBz9g9OjRZ+S80kN8OnTo0KHjrIR+g4ohlGdS\nXFzM7373O1pbWxk7diyAiNj+X4PyGMvKykSaqrS0lEAgQF1dHf369QPg8ssvJzMzE4vFot8IvgZK\nQRygpKSE9vZ2+vTpQ1ZWFkBEaEZHCCe6ZX5dd+5oPrutrQ2A3/zmN6xZs4ampibp4TZv3jx69Ohx\n3DBarG7Hqi3Qnj172Lt3L1arVXrUqV5nZ0pxSF+5MYQ6OB555BH2799Peno6V155JRDfOO7xoFTg\n1Wb57LPPePLJJ+WaDzBr1qyobIhAICDtD371q1+xdOlSWlpaRObfaDTi9/sJBAIRrUCSkpKYPXs2\nv/71rwFi1v5dITxnGAwG8fl8OJ1OIBSbd7lcpKenS/sG1VDxTM6lmkcI9Ubzer3SIgQQVfQzYeS/\n7lCL53jC5/VYHcDj/Zz6mVjMq9/vZ9GiRUBozxUXF+P1etm9ezcA5eXldO/e/bjPjsU7CwQCtLe3\nA7BkyRIKCgq49tprGTVqFBBycFTPPeXoxNNpPO8N1LGbJF4vNhgMSn+htWvX4vP5mDhxInl5eRE/\ncyY2qt/vZ+/evVRUVJCWlgaE8mROp5MjR46wYMECAG699daoeN+BQIDNmzcDsHz5crk5KS8tIyMD\nv99PW1ubNOnz+Xy0tbXx/PPP8+abbwKwefNmuWFFE6rFhfIkDxw4wMqVKzl48CB1dXVAqI9VbW0t\nHo9HRIeTkpL4yU9+wpw5c86YELHBYJA5GjJkiHyXM2U0NU0Tx6y9vZ2tW7fy6aef0traCsDw4cOZ\nMWMGffv2jUvSXdM0qqqqANi2bRsmk4mpU6dKXy3VQsXtdkcYMovFgtVq/Urjwa7CZDJxzTXXAKEO\nv++++y4dHR08+eSTAFx00UVxXUsOh4OXX34ZCO3TO+64g1GjRknDQtWB+PDhwzKvvXv3jlv/tvPC\nQIW3f4fQYdvY2EgwGMThcAChzWIymRgwYID0hYqVlwShA3bu3LlAaJLtdjv33XdfRMM5Ne54GSnl\nWdfW1uJ0Opk0aZIYCU3TWLRoEYcPH5aW1NEcl3q2pmlYrVYKCwtlY/Tr1w9N06isrGTXrl1AyKh/\n9tln1NTU0NzcDMC3v/1tampqutztM7xBnMfjYcuWLfz973/n4MGDAFRVVdHc3BzRkO/YG576nMce\ne4xp06ZJp+R4GoZgMIjb7RajrhLsiYmJMu54Nb8LBoO4XC5efvll/vSnPwGhlu9erzeCJWa323nn\nnXf43//9X4YNGwaEDH20xxgMBuns7GTu3Lm89957QKiJYUFBAX369KGgoAA42rNq37594oxs3bqV\nlJQULrvsMkaMGAEQtTCpwWAgPT0dgClTprBp0yby8vL4zne+A4RuLPE6DzRN4y9/+Yvc6C699FIK\nCwuxWq1iJM1mMxaLhQ0bNlBSUgLA7NmzGTFiRFxIEzpJQocOHTp0nJU4J29Qx3rATqeTiooKPvnk\nE+BoGMlut4vn3qNHDzIzM5k6dSrXX389EGq7frwrfDRCb/v27WPfvn3y9379+nH11VeLZxLvvjXB\nYFCek5OTI4lYNY7W1lY2bdoEwMSJE4Ho3QaCwSB9+/YF4Kc//SkNDQ389Kc/JSUlJeI5ubm5kiz+\n3ve+R11dHSNHjqSpqQkIeeSbNm3i4osv7vJ41K1jy5Yt/OQnP6GiokLyTZqmYTKZCAQCclvLy8vD\naDRit9ulFXtHRwcej4eXXnqJ//7v/wZCN4R4esBlZWWSQxg4cCB+vx+3201GRoZ811iGktX6qaur\n46qrrmL37t0RNyYF9Xyv10tZWRkvvPACv/zlLwHo27cvJpMpKq3G1XgaGxu57rrr2Lp1q0RWjEYj\nLpeLuro6qUf0er0yxytWrABg2bJltLe3s2nTJt56660ujed441NnUlVVFQ6Hg5tvvlkiGfEM+dfU\n1PDmm29KLviCCy4gJSUlInynbphWq5U9e/YAMHfuXJ566ikGDhwY84jBOWmg/H4/q1evBuCtt95i\nx44dOBwOCecFAgESExMZOXKksOb69OlDc3NzBJ/fYDB8pf10NBAIBHjqqaciNsYjjzwSEd6Ld9La\nYDDIYgoPI6gNPXfuXBoaGsjOzua+++6L6hjNZjOFhYVA6DAyGAxyIB07RgWTyUS3bt2YPn06r7/+\nuoz12WefZfLkyV0aW/jv+v1+xo4dG5H/stlsZGRkUFBQwBVXXAGE6tYGDhyI2+3mqaeeAkKNMf1+\nP8uXL+fee+8FYPDgwac9rlNFTU0Nv//97xk3bhwQMqJut5tAICAGKhqH/okQDAapqakBYNKkSVRU\nVESEQE0mE4mJieTk5EjhZ1NTEzk5ORQUFMi/aZomZI6uhNKCwSB79+4FYPr06VRWVkY4Znl5efzq\nV79izJgxJCQkAEfPgN69e8vP1dfXEwgEaG5ujjoDMhgMSg520aJFmEwmrrzyyrifB52dndx4443U\n1tYyYcIEAGbOnClhYjWPJpMJu93OgAEDaGlpAeDLL7/kN7/5DX/5y1/Izs4GYneenXMGStM01q1b\nx6OPPgqEPLeUlBS6d+8uh0Nqaipz5szhoosukkXn9/tpbW0lJSUlYnGG//fYP58uvF4v69evQJBM\n5AAAIABJREFUl79bLBYuu+yyiIkHvuJpxnqRHu/z1aH8xhtvYLVa+fOf/9zlHM/xnns6G91oNDJn\nzhzeffddIHRb3rlzJ4FAoEuJZIPBIIfjxRdfTE5ODm1tbRw+fBiAAQMGMHbsWK655hqhayuD6vF4\n6NGjh4wPQptdfV488eGHH1JWViZ5EpPJhNvtxmw2f2WNRxuBQICioiKmTp0KwJEjR+Sm1r17dyBE\nmZ46dSqBQIB169YB8M9//hOz2czIkSPJzMyUMXY1HxwIBNi3bx+XX345EDoXlHFSztHy5cvJz8//\nyr4zGo3U19ezcuVKIJQ/tlqt5OfnR/39ud1u5s2bB4Ro3Zdddpk4E/GAur396U9/Yt++fRiNRqZN\nmwYczQUeex6azWaSkpLkpuX1evnwww/Jycnhj3/8I0DM1v85Z6B8Ph//+te/5CX26dOHqVOnMm7c\nOAkjdevWjfT0dMxmsxgEn89HdnY2FotFDrdYbd6GhgZaW1vl2Xl5eV9ZhOGU1zNV5xMIBHjuueeA\nEIlk4MCBzJw586yqOxo+fLhUtZeWluJ0OvH7/V1mOqnD0Gg0Mnz4cJ5++mm2b98OhDZgQUEBXq9X\nmEspKSn4fD42bNggt3ej0UhycjK33367GK14skQNBgN9+/YV0o/JZCIvL4/s7OyY1j4Fg0FaW1uZ\nNWsW9fX18u9ms5mZM2fyl7/8BUAMUH19PUVFRUCoVEBRltUYzWZzl8gcgUCAbdu2cf311wvRQRmn\n/v378/HHHwPQq1cvmfdwI6VuS+Xl5UDICdY0jdzc3NMaz4mgaRrPP/88CxculH/bt28fxcXFDB06\nFIj9+lHv4q9//Ss+n4/CwkK5/YeH9sIjLJqm0djYKKQWVbP45ZdfCjuzW7duMRm7TpLQoUOHDh1n\nJc65G5TFYuGGG26Qa7vP52P48OF069ZN6NGpqalCAFB5IBUWOhF/P1rWPxgMUlZWJp43hPIX6jZ3\nbDGo3+8XTzJe9Q/h2lsq3GAwGL6SJzsTOJbG7fP5It6Ly+Wio6MjquM0Go3k5eUJ+eLAgQOsWLEi\n4hnJyclUVlayc+dO8dLz8vKYMGECP/7xj+OuUxYMBhkyZAh79uyROh8I1ZTFWjnC6/Xy7rvvUlZW\nJv+WlpbGgw8+yMMPPyzhHq/Xy+bNm/mf//kfiouLgRAxacSIEQwZMqTLkQy1VtatW8cNN9xAS0tL\nRKH3sGHD+OCDDyJCsupZ6ial6PHvvPOOhLuDwSAmk4nRo0ef1rhOBL/fz8KFCyVU5vf72bhxI5Mn\nT+ZnP/sZAD/72c9iQruHUIj8Jz/5CRAiHNntdl566SWphTyRukZHRwder5eBAwcCoVDgJ598gtls\njjjnYoFzzkCZTCYmTJggbC+fz0dLSwuNjY2ywOx2O5qmEQwGhZllMplISko67mdGezFs2LAhgp03\nadIkOTRUaMHlcuF2u/H7/bJA4ilLo8Ii6oqenZ0dNeWI04WmabJ5PR6P5C6U2oUy6mqeowmj0SgK\nET169KC+vp7169fL4R+uuqHeUVpamhTvxrumDUK5Mk3TZM00NzdHFIJHG+o71tTU8Nxzz+F0OsWI\n33TTTcycOZOqqiohtSxdupTq6uoIJ8Pr9ZKcnCy5PYXTeW/KUZgzZ46E1NWe69+/P4sWLYpQZTjR\nAbx//36WLl0aUTuWk5PDpEmTTnlMXwer1cpnn33G1q1bAVi8eDFffvklZWVlvPbaa0Aolzd37lxy\ncnKiypALBoNs2bJF1nMwGGTMmDFMmDDha9+91+tl//79pKSkMH36dAB27tzJrl270DRNztdY4Zwz\nUHCUWQKhQ10xhcKp59XV1ezYsUMoytOmTRMGWazp3FVVVRGsuXCyhrrRaZpGcXGxqEwAXHXVVXzr\nW9+Ki5Fyu908/PDD8qw//vGPZ/T2pHJy4XNTVlbG2rVrxSCp/7d9+3ahCUdzLtUhmp6eTlpaGuXl\n5cJc0jTtK8l1j8fDunXruOuuu6Q4ddSoUXEt1i0pKaGhoQEIUfPjIQW1ZMkSqqur0TRNbo4HDx7k\n17/+NRs3bqSxsTHi541Go6yz5ORkBg4ciMVi+cpt+VTHvmTJEgAqKytl7ah85cKFC+nTp88JoxLq\n2e3t7Tz33HMRty+bzcaDDz4Y9RyUwWAgKytL5M6uuOIKPB4Pn332mVDc09LSqKioIDMzM6rrSNM0\nnn32WSFJJCQk8Oqrr57wrFFrvbm5GY/HQ0FBgTj4JSUlJCcnY7FY5ByOVSnDOWmgIJKBZzQapW4F\nQhuiuLiYV199VeoL7rjjjrhpSCnGl5o85dWGT6LBYCAhIYGDBw9KhXZpaSnz5s0jNTU15uMsKiqi\nvLxcFuj48ePPODkivEbk4MGDbN26lc8//xyv1wuE3pnf72fBggXCHotmewL1ORaLhdTUVKmHUrBa\nrRGHqqZpYqSuvvpqAF544QVmzJgR83Ct3++nvLwch8PBgAEDgFAdWSzn0OVyAfDBBx/In5XzsHHj\nRgKBQIQRVwwwtQfVZxw8eJDi4mIGDRoEHE3On8ohFwgE5Kbm8XgwGAzYbDY5/BMTE0W6Sq3x8BuW\ncgpfeukllixZgsfjkf83atQoZs+eHXOHTY156tSpwnzcv38/GRkZUV8/nZ2dbNu2Tb7j8OHD6dWr\n13F/NlyA2OVyMWjQILKyssQRWrp0KWVlZUyePFlqGWO17nSShA4dOnToOCtxzt6gjkV4SA2goqKC\nQ4cOMXPmTCDkUcUr9NKnTx/MZrN4YCrXpGmaFBM3NTWxZ88eOjo6JC7s8/nYvn07kydPjjlNeMWK\nFXg8HvGiOjs7z2izO0Vo2bhxIwAvv/wyNTU1OByOr9RlVFdXi2ZfLBqpqULX/v37y2crJZD29nYp\nTlQ33ubmZtELvO++++js7OSOO+6IyXpTIWKXy8XevXu58847RVcu2vVr4QgEAhw4cACAvXv3Skjt\nWB1MdSsAKCgoIC8vj4qKCln3ra2trFmzhkmTJknS/XTHo3JQCiaTSfbSv/71L7Kzs7ngggskgqHy\njC0tLaKQ/95778ltUNWOzZs3j4yMjLjthfCzy+v1xoQkUVNTE5EvGjFixHGfEQgEqK6uloLn1NRU\nBg0ahNfr5aWXXgJgzZo1WK1Whg4dKuUzeqHuKcDhcPCPf/yDzs5OKdyLF/lATZQib0BIXPGKK65g\n/fr1EhLJzc0lIyODPXv2yHW6ubmZPXv2MGnSpJgm3YPBIO+++y42m02K9LKysqRAMd7w+XyUlZXx\nzDPPsHTpUvm33Nxc0tPT5Z01NDQQDAZpaWnh1VdfBUKEhvz8/KiGREwmE2PHjiU3N5fevXsDIQWM\nY4sR/X4/o0aN4uGHHxZZK4fDwaOPPorVauXWW28FossQVSHQhoYG+vbtKyxWNe5YIRgMSiGr0WiU\ndaKe6fP5MBqNdOvWTfJxhYWFrFq1iueff17GbbVa6dOnDz179hTjr9h1p/KegsFghDyQwWBA0zS2\nbNkChEJlFouFIUOGiIHyeDx4vV6++OILKcpWoT6r1SoMtzFjxsSFUavyrk6nUwRtExMTY8IITUxM\nJDU1VYg+hw8fxuv1Rkhzqb315JNPyjsbNmwYHo+HxYsXyxjdbjfjx4/n9ttvj/l5cV4ZKHWoV1VV\nUVtbS3JyMt/+9reB+KlMGwwGJk+eDBz1Knfu3MmePXtExl+NUVX+K88mJSVFkuyxziVUVFQwbtw4\nHn/8cSBUVKnUuuPlOarc0tNPP82LL75Ic3OzHGQ2mw2/309eXp4wvsrKyoStqTy8uro6cnNzo/rO\nzGYz/fr1i5C/UcWkEJn8HzNmDNOmTRN9vqamJpqamnj88cdF2WTUqFFRGZvKeUFIB9BqtbJ161Yp\noIw1QUIV3vbs2ZPq6uqI9ZycnMx3vvMd7rrrLrlhrlmzhtdee42mpiY58C+88ELuvPPOiJYOp7M3\njUajOA9lZWXC+lS3IZfLRVJSEqWlpezcuRMIOYAul+srfccyMjL4wx/+wF133QXE1tAHAgExigaD\nAafTyd69e/niiy+AUDGxmuNoIjs7mwkTJoiR2bRpE6+88gr33HOPzIPSSdQ0Tcbzz3/+k/b2dlpb\nW2VcBQUFPProo/Ts2TPmZ8V5ZaDU4bZs2TL8fj/XXnutVNnHE6NGjZIaFTgavlK6XxBqgKfCJGqT\n9+nTh+HDh8fcmK5bt47MzEx++MMfRoQ/lExOPG6bmqZJY8RnnnkGj8dDamqqHG45OTlykKj2BKmp\nqdIGQ91O9+zZw+DBg6PqyWmaRkNDA8XFxRICTUlJIS0tLYJufujQIQ4fPkxlZWVEXU0wGKS+vl7E\nd0eOHPm1G/lknAL1LsKFdd1uN6mpqVKmEEsYDAZROxg2bJg4D4o1N3r0aC6//HLMZrP0QXvssceo\nra3FaDSSk5MDwIwZM5g4cSLJycldWudGo5Hf/va3APz85z+npKQEs9ks78JkMkk/p4qKCiBktJTT\nqPbcsGHDWLZsGfn5+VGndYe3a1HORXNzs7DhkpOTcbvdHDp0SMo9CgsLJdQYTdhsNm655RaWL18O\nhPqbzZ07l8WLFzNy5EggFOYvKSmhvr6e6upqIHTDVNJgU6ZMAeDXv/41o0ePjsstUydJ6NChQ4eO\nsxLnzQ1KdRKFUB4gJSWFH//4x2ek06nVamXDhg08+OCDAHz66ae0tbWJ96bGazabSU1N5brrrgPg\niSeeiGgBEm2osMb69esZPXp0BM1UNXlU9UWxRDAYpLi4mPnz5wPIzW3EiBESkq2oqKC6upqWlha5\n0RUUFNCtWzdSUlIkLJqbmxu1OVY38L/97W+89957OJ1OUXqeMmUKHo+Hbdu28dlnn8nvKJFiNYal\nS5fi9/vJzMwUJfRvwjfNt6ZpdHR0AEep3h0dHRQVFTFgwADJjcUy3GI0GsXT/u53v8uYMWMoLy8X\nQoTL5WL58uWUlZVJGKm2tpZAIEBycrKs8e9+97tdvj1B6Lsqkdz58+fT2NjIqlWrJAxlsVjYuXMn\nNTU1cltSqi0ZGRnMmTMHgIceekio0tGEKmJVnaQ3bNjAjh07sNlszJo1CwjlNVesWMGqVatkPWdl\nZcXkBmUymZg0aRJ33HEHAK+++iqdnZ1s3LhRhK2VggYcXUtWq5UBAwbw8MMPc+211wKhaEK80gDn\njYEKBAKyiZuamvjOd74TlxjpiZCUlMQLL7wAhKrDFy5cyIoVK4QQkZGRwZgxY5g1a5awmaJZ03M8\nqM3rcrlISUlh+/btwsLZu3evKDjHA7t375Y+RspwlpaWyr+pTp42m01CS6NGjaKwsBC73S5J7okT\nJ0alB1MwGBTD8+ijj+L1ejEYDMJce/fdd+nZsyft7e0y3smTJ3PbbbcxYMAACeVcf/315OfnM3jw\nYAlXnuphfGzILxgMUl1dzaFDhyT0kpWVRWdnJ4MGDYpLSFbV7UGo8DwrK4uKigoJY/p8Purq6mht\nbZU1brFYSE5O5oknnuDuu+8GorvG1ffu1asX+fn59OnTh/379wOwefNmevfuTUdHh4Sxs7Ozufji\ni7nxxhsZMmQIQExrnUpLS3nzzTcB+OKLL3A4HNJKHqC6upqysjJcLpfUQV188cUx6aprMBhISUmR\n1vJXXnklL730Ep9++qk4PUoAIS0tTRyzW265hQkTJpCenn5GnH1DeOHhGcZpD0TpRf373/8G4JNP\nPuGmm27iqquuimtV/7FQh5bKQalDD0KLwWKxxFXNXC3E6dOnc+jQIdLS0oSinJOTw2OPPcaAAQNi\n/s4CgQBvv/02999/v4xLeW8qZ5iWlsawYcOYMmWKFF8WFhZ+heF0Im3FU4XP5+Ohhx4CQt6lmit1\ngKWlpXHJJZfw4x//OMKhUHN4LI4tyu4KFNNry5Ytop6gaRpTp07l8ssvj3uzO03TcLlcFBcX869/\n/QsItf6or6+npaUlQm7oqaeeYsKECTE3osfS3f1+P83NzaxYsSLCocjNzcVut8t4YhmtcLvdwip8\n++23WbdunbQlgdDt22AwRPRgu/7667Hb7XE5t9TNXCnSq3ejhA/gKEOyizjtDzgvblCqCZhqg+D3\n++nXr98ZV0YIX2QqaXsmoRZdTk4Omzdvprm5WUIL//Vf/0VBQUHcDPqoUaO44IILANi2bZvI5igD\n1K1bN+6//34mT54sB3C44oDC6czxsQoREFpDt99+OwBtbW0cPHiQ/v37S5hu3Lhx9OrVK+K29nXP\njubaU/U748aNE0q5z+cjJSUlJt72yYwnISGBIUOGiBDruHHjePvttyktLRWR1fvuu49+/frFxfNW\n7yBceLlbt27MmjVLHEWz2Rw3h1ARC8aPHw+Ebp1K6FiVJCQlJZGSkkLfvn1FBzKcLRprmEwm0tLS\n4kKyOV3oJAkdOnTo0HFW4pwP8amWDMXFxXzyySdAiL45a9asE6qX/1+FCn989NFH/PnPf6a+vl4q\n6qdPnx6VXM7JIhgMSk6submZxsZGabgHoTlUNOFYPPt4nxsu5KtUwmMdCjpXcSyN2ul00tbWhsPh\nEJHV1NTUmM3hyY4xHPocnjGc9os/LwyUYqAp0dXCwkLy8vLOSFJPh47zHcc7M5SxCq/VOlZ+7Ezg\nTMp36RD838tBhW8Sk8lEQkKCMNASExP1RalDR4xwooafZ9oYHQ/6OXBu45y/QckvdzFxrkOHDh06\nYoLTPpDPPpdHhw4dOnTo4BwO8R0L/dakQ4cOHecX9BuUDh06dOg4K6EbKB06dOjQcVZCN1A6dOjQ\noeOshG6gdOjQoUPHWQndQOnQcZIIBALSkkOHDh2xh26gdMQFSj4IQkKnSiIHImVz1J/PJILBIH6/\nH7fbjd/vx+/309bWRnl5OfPnz6ejo0Nau+g4+6HWlFpj4evsTK81HV+Pc5JmHr6w/H4/jY2NfP75\n53z00UdASBF43LhxjBgxQvq+WK3WuLe3UGMMBAJyKKvWzna7nZSUlK+0jzgfoWmaSPrfc8899O3b\nl/vuu096UQUCATRNIxAISM+hzMxMzGYzwWCQhoYGALZv386BAwe47bbb6N27NxCb8gKDwSA6fGoO\n3W43L774Ivv27WPcuHEAXHDBBXEtbwjXL3S73aSkpJxwPatxKykwv98vrUPC2ymcz1D7rrm5Wf7N\n5XLhdrtF9cJqtZKTkxOhuxhPRfGzFeGGW8lFnYl3ck4aKDjaa6mkpIQnnniCLVu24HA4gFCjtE8/\n/ZTs7Gw5BNPS0hg+fDjjx49nwIABAPTs2VMk+KM5LnXYqv5L27ZtY9WqVZSUlHDkyBEgdGj/4Ac/\n4NZbb41Lwzk1NoCWlhY2btxIQUGBdBNNT0/HZrNhMpnk8IrWezEajRw6dAiAZ599FpfLRd++fSOe\nYzQaCQQC0hXZ5XLh9XpxOBy88cYbQKhbrc1mIyMjg7vuugsg5gZebUqbzUZpaSn79++XjrEjRoyI\n69y1t7fz4osvAiE5r9mzZ5OcnPy1BqqmpobnnnuOzMxMbrrpJgD69OkTlzGfaQQCATo7OykuLgZC\ngrZ79uyhqqqKyspKIHQGTJs2jfz8fGkamJmZGfU2JkrU2ul00tLSAoQaq7rdbjRNo6KiAoCRI0cy\nfPjwuK2rY8eo1o3ah62trbS2tpKbmyttOeLp3OghPh06dOjQcVbinLxBhV83NU1j3759dHR0kJ6e\nDsCECRO48MILsdvt0hxs165d1NbWsmfPHrKysgCYNWsW48aN63Lb50AgQFNTEwAVFRV89NFH7N27\nV9TV6+rqyM7OJhAI0NbWBoS8p4cffpiJEydKV9tYInyMb7zxBps2bWL8+PHSXK6qqorm5mby8vLk\nhpmSkhIVbykQCDBp0iT5M0R2wg0PR6kbkd1ux+fz0dTUJI0oKysrMZlMlJaWxj3ckJCQQFpaGi6X\nK2qdck8FwWCQt956i0WLFgEwfvx4rFbrCW+56p2++OKLLFiwgP79+0vb9bNR1DXaUKHNmpoa9uzZ\nI/9eXl7Oxo0bZS/U1NQQDAYZNGiQNF+cPHkyRqMxqg1G/X4/e/bsYenSpezduxcI3ZZSUlLYvXs3\n27ZtA0I3utdff52srKyYry+Vk/P5fECoUWdRURGrV6+WqEVNTQ1er5eMjAweffRRAB588EHsdntM\nx6ZwThooOHo4mEwmMjMzyczM5KmnngJCE2+xWPB6vdTW1gKwe/du1q1bx+eff87u3bsBcDgc9OrV\n67Q7yQYCAdxuN1u3bpUJ3b9/v3Sq7datGwBXXHEFU6dOpbGxURbiP//5T9rb2/niiy8k5BLrBVla\nWgrAqlWr8Hq9WK1WWWiNjY2sWLGCuro6CQVdeeWV0oK9Kwh/t8cLXZzowFchv3BDZrFY4tKW/lgY\njUY2btxIe3u7zGs8x+ByuVi4cCHl5eUAXHPNNV97gCoDtWzZMpxOJyaTSfqjRWvc4f2glNN47BwG\ng0E0TZND0O/3U19fT21tLX379gVCeaD09PQIpyUaUHtT5T/37NlDSUkJDodDQliKDLNt2zY2b94M\nhJzHmTNnYrFYojaeYDDIZ599xo4dO8jMzATgsssuIysri+7du7NhwwYAduzYQXFxMZmZmVEPMYb/\n1+fz0dnZySeffMKqVauAUCqipaWFhoYGyXWq32lvb2f58uVAyDkaN25cXEJ956yBUmhrayMxMZGR\nI0cyatQoAGmfbLVa6devHxDyTAYMGEBRURGHDx8GoL6+HrfbfcrPVJPc3NyM1+ulqKhI8l85OTn0\n7NmTBx54gKFDhwKhxm0GgwGv1ysbY9GiRQSDQXr16hUXTzw8vpyQkMCcOXOYOnWqGIyamhr+/e9/\nU1RUxM6dOwG49tpro/LsU/l+4RuotbWV6upqOciOHDlCTk4O48ePj/sNKhAIUFtbi8fjOSMtso8c\nOUJJSYk4FA888MDXGhqn0wlAWVkZRqORJ554Qggo0YAibNTU1AChvZaSkiIHvnr2yy+/zP79+yXv\nYjQa6ejoICEhgcsuuwyA3Nxcvve971FQUBDVnKLX66WhoYGNGzcCIefR6/Vis9nIyckBIC8vj+bm\nZnbt2iVGtKSkhL59+zJlypSorTODwcA111zD0KFD5VxQPevsdruULyjWaLSNUyAQwOl0CmGkqKiI\nRYsWsX79ejkPATweD0ajUc6F8LGom+jTTz/Nq6++KudaLHHOGij1YtSCLy4uprGxEYDevXtH/H8I\nbaC0tDRSUlJITk4GQhsjLy/vlF+y+vnU1FRcLheXXnopY8eOBULGsXv37hHdaQ0Gg1CX//nPfwKh\nzZOens7IkSO78hpOacxq0fXq1YsxY8Zgt9sjkqINDQ10dnbKBjoTDEPluZWXl+NwOEhNTZU+X5Mn\nTyY7O5uePXvG3UCVlJTQ2dlJMBhk4MCBcX02wOHDh/H5fAwePBhAvPAT4fXXXwdCB0x6ejqXXHJJ\nVN+Zx+Ph7bffFq86KSmJqqoqdu/eTWdnpzzbZDKRlpYWEbpNTk7m5ptvlvfYrVs37HZ7VG+kar/V\n1NQISaKlpUUMqXIUS0pKqKqqiih9aGlpYfPmzVxyySVRG4/ZbKZ///6kpKRIVEKxRA8cOCAObnZ2\nNqNGjYrKXIUzUB0OB9XV1ezYsQOA9957D4fDgclkkvEoslJSUpKkSxobG+ns7KSjo0PCoqtWrWLh\nwoXce++9UQ2DHg/nfzBahw4dOnSckzjnb1BDhw4lLy+P4uJifve73wHwpz/9iaSkJLm5QMiLWL16\nNdXV1XLbefTRR0lJSTltb8VisWA2myW2Dxw3jq6u2EeOHKGoqAgIeZK//e1vJRwZaxgMBqHRXnrp\npaSnp2MwGMRrXLt2LZs2bSIxMZHLL79cvks8ocIQADt37mTQoEFYrVYhtezatSvqYaBvglo/xcXF\nWCwWEhISJOQYzzFs2bIFr9fLRRddBBw/l6fg8Xh45pln5O+zZ8+OanjP5/Pxt7/9jWeffVbWj7rl\n9uzZk7y8PABuvPFGhg4ditlsljn0eDxkZmZKOFB9XrTrbILBIGazGYvFInlot9uN2+3G5XLJs1WO\nLLwjsM1mIy8vL6o3OoPBgMViIT8/XwrWNU3D5XIxb948eda0adOiciYEg0G5lW3bto3s7Gza2tok\n/z5x4kQcDgdTpkyR+TKZTNTU1GAymYT2vnHjRkpLS9m5c6eMu7Ozk7a2trhQ4c9ZA6WQlJTEggUL\n+P3vfy+1De+++y5XX301aWlpEg9///33WbRoEXl5edx7770ADBkypEsvWW2qb2JSBQIBOjo6eO21\n1yTOfeGFF3LzzTfHNVSl2IrKeIfXav35z3+mo6OD6667juzs7LiNKRyaplFVVQWEQh2pqal0dnZS\nVlYGhDZGYWFh3BhE4UhMTJQ6MRWGVAWMsUYgEGDbtm0YjUYJi53ouYFAgGeffZa6ujogFAp86KGH\nojJOdUAtXryYZ555Bp/Px6WXXgqEnL2srCySkpJkfsLJLep3DQaDOD7qv8FgkM7OTqnJ6wqOLVCu\nrKyUcJ5SkQiXq1J72GKxyP749re/HdXwXjjCzwqPx8Onn37Krl27hEH4yCOPRMUxVDlvgPb2dpKT\nk0lOThYmntVqxe12k52dLWE6g8HAsGHD8Pl8XHjhhQBcdNFFzJ07F7/fH1G8q5zdWOOcN1AGg4HM\nzEx++9vfCgX34MGDrF27lqSkJJYsWQLA1q1bycvL46GHHuKCCy4AQpMUywMm3EAdPHiQDz/8UDbv\nyJEj43rQGgwGOQCMRqMUCH7wwQcA1NbWkpqayh/+8IczVkWvPEoIFQ4r71dR80eMGMGtt956Rsbn\ndDpxOp0kJibKho4Xi8/j8VBUVET37t2FWHA8aJrGzp07eeaZZ2Tt9erVKypMTE3TpNj68ccfx2Aw\ncMMNN/CLX/wCgIyMDIkeHC+CcGxOOBwqFxINhJNsampq2L59uxjH8ANWjcdsNmO1WsmOPOlMAAAg\nAElEQVTIyGDGjBkAPPzww3KriAXUO9A0jaeeeor29namT58OhOYrWti/fz8QijL16NEDj8cj71kx\nJ4+nmmEymcRYKyat2+2W92cymejXr19c9uE5b6AUbDYb3/nOd4AQs+/jjz/m4MGDsqnS0tKYPHky\nI0aMiPAYYoVwCm5tbS3z5s2jpqZGNktHRwcOhwOLxSIhq5gzYv7/bdFkMtHU1ERiYiKvvPIKEEpo\nDxw4MG4hx2OhWGEq+Z+enk5HRwebN2+WuqwHH3ww7sQNtSmXLFmC1+tlwIABEq6KF2pra2lvb4+o\nS1NhKU3TxFMuLi7m0Ucfxe12yxpXB1NXlBEUzfinP/2pPPu+++7j/vvvP2nj93VqF0rtpasIBoPi\n4Pztb39j/vz5lJaWRug+qrGodd6/f39GjBjBlClTuPrqq4HQ7T2aFPPjjRNCNxsV8p82bZqMLVpQ\ntyBFHd+wYYMQoAoLC0/KKfB4POzcuTPi1pmYmMjgwYPjYqB0koQOHTp06Dgrcd7coOCoflRlZSWl\npaWSu4CQsOc111wT18S/yn8tWbKEyspKcnJy5Abl8Xh4//33mTlzpniQSsQzVqEj9blKB6+8vFyS\noSaTie9///tnLLyn6mfUDcnpdLJq1Sqqqqqkvi0aOYpThZrDL774AqPRyOzZs086b6k899OdT/X7\n27Ztk7zX+++/D4So2Vu2bOHQoUMSjqmrq+Pw4cMEg0FZ516vl46Oji6RgYLBINu3b5cyjksuuYSb\nbrqJjIwM+W6K1h0uXhteXqHGc7wxREMPU93AH3vsMQAWLlxIW1vbV9TKw8kKANdddx233HIL+fn5\ncquKtVismss//vGP2O12CgsLZW0HAoGonVGKFl5WVsZnn33Ghg0bGDRoEBA6D2+66aYT1jKptbdu\n3Trq6uoiwrQ//OEP4xZFOC8MVDAYpKmpifnz5wMh2Z6MjAxGjBghDLurrrqK7t27RyRsgePGzKOB\nQCAgieqdO3fS2dlJjx49pO5p8ODBDBkyBLvdLgvW5/NhNBpJTU2NiSFV3zM9PZ36+nqWL18uh05G\nRgZXXnll1J95MvD7/Xz55Zfs2rVLCgnLy8vZunUraWlpXHPNNcDRfkyqXkMhloeJKkQ9cuQIFouF\n2bNnn9TvhSuP22y20zqAlXFctGgRLpeLjo4OYed1dnbidrsxmUxSs5KWlobNZovIk3V0dAgp5nQR\nDAbZs2ePECIefPBB8vPzI9aoIiUcOXKErVu3AjB27FiSkpKki0D4z4ajq3tQGcF169aJAXc4HMd9\njtFoJCsrS9bULbfcQr9+/SJEkmO5njRN49lnnwXg7bffZujQodx9991Cnuro6IhaAaz6Pvv27eMf\n//gHgIRA9+3bh8vl4uabb5aiZeVcqPcJISWSQCCAwWCQ/NhDDz0Ut/zrOW+ggsEgxcXF/OxnPxM6\n6ZQpU/jRj36E1WqVBHtSUhKZmZl4vV5ZDDabDbPZHBNjoGkaK1euBEKL4ciRI0yePJkf/OAHQMgD\nVs9Wi8HhcEjLCbUAYrFZLBYLzc3NvPHGG/Lsvn37xl1BWd1433nnHf77v/9b1J3hqBc7fvx4+bmG\nhgbMZnOEDIzaVOEV79EinwQCAZYuXSpjHTRo0El9ttI3U+oJp5NwDwaDotm2e/duOVzVuwgGg9hs\nNtLT0yMKXtVzlYGaMmVKVAzU5MmThV6fmpp63APKbDZTUVEhhqG0tJS6ujrGjBkjrK/wdX3sTetU\nx6ieo2kara2tvPLKK7S3t3/l58LXitFopFu3blx33XUAspbideC2trby4YcfAiGH9Hvf+x5TpkyR\ncZeUlDB8+PAuF8AGg0FxPjdu3EhDQwOapsmaTElJ4e9//zs7d+6U3NvUqVOx2+1omsbChQsBWLFi\nBUajkYSEBJ5++mkgdJbGi8F6zhoodbB+/PHHPPbYY9TX13PxxRcDcO+999KjRw9MJpPopsFRuqvy\n0i0Wy3Ep1dF4+V6vV6rsS0tLGTx4MI8//rgcVsfzGu12Oy0tLdTV1UkIIhakAKvVSnl5OU1NTXKV\nHz58OJqmxWXhqc2jNkZRURFut5tAICDf12q1cvHFF/PQQw9F9PRyuVy0tLRISEQxx8JpsNFolRAM\nBmltbeWtt94CEG1Hh8MhSiTHgxLldTgcYmyTkpJOWR4pPOGfnp5OIBDg29/+thxcHo9HtAl79uwJ\nhJyMxsZGzGazEEumTJnSZcfDZDLRv39/qaU60WFuMpkYNmyYvJ/ly5dz6NAhunfvLus5EAh8pYfV\n6cxVeCTE5/Nx8OBBDhw4IM9W8xWuEKFo5k6nU9ZKWlqa3J7iIQKs0g8QciiUOopaH3v27MHlckVF\noUEZwrKyMqxWK5qmyVpobm6ms7MTl8vF9u3bAZg/fz5Op5N9+/ZJDZWaL6vVKsLbEydOJDs7Oy5G\nXSdJ6NChQ4eOsxLn5A3K5XLxpz/9CYC//vWvkt954IEHACSmfOwtRXlPqqDX7/cfN9kXDe97165d\nfP7550Aorty7d28yMzO/1uuw2+18+eWX9O/fX/IKsRJk9Hg82O12CVmNGjUq5rpaChUVFdx+++1C\nsVVK26mpqfTv3x+AGTNmcOedd5KQkCDfX+Wgamtr5T0mJCSgaRrl5eWiU3e64SLlafv9fg4cOMCC\nBQuorq4GQmGkxsZGFi9ezPe//30gdKNTjehUvvHQoUMkJyfTs2dPCWudroqDehdXXnklnZ2d9OnT\nR1rNK69fiepCyCvOzMzkggsukLBfV8gRCgaD4aTChKrWTq3d4uJiHA6HdEeGyBDf6bZbdzqdEaoj\nNTU1bNiwgZycHCEc5eTkUFpayoEDB+Tn1Lqoq6uT/KDVao1bB91gMMjq1atlz82aNYu8vLyIzs29\ne/eOWqhdfU5OTg7Nzc0kJSVJ2M/v9+N0OqUhIYSUWsLz83B0voxGo9zI7HY799xzDxkZGbpY7LHw\n+/18//vfl6Jcv99PTk4Ojz/+OOPHjweOLwOjEn9Lly6V0Null17KhAkTYjLO1tZWiStrmsaBAwek\nHgWIOHRVXHjHjh189NFH3H777XI4xQIGg4HCwkIsFovkFS6++OITtg+PJlRep7y8XAgsl1xyCcOH\nD6dv375iZEaOHInJZIowHF6vF03TMJlMYhC8Xi/5+flS2Ku+36lAPUPN16FDh1i+fDkHDhyQuiyl\nIPHWW29JSE3lMdxutwikqs6siYmJXQrPGgwGCT/fe++97N27l7q6Oml139DQgMVioaKiQta72+0m\nMzOT/Px8MYrRVOM+GWiaJiGjQ4cOMWHCBPr163fcnOrphvaqq6v59NNPJVRfXFxMa2ursBUhZLSq\nqqpwuVwRhtBkMtGjRw/GjRsHxLe9u9/vZ8OGDcIWvOWWW8QpVGtcyWlFA6qNz5AhQ1i1ahVHjhwR\nA+V0OsWxUs9W78lgMMgYevbsKWNUYb/9+/dz4MABxo4dG3NW9DlnoJxOJwcPHhSig2KXDBw4UP5N\n5TLCX57b7ebDDz/kpZdeEiORmJgYkziqwWBgyJAh8tk+n4/y8nLeeecdbr/9diDkufl8PrZv3y6J\n+KqqKvFojjVk0UZSUhJGo1E8pvb29tP2aE8FPp9Pmkuq75iSksL06dPJysoSRlF4TkC9R4vFIp6v\nKhBNSEggOTk5Ql/tVDxQlZfw+XxS1P2Pf/yD1NRUpk2bJhI069evZ/Xq1Xg8HmlE2dnZic1mw2az\nUVhYCByV8onGrUUdDLm5uRgMBnJzcyXHkpeXR1NTE7t27RIikN1uZ/z48RHq4dHE8ZQYwv+f1+ul\nvLyc5557DggdyDNnzoy6YovNZqOkpCTCSVUHrlrPfr9fcqoKVquVMWPGsGDBArlpxbOswu12U19f\nz5gxY4CjckF+v1/mMC0tLWpjUsoYbrebxsZGGhsbxTgqpyy8JMFoNDJp0iTmz58vjD2lJFFRUcGb\nb74JhPZhcnKyTpI4FsFgELvdzn/8x39IjZNKJC9btozFixcDR1sTBAIB8dJVHyij0SghiHAjEm30\n6NGD2267DQhJ23d2dvK73/2Of//730Ao8V1TU0NpaakskJ49e3Lrrbcyffr0mCsmJCYm4vV6pVZC\n3QBiDU3T2Lt3L0eOHJHDJC0tjaqqKgoKCr7ikYVvArPZTGpqaoTQqKJwJyYmnlaSO5xckZubC4Tk\nZmw2G5MnT5ab7PTp06mvr+fjjz9m6tSpQMggKMWBcO8z2htXia2GkzNSUlJwOBy0tbVJO4nevXuT\nm5sbMwkv5RxomibtMdT37ujoYNOmTbz11lscOXIEgOuvv56hQ4dGXXQ1OzubKVOmsGDBAnm2Gtex\nTpbRaBSn55FHHuG+++6LSx+j48HtdtOrVy/+4z/+Azh6+NfU1Mh+jwbBR0F9psVi4dZbb+WKK66Q\ntIPNZhNHUb2fkSNHRuwjBZvNxtChQ/n5z38OwOrVq+ndu7fQzyF2hl4nSejQoUOHjrMS59QNSlWB\n33PPPUyePBkI0cxVg7Fdu3YBSNw5vEDQZrNht9vp378/999/PwBjxoyJmeW3Wq38/ve/B0Ke9tq1\na2lvb5eW3eq5F110kYSRbrnlFoYNGxbR7DAWUNf7xMREybt4PJ64eJUGQ0hF3Ww2yw3K6XTS0tJC\nW1ubJJBVPix8TOHFlOGEjq6M22q1SgxezcMDDzwg7RrU8+x2OykpKdx9990SmjxeKC8W7zC8w6mq\ng+ro6GD//v10dHQIRbmgoICCGLYjUe/iyJEj1NXVYbPZhIDw7rvvsnv3bpxOp7Rr+dWvfhUTbUeb\nzcaFF17IDTfcABzVSfR6vRHqHUlJSYwePZoXX3wROKqgcqbUUux2OwkJCaxZswY42vlb0zQRsI7W\nbVOFmhUSExNJSEjglltuiXjOyb6L8MhTdnY2ra2t2O32mOegDPHIO5wkTnog4e3L1TW5o6ODtWvX\nAvD5559z5MgRMjIypB6ke/fu9OzZU8IgwHGvs7GAUlZ+//33I9hDo0ePZujQoREt6tVBFOtxlZeX\nc80119DQ0ADAvHnzuO2222K+4AKBAFVVVcybN08W/OTJk/nWt75FQkJCBNvrTB0kZyOUU6HyrA0N\nDZSVlbF69Wo5bO644w569uwZs4Jrdfg3NjZy4MABdu/eLYzYyspKEhMTufDCC7njjjuA2DFQVd5Q\nhaV3797Nxx9/TEdHh/R6GzRoEP369SMlJSWmRe+nAqfTyR/+8AcpwE5ISMBut/PLX/5SCA3R7ioM\n0fve6vMcDgf79u2jrq5O1Ge+gdhx2gM4Jw3UuYpww6oQfkuIV3U2IK3qFTPnN7/5DTfccEPc1SR0\nnBqOLTr1eDwyZ6crqXSq8Pv90gI8nPael5dHcnJyzAk+5yo8Hg8LFixg2bJlQIiY9MQTTzBt2rS4\nq/R3BZqm0dnZSVNTkziZ36BIrxsoHaeGYDAYQTHNzMw8I40AdZw7CJcWgpC3r9aPcrRipW15PkDd\ngsObN8aT5h5taJoWQY//Gpz2F9RJEjp06NCh46yEfoPSoUPHNyIYDEo4L7x+MJb0eh3nDfQQnw4d\nOnToOCuhh/h06NChQ8f5Bd1A6dChQ4eOsxK6gdKhQ4cOHWcldAOlQ4cOHTrOSpwXBkqp8urQEWvo\n60yHjvjhnJUNUAeF+q/X6xVlc7fbLWrLqj+MToE9O+DxeCI0EnXo0KHjRDhnaebhhsntdnP48GHe\neecdALZt20ZKSgrdunXjySefBBBJDh3xh+oVpP4M0W0roEPHuQ7lXAMxF4s+A9Bp5jp06NCh4/zC\nORniO1bNXLVNX7duHRC6QSUlJXHnnXdGtFY/k2EllSc71jM6k2PSNI3du3cDobbZ06ZNi2pHz3Co\n9hiapkkzyVgrp0cTgUAAt9uN2Ww+7dby5zsCgQBtbW2iMp6cnExycrIuQPwN2Lp1KzfddBMA3bp1\n49NPP41bx9qzHefkygnvYGo2m7Hb7SQlJYnYaXZ2NiNGjGDIkCERmyXeUGNU7afDyRwGgwG/34/V\napXDO97GyuPx8Prrr8uzp02bdsKf7Yp0v+oBpZ5jsVi+4jCcrZtRfW+n00lZWZm0b9ERCa/Xy9NP\nP83ChQulfXmvXr344Q9/yM033xxTte5whzVciDUYDH6ly+6xgrYGgwFN00S4FeLb6qW9vZ3vfve7\ntLa2AqGeVWqv6DhHDRQQscB8vv/X3tnHRl3eAfxzves7vdLrXekLhbZAYbyVOijTISLlJQME0clA\nFkeYGRPdFpZsCbqgRJJlhmwuWzKDDINjbkxQEMeLQQSZoEOwSsv7e99L22uv1977/fbHL8/DXa0T\nSu96Nc8nMZLLtb+nv9/ze77v368Ph8PBAw88AMDjjz/OhAkTyMjI+FLPsGggXgzxop45c4aTJ09S\nVVUlBaYY0ufxeJgxYwYAa9euDZtfE2laWlpkEsnYsWP/7/ye0M/vVFiFWo5+vx+HwxE2LhxuDZS8\nU6uq++ETKeLi4sjLy+sXi7c39zv0oI7kgSsO023btvHyyy/j9XrlbCCDwcCRI0coKyujqKgIiMy8\nI5/PR0dHB5999hnvvvsuALW1tTQ3N2O32+VQTr/fT2pqKsOGDePb3/42oCuutbW1pKen88gjjwD6\nuxCNOJCmaZw8eRKv10thYSGgj70R40r6i1CB7/f7+7Xr+oAXUKLlv9FolAIqJyeH+Ph4zGYzfr8/\nquvSNA23280777zDjh07AKipqeHmzZth4y3EdzVN48yZMwBUVFTwwgsvMHny5Ii7Rfx+Pxs2bOCT\nTz4B4Pvf//5tbcBQgXAnwkEkSbS3t9Pa2sp//vMfOV3Y5/MRCASYMmUKDz74IKDPlxH3IFQ79vl8\n+P1+eR+7urowm81h03ZNJlOfHYRiQKDb7ZZTSqP1ooZmqvp8PjnlF3qe5hsIBHA4HBw/fpz6+noA\n5syZQ05OTkT2k6ZpHDt2DIBf//rXxMfHs2rVKn7wgx8A+h47dOgQb7zxhpxiPWTIkD67f0II19fX\n86c//YnKykquXr0K6C7l1NRU4uPjycnJkdeeNm0aOTk5ckhoRUUFNTU1tLe3y2a40bKgPB4PmzZt\nIjs7m9WrVwMwYcIETCZTn8+GE0qLeG98Ph8ej4ekpCS5N3w+H263m/b2dnkf9+/fT1paGr/85S/7\nxQulkiQUCoVCEZMMWAsqVLtsaGigurqaKVOmAPo4AKFhhiZJRHJirViPy+Viy5YtbNmyhYaGBuDW\naOfMzExpSXi9XgKBAG63W46B//jjj/nrX/9KQUEBVqsV0LW5SLiVXC4XVVVVlJSUADBs2LAv3ZvQ\nexxq8gtrRVin/y++IIa0ie8K62nPnj1UVlYC+vhyv9+PpmkyjrhgwQLWrl3L9evX2bdvHwCNjY1o\nmsbEiRNxu92AroVaLBbmzZvHuHHjgL5z9QWDQWmJ2O12ioqKwmJnX3edvhq57fF42L17N9XV1eTm\n5gIwceJEkpOT6ezslK7kbdu28cUXX9DQ0CAnnKakpLB48eIw16mIz3i9Xhn/7M0anU6nHO8eDAZ5\n9dVXmTt3rrxWIBDAZDLx2muvcfToUQAeeeSRPkmO0TRN7ql3332XM2fOYDAY5H4uKytjwYIFYdZj\nqGUkrCWv18v58+cZNGgQw4YNA752+F6fceHCBerr65k9ezYzZ84E9LNCvDN9/d57PB4+/vhjAHbs\n2EFDQwP33nsvQ4YMAfR3++zZs7S1tfH+++8D0NbWxqRJk1izZk2fruV2GbACShAIBGhtbSUtLU1u\nfGG+h2ZciYce6cyxM2fO8Oabb9LZ2SlfluLiYmbNmkUgEKC6uhrQD+orV65w+vRpamtr5c97PB7q\n6uowm82A7qqIhIBqb28nLS1NHngJCQlSgHd34wnTX3xP3NPbvZdiNLn4ma6uLpqbm+XB6vP58Hq9\nYfVS27dvZ9++fTKhAvSpv4WFhfh8PnntxMREHA4Hzc3N8nt95c5yOp2cPXsW0J+h3++X64NbB173\nw11kKYrvpqSk9GpN4uc3bNjA1q1b6ezsDBOOwWAQg8EQNmI9EAjI5BvQszNFvK97HPFuBKemafz9\n73+Xz3XWrFnMmjUrLFZhMBjIzc3F5XJx4sQJAObPny/fz7tB0zQZW8rOzqa8vJzMzEwmTJgAwKhR\no+Tcqp5cocI9eOTIERwOBz//+c+lgIpWnHHPnj3U1dVRUlIi6zTFc42ESzYYDHL9+nVAd23GxcVh\nMpnkvQBIS0uTcSfQ93JeXl6fPLPeMOAFlLCMhg4dSlZWFqAfWsKCCvW5dnV1kZycHJGHLw6T//73\nv8TFxTFixAipFS1YsIDU1FQAhg4dCkBlZSU1NTXcd9998hCMj48nPz8fn88nYx9iE/U1wWCQoqIi\nbDYboGcTxcfHS0EOyJiL0WiU6w9dy+0ccOJ3iMB5SkoKkyZN4saNG6SnpwP6y9LS0hIWLxQ+c4vF\nwtSpUwF4+OGHGTFiBJmZmfK7TU1NXL58mUGDBkmtOCMj466tFvEyi+dlsVgIBoPU1dVx48YNeZ1z\n585hs9nIz88H9OfV1dVFXV0d99xzD6DfszuNXWmaJq/z1ltvYbfbwyzMhIQEBg8ezPjx46WSkZ6e\nzuHDh7ly5QqZmZkALFq06EveBNCfixBivcHv93Pt2jW++93vAvDKK698qfjaYDBgNpuJj4+XcdbO\nzs4+E1BiT86fP5+urq6wv0kkOXQXysFgEKfTyfr16wHYu3cv06ZNY+LEiVErexAKYE1NDaNGjSIr\nK0uuUwiLu3k2PWEwGEhKSqKgoACAmTNnMmPGDKZOnRp2bZfLxaVLlzhw4IBca2lpab+VwwxYASUe\nZG1tLS6Xi9zcXKnZd9fiQD/Idu3axaxZsyguLgZ6DjT3Bk3TsNvtgB5MLy4uRtM0ueHff/99Kioq\nuH79usziS0hIIDExkeHDh0vXFIDNZiMrKyvM8utrNE2jtbUVh8PB7t27AcjLy2PatGlYLJYwl0hf\nvLRxcXFSy8/MzCQtLY2hQ4fy2WefAXDs2DF2795NY2OjFDwpKSkUFxezatUq7r33XkDXlBMSEqSV\nIH7f2LFjcTgcfdotxOv14vf7ZfaZ0Wikvr6exsZGrly5Auh7qra2lvLycnnPxP0rLi4Oc3W53W4p\npG8XIXAtFgsOh4OsrCwWLlwIwA9/+EOysrKkMgb63ktPT2fr1q3MnTsXgBEjRvRoRdztvjcYDFit\nVsaMGQPwlRmgPp+P48ePS+WxrwLtoUoPIPeFQFiXEJ7tV1NTw8aNG9m7dy+g77M1a9ZIwR8NxDqz\ns7MxGAw4nc6wtScnJ0ckFGEymaTSNGrUKDIyMsL+boPBQEpKCk6nk9bWVvn5vHnz+q0MRCVJKBQK\nhSImGZAWVCAQoK6uDtBdC/n5+UyePLnHgK9wV23fvp3Nmzezd+9enn/+eUCvdxAuv+4W150irj19\n+nQefPBB7Ha7TOHetWsXX3zxBXArbpOeno7VaiU7O5uuri5A19pLSkrIyMiQFkckTGvhvqqoqJBJ\nAK+88gqlpaUMGTJErrEvtSbxd4jC5ISEBOmG8ng8vPfee2ExwsGDBzNmzBgKCgqkNSliByaTKczV\n6PF4wmIqd7vuQCBATU2NdE+K32mxWAgEAtJ1a7PZiI+PD2t+21NMqjcBb4PBIIuCV6xYwY0bN1iw\nYAETJ04EbvUyDI0ZGgwGrl69SlxcHPfff3/Y9yKBzWaTFlRPBAIBNm/eTEtLi6w76kvXVXd3YneX\nfmdnZ1hcUyTcVFdXM3nyZACWLl3KpEmTomohiDVqmsa1a9c4fvy4jFdbrdaIuRoNBoOse0xJSZE1\ncmL/iH9fv35dWu9Go1G6BfuDASeghHvqxz/+MaBndj311FNfmXkjNufrr79OXV0dgwcP5vDhwwBc\nuXKFjo4OZs2aJWslxIO7U0Q8paSkRFavi83Q3NyMyWQiOTlZuqYCgQBWqxWfzycDyElJScybN6/X\na7hdDAYDpaWlzJ49mw8//BCAuXPnYrVaI14DIl6C+Ph4GWAXtSgiUQL02pZDhw7R2dnJggULABg/\nfjw2m424uDh50CUmJuL3+2UM4m4IjQ189NFHjBs3TiaHdHZ24nA4yM7ODktguZ1r9lbJEC6shx56\nSB4uoUqYEE4iXnnw4EH27NnDyJEjpeCI1GEnlBzhsh49erR084n7WF1dzTvvvAPA4sWLgcgmIIQq\nB+K9On78uIzlOZ1OTCYTZWVlrFq1CtBdxNFsuaVpmuwaUVRUxMGDB2lvb5fnQqQLYkMVKZGdGyow\n3W43+/fvl2uwWq3yu6pQ9zbwer289NJL8lBPSUmRKcrd03qDwaBMURbZTOPHj5fp6O3t7Xz66adU\nVlbyzDPPADBy5Mg7TkoQmgiEJzUIf29JSYk8REI7SZw6dYq//OUv0oqxWq2MGzeuTyy6r1vvkCFD\n+MlPfsKcOXMAPUHD6XRiNpujEhDVNI1z584BehJAR0dHWBGzx+Ph5s2bnDx5UsYtvF4vI0aMwO12\ny+QOoSGL4kbx9/UG0VH6tddeo66ujsuXL8vrXL16FaPRyMqVK7FYLL37o+8Q8RxEbK17AbIQTp9/\n/jkAzz33HK2trYwcOVKWKUQKcd+FALp+/To/+tGPsFgsMka3fft2AoEAc+bMoby8HIhst4/Qw7ap\nqYlDhw7x6aefcvnyZUBXClNTUykuLpaCPlop5QKPxyPX2NnZSUNDAx0dHTJxJJpCQJyZQokPBoO0\ntLSQkpIi1+hyufD5fF+yfKO1zgEnoDo6Ovjwww/lTTWbzZjNZlwuV5gWEggE2LRpE7/5zW8AfXMO\nGTKEFStWMHr0aADq6urYs2cPVVVV8qDOzc2VGnJvCH1wQjMzGo0yjVtsxNCMGWFJDBo0iBEjRvTo\nJupLxIssrA/xmXA1RhKRdenxeDh//jwQPmogFNEZoaqqSn7W2NhITk4Op06dAj9WGV0AAAvpSURB\nVG719ps0aZJ8qXqjEYtmsKDvn6KiIlnCAPo+q66uls8KIv+Sit/fUwmASCmvqamRLuurV69SUFAQ\n5lGI1BpNJhN5eXn885//BOD06dPs3LkzzEJIS0tjypQprFy5MioNY7tbUHa7nczMTLm/mpqa6Ojo\n4MCBA4wcORKAZcuWRTVDze1209zcDMB7772H3W7H4XDIjMRo7anQf4eeU1arlfb2drnPzGaz9Oj0\nhwWlkiQUCoVCEZMMOAvK5XLR0dEhpX58fDxdXV00NTXJKumdO3fi8Xg4fvy4tBDMZjPr168PS+nO\nyMigra2NxsZG3njjDUA3wRctWnTbwVxRrxPaG/CrNI3unx87dkzWHwE89thjpKWlRUVTMRqNpKWl\nyfqdc+fO9WkPu+4Ijcxut3Pw4EGuXbvGpUuXAEhNTSUxMVH25APCNOHQhr9tbW3YbDb5XEUnhfvv\nvz+s88Wd3MNgMIjL5aKmpgbQ+9elp6fj8Xjkek6cOIHT6ewXLbKnawaDQXw+H1euXKGiogLQY5iP\nPfYYY8aMifg6jUYjTzzxhHznDh8+jN1ux+VyyedlNps5ceIEmzdvlkkAkS74FNcuLCzkd7/7HQaD\nQRan3rx5kxdffJGKigp++9vfAvC9732PzMzMqDzXYDDI+fPnpefg888/JysrizFjxvTLSBJhFYWe\ndQaDgYKCAnm+Llq0KKrd3bsz4ASUzWajvLxc1u/YbDY8Hg/JycmyFunChQvU1dXh8/lkoPkXv/gF\ns2fPxuv1SldWTU0NVquVp556iscffxzQXRcul+u2BJSmadJNJ0z0vLy8HjOnutcz1dfX89xzz9HZ\n2SlrbVasWBHRgG1oPUgwGJRxJ9DdH1lZWRETUMI1tmbNGg4cOCBHv8MtoW40GsMyikQjYOGi0TQN\ni8XC6NGjZV0N6MXPY8aMue0WRD2trampScZtRHKLz+eT2aLV1dWcOnWKhx56qLe34K4InX8m1nbx\n4kWef/552VFh7NixrFy58raTN+6WlJQU/vCHPwBw9OhRDh06REdHh3SR+/1+/v3vf7Nv3z62bNkC\nwE9/+tOIHsbi705MTJTvsFiPz+dj2bJlnDlzRrYhO3LkCIsXL47K/WppaeHtt9+WAio+Ph6bzSaL\nqaNJTy47Ubv5ySefyJj58OHDo7qu7gw4AWU0GiktLZVxCYvFgsViQdM0HA4HoCcg+P1+kpKSZELE\njBkzaGtrw2g0Si3u2rVrPPHEExQXF4e1GrnddveaprFt2zY2btwo01affvpppkyZ8qXgq4j7NDU1\nAXprGLvdjtls5o9//COgC7dIvSihwdBAIEBcXBwul0tajiUlJREtVhQb/vz587S2toYVJgphlJiY\n+KU2SoFAQMY0PB4PgwcPJj8/P6wzx+DBg3sd7NY0DafTyenTp+Vzz8rKwmw2k5qaKrt8NDc3k5OT\nw8iRI6NeVS+spdBOG3a7nZdeeiksPvfoo48ydOjQqK3PYDDI92bu3Lk88MADJCUlyeu3t7fjdrs5\nePAgb7/9NgBLliwJUy4ivT4grFRg4cKFvP7667Jbd3V1dVSGmfr9frZs2cLFixdpaWkBdGXYYrHI\nbiX9jUj2Ci3Sveeee/rNeoIBKKBMJhMLFy6UD7miooK6ujq2bdsmZ8E4HA4SExPJyclhyZIlwK3e\nXMnJyXLDLl26VD6U3jwEn8/Hpk2bqKur46OPPgL0JpXFxcVhk0R9Ph9VVVVs2bKFN998U65x0KBB\nvPDCC3LERCS1qNCGrSIN/oMPPpCbcfLkyRHLaApt7Dl9+nQuXbqE0+mUVkFSUhJFRUWUlZVJi85g\nMHDs2DGZWQi3LL+2tjb5Uqelpcmsx94eMm1tbezbt0/ei6lTpzJnzhy8Xi9jx44FYNq0aaSlpUVl\nTpAg1Gpyu90EAgHpJdi8eTP79u0LmyW0evXqqGvi4p6bTCZSU1PD3qWMjAzWrVtHS0uLzF6tr6+X\n5QzRRrizhg8fLpUet9sd0Vlx4ne3t7dz48YNOjo65N5NSUmhvLy8T1pz9RWaphEXFyc9T8K701+o\nJAmFQqFQxCQDzoIyGAykp6dLLeSDDz7g1Vdfpa2tTaYJm81m7rvvPp599lnZ3Vho2T35Xbs3lPw6\nQuMAomu10Gx3796N1+slPz9fdinftWsXVVVVdHV1Sc3RarWyceNGlixZEnGtV7ixhO+7q6uLixcv\ncuzYMdnnbvz48RHV4oRL7plnnmH58uXU19fLz2w2m4z7CEtr//79sumqiF/V1tbS2NhIW1ub9I1/\nVUeFOyEnJ4elS5dKF2h+fj5Dhw4Nm/Dbn8kRomGvuB8AVVVVsovF3/72N+DWPe4P4uLi5EC8UOtI\ndAwR1ml/dcUGZHlDa2urvLcdHR0RteaEK3vnzp1ygoC4dklJCcuXL49ot487RYwvEu7+/p7uO+AE\nFOjCRnQXuHTpEhcuXCAYDMpD68UXX2T+/Pm39eDvpommaL547tw5mXhx4sQJTp8+Lauy4Vb8JSkp\niYcffhiAl19+OaqmvdfrlQ0ya2trSUpKorS0lBUrVgB93z05lNBMoWHDhjFs2DCZ1SUIBoPyP9A7\nD0yZMoUjR47INlFiOmp+fn5YYaNwK/XGVSMaZIrWQOKz0P/3N2Id1dXV7NmzB9AFeWFhIUuWLJFK\nWH+uV2SztrW1Sde23W5nx44dVFVVyViVzWbrt3X6/X6OHj1KdXW1fCfLy8sjKqDEGbB3714qKyvx\n+XyUlpYCsGrVqphy7wGyy4sQUOI+9RcDUkCF9pRau3Ytq1evxul0ysFbkdZIxO82mUysX78eg8Eg\nK+qdTmfY3B7Q2yAtXryYdevWydEI0fTBGwz62APRjulf//oXxcXFPP3001Ef4/xVz8VoNIbdE6vV\nSnp6OllZWUyfPh3QLb+8vDxycnKkwAv9md4+8/4aJXAneL1eamtruXDhAqDvqZkzZ/Lkk09GvRvC\nVyEyC9966y1AVx5rampISEjg0UcfBXrfSux26T7LzGAwSMv4wIED/P73v6exsZFvfetbQOSTAIQF\n1d7eLr0sosNMQUFBTAknCE9YAsIK0/uDASmgQjGZTGRmZsrGo9EgVEDl5OTw5z//mV/96leAbkEJ\nV0dZWRmg12QkJib260GYkJDAsmXLAN2tUVZWRmFhYUwdzt27cMTFxZGbmyunwwaDQdlotqeK+G8q\nmqaRkJCA1WoNO9zmzJkTdQXjqxAdPZKSkmQ9WUNDA8OHD5ep1BD51kKhpRRer5f6+no2b94MwNat\nW+nq6sJiscjWZkLRjRTCEnn22Wf5xz/+wfLly6XCFUvvnkDTNBITE6WHQpR+9Nc7Fnt3SKFQKBQK\nwBDJFMs7JGYWcjeE3s9Y0+xD05b7szr8ThCxDbg1hC50REBfDZ2MZYLBIB6PRxafgz7sLjU1NeLd\nr+8EUU4gSkA6OztJSEggIyMjar3mQpucnj17lm3btrFjxw65npycHFavXs2TTz4JENVBhQMBv9/P\nhg0bZAPidevW9cW04V4/9AHv4os1YuWw6InuhYsDgdD7KbrWm0ymfs8uigahrZtEc1bhFhIu0Fja\nbwaDgfj4eLKzs/t1DaDHoYV7+Gc/+xkA3/nOdygsLCQ3N7dfWgsNBOLi4igvL6eyshLQC9TFNOj+\nSB5SFpQi5gk9qNva2ujq6oqZ6vtIIqwBMU6ku0CKJeGk+GagaRqdnZ1yBFBWVpaMc/a2lRh3YUEp\nAaUYMIjUfTG+5JtOLLuLFYo7oNebVyVJKBQKhSImURaUQqFQKCKJsqAUCoVC8c0illJZlJNdoVAo\nFBJlQSkUCoUiJlECSqFQKBQxiRJQCoVCoYhJlIBSKBQKRUyiBJRCoVAoYhIloBQKhUIRkygBpVAo\nFIqYRAkohUKhUMQkSkApFAqFIiZRAkqhUCgUMYkSUAqFQqGISZSAUigUCkVMogSUQqFQKGISJaAU\nCoVCEZMoAaVQKBSKmEQJKIVCoVDEJEpAKRQKhSImUQJKoVAoFDGJElAKhUKhiEmUgFIoFApFTKIE\nlEKhUChiEiWgFAqFQhGTKAGlUCgUipjkf1WhYmP1HiTbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71400d7f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Interpolate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF8ZJREFUeJzt3WesVNXbxuEHe0FBEBUQxIYNGygWVIwVDYiCxERjNzaU\nEEPEbjQxGqMfLLH3FjQRYo0lEAQ0oihRLFgRBQVRKYK9/D+8rzf3Wp49zqlM+V2fbmD2nMNes/es\n7GeVdn///XcAAAAAERGrrepfAAAAAJWDziEAAACEziEAAACEziEAAACEziEAAACEziEAAACEziEA\nAACEziEAAACEziEAAACEziEAAACEziEAAABkjTb8WWzi3PratcB70E6tizaqfLRRdWhuO9FGrY82\nqnwNthFPDgEAACB0DgEAACB0DgEAACB0DgEAACBtOSGlJv3998rxsn/++afye++9l7xu4403Vt50\n002V11xzzVb87QAAABqHJ4cAAAAQOocAAAAQOocAAAAQxhyWyccW/vrrr8pvvvmm8qxZsxr8+4iI\n/v37Kw8aNEi5V69eyu3atcTau0D18+vt999/b/A1/vc+dnettdZqvV8MAAr89ddfhX9evHix8tKl\nSxs8vkePHsmf11hjZRdttdVWPstri74CTw4BAAAgdA4BAAAglJXL5GWu3377TdmXrLnpppuUFy1a\nlBz/4YcfKq+77rrK3bt3V6Ychnr1008/JX++/vrrlV9++WVlv/YOOeQQ5TPPPFOZoRqoBr70mX+u\nf/75Z+Uff/yxwRwRsXz5cuXevXsrd+jQQXn11VdvmV8WCS8X+3Cym2++OXnd1KlTlf0e5yXigQMH\nKu+zzz7J8X369FHeYYcdlDfYYANlH1LjQ23ytm/ssnk8OQQAAIDQOQQAAIBQVjZeOvYckT7Cf+ih\nh5RvuOEG5R9++EE5n7U0e/Zs5RkzZigff/zxzfiNgeri15VfLxdeeGHyuqeeekrZyzFeMttyyy2V\nu3btqlwvpeT8HuVlyhUrVih/8803yh999JHy+uuvr+xlroiIzp07K/s5979fZ511lPMSlrdBvbRH\nObxkfN111yk//PDDygsWLFD+448/lP18R0S0b99euVu3bsq77LKL8pVXXqncs2fP5HjapXH8+rrv\nvvuU/RznpdshQ4YojxgxQtnba6ONNlL+9ttvk+N9eNq8efOU/X7n12dLtilPDgEAACB0DgEAACB1\nX1YumoXsC11HRHzxxRfKH3zwgbI/3t1www2V/RF0br311lOu90f7fv7zUnxRaarez1m18Tb2xV99\nRrLP+o+I2HPPPZU33nhj5aOPPlrZSzb1MtPfrxEvHUdE3HbbbQ1mv68VXUdrr7128l5equrSpYuy\nlya9TObtFZGWrL1t6u3aze9pzz//vPKNN96o7G3pJf7NNttMecCAAcl7eRu99NJLyg888IDy448/\nrpxvzOCzX+utXcrl965XX31VefTo0cp+7YwbNy45ft9991X2oRd+vv1ndOzYMTl+0003VfZ7p/ch\n/H2LFs1uCp4cAgAAQOgcAgAAQNrlM95aUav/oPz/4o/0fdZX0d8vWbJE+ZNPPkney0vJPsvSyyc+\nE+27775LjveZRl4quOqqq5R9JmATtURtoM0+EBFpyWvhwoXJv/m5LVr0symKStltVMauujZqCj/H\nPlv2nHPOUfYZ/L6Ib0RatvF/82unuZ+DEiq2jfwec8011yT/ds899yj7ddWpUydlX1TXy71eooxI\nF+r3++L8+fOVfRiNz5CNiDj22GOVt99+e2Uve7XAddXcN2iVNvLPvu+nGxHRv39/5c8//1zZz9Ej\njzyivOOOOyr7uct5uwwfPlx55syZyqeddlpyzO23317WezdTRbZRub7//ntlv3Z89ZJnn31W+YAD\nDkiOb+xnPO/D/PLLL8p+TfvM9RbYf7nBF/LkEAAAAELnEAAAAFL1s5X9MWy+P6vPMPbHwL7/oPPX\n5LOV+/Xrp+yPdL0c46VPL6VFREyfPl3566+/bvB39plKtbwnprfZ3Llzle+4447kdV4OO+mkk5R9\n9qTPFCuaAeZDByLShUa9rOw/r9QCv+Xwn1+PMwGXLVum7GUuHzpw4IEHKg8ePDg5fuutt1beZJNN\nlFux/FWx/LPkCyRPnjw5eZ3PePUyrw9dKSore/kqIv3MemntlVdeUZ40aZLylClTkuN9luW2226r\nXA/t5/ebBx98MPk3v/d7G/m59EWRy713bL755sqjRo1SPvvss5UnTpyYHOOfl3xYQT3z6+3cc89V\n9v7BY489puyl5Obe6/OyspeJfahH0fAMFsEGAABAq6BzCAAAAKFzCAAAAKnKASBel/fxY/mYNR8f\ns/POOyt3795d2cczHX744co+fjD/mUV1fX+Nv29ExH777afsy+L49HQfC1nLYw59POdzzz2n/Npr\nryWv83Pgq8P7eEBfgd7/3pcXeOedd5L37dGjh/Jll13W4M9r7ury9TjO0D/LI0eOVP7yyy+Vjzvu\nOOWhQ4cq+xI1EemyTr5kTT2eV+fjBPPdFHwnmYMPPlh5t912a/AYP5e+LE1Eei/zJaX8fX1c9YwZ\nM5LjfQxbvbWfL/3jO5REpGMxx48fr9yUcYauqC297bbbbrvkmDZcxq6q+DwA/1wfeeSRyt5XaM3P\ntH8nNba98tc39vfkySEAAACEziEAAACkKsvKvuzCddddpzxhwoTkdV6m/fPPP5W32WYb5WOOOUbZ\np4rnZd2iR7JFu6344/z8vdu3b6/sywn477iqlkJprZ/r/7fPPvtM2R/b58sp+O4AvnzNnDlzlB99\n9FFlL1/6JvNeiouIGDRokLKXmFt494a64J8XXyrDdz8ZMGCAspeVvZSc7yTRkhvI1xK/RvxcRqTL\na/jyM0U7Ovl1kX/evV09+1Isnv36jogYOHCgcr21n9+f/NxHpEs2bbHFFspNud8ULeN2yy23KPt3\njZdFI/79HYX/M23aNGX/Tj/vvPOUW3LYl/chPOd/Lromy10eqrHf7fV11QIAAKAkOocAAACQqikr\n+yNR35j8/vvvV85LGz179lS+/PLLlX2Gq5dWmvKo2Esmnn3mbET6+/vrvOzw8ccfK/vq+W05c7m1\nyqk+k9VLvj5EYK+99kqO8bKZzx73ktmPP/6o7LP/vLRzxBFHJO97/vnnK/tMSjSeD4t44IEHlH1m\npJfxi2Zl5u1Au6zk58lLgb7zTER6bu+8805l3zHj9NNPV/Z2ye8xfr/yVQR8GIeXlW+66abk+G7d\nujX4+9cqP19elsx32vLZ3uUMVSo1Q9XvnTfeeKPyvHnzlH1IxymnnJIcXw+71ZQjL+W+9957yn37\n9lX2VU6a+5n2n+ntmO/e5telH1M0i9nbNO8PNXbYFE8OAQAAIHQOAQAAIFXzXNlnY3mJ2MuK6623\nXnLMWWedpeybY5ezaXWpBST98a6XS0ttgO2l5EWLFil7mcYXL/UZ1fmC3NXCz6GXz/2xvbfZkCFD\nkuM7deqk7G3mC4z7efXFZ/3YSy+9NHnffPYyypdfFz4UwkueO+20k3KfPn2U/Xr1Wet77LFH8r61\nvAh8c/h9Jb/f+WxUn91/6qmnKo8dO1b5008/VfbZxRHpAtdPPPGEsg+X8ZUivI3z37MeeAnPh80s\nX748ed2CBQuUizZA+O6775S9LO33zYiI6dOnK0+ePFm5X79+yldffbVyPju53tqoSF5+9Wtn2LBh\nyr7IfNHKIvnMfP83b2/vA/iwDZ8dHZGupuErnnj/oKjEnA/NYRFsAAAANBmdQwAAAEjVlJW97OH7\nKTtfNDki4oQTTlAuZ/ajl4vzR83OH/36o2IvuZSa/ed51qxZyl5yGDFihHK+J2a18P+nn7Ott95a\nefvtt1f22WA5L6/4ufWyppcpne9n2ppW1cLlbSmf2edlYp8V7isFzJ07V9lLlF6+Oeigg5L3rdXz\n15q8pLXnnnsq+57zPnP5rrvuUr733nuT9/LFk32G7RVXXKHsw13qvb38e+Crr75S9j3hI9JSvL9u\n2bJlyv494KXM/DvMV23wYRy+0LYfX+9tVCS/p/Xu3VvZh3r5+ff7nredrxgQkX5vffDBB8o+DOCj\njz5S3mqrrZLjvR/iQzf8fb3c3JJtzJNDAAAACJ1DAAAACJ1DAAAASEWPOfQxXF6vL5quff311yfH\nd+zY8T9/hi8h4ONGFi5cmLzOa/+++r/X+0stv+G/s6+IvnjxYmUfQ+I/v1rHHDofi+G7v2y++ebK\n+dIcRefz559/VvbxOW+99VaD78uyKC0nH4vrY2R8zK8vXfTss88qv/rqq8pDhw5VzpeAQPP4+ezf\nv7+y70Lk4wzzdt1iiy2UTzzxRGUfZ0ibreTLY/kyI2+//XbyOh+r5mPN/Hz7GM/OnTsrv/POO8l7\n+X2ta9euyj7uzJeAy3ftKtohxb+ravXe6f/H/LPfq1evBl/nfYXnnntO+Z577lH28fQR6ZhP/4x4\nv8Hvobvvvnvh8X7tFbXr2muvHS2FqxsAAABC5xAAAABS0WXlop1InD9S9cfBpd7LHyP7+/pq9h06\ndEiO98fr/ki4KaWV2bNnK/suBStWrFD2qe6+u0u18hKGl5X9UXs5yw1FpI/OfakIf9Tuy6o0t/yV\n7wpSpGhj9GpXagcA363Gr6U5c+Yov/zyy8pe8vIdcShRtixvM99xw5d+8nNeameHd999V9mX06DN\nVvLP9ejRo5XzXUn23ntv5UGDBin7sBt/L7+m8mFSM2fOVPahSj4kyZfL8bJoRPod5/deX3bMf/9a\nam//fPv3bkT6//fvJ19Cb/z48cq+I45fXxHp99DIkSOVfRiAD6fKv2uKysdF/RGWsgEAAECroHMI\nAAAAqeiysj8qL3pc6o9hH3/88eTffIcUfwxbVD72nM/k8uOb++jWd/Lwmbf+2N83vq9Wfp78fHoJ\nw/++1Hktml3mJTN/PH/YYYc14Tf+b/475iWAWt2BoFRZvWg2nw+X8N0b/Bz5jOZ8l4JaKsuvCr4K\nwlFHHaXsQ1p8d6IzzjgjOf6NN95QHjdunLKXGX0nDi+/RdTutVDEP6/+eb/ooouS1/luTX4fLLqv\n+Pv27ds3eS8fRuUzmX2Gq8+c9u+9iLSc6j/Td9/IV5CoFV6u9ZnHEemM42233VbZz4Wf41Irlvgw\nDL/efGiUt72XtCPStij6Ds2vvZbCk0MAAAAInUMAAABIRZeV/RGtLwa6aNEiZZ/NddtttyXH++K7\n/kjYH+2PGTNG2R8nN7fMlZfilixZojxp0iRlf4zsj4p9Vlut8VlvTSk/+bn1RWa9jX1h0XLL1c6P\nKTo+//taKqX5efHPaD7j0cv6vlC9z770BX59uITPaM6vNzSet9Nxxx2n7CXHzTbbTPmFF15Q7tSp\nU/JeXoq+4447lH2jAZ8te/HFFyfH57N064l/V3j5MaLxM3799XmJ189x9+7dG/z5/p2St4mXVmfM\nmKHsQ518NYJqb1O/p82aNUt5woQJyet69OihfPXVVyv7uTz11FOV/bt9ypQpyXv5rGQ/vuhzkN8H\nfWMMv4+21gxlx5NDAAAACJ1DAAAASEWXlX0Wjj/Gvfvuu5X90bgv+BkRMX/+fGV/JO57V/oClj7D\nzx/hRqSPgX0xSs8+u9pL3xER11xzjfJrr70WDfEZTIceemiDr6kF/njfc7mPx73s4e3fu3dvZd9b\nudT71lIpuCV5ecP3g/WZ9hERy5YtU/YZyvvvv3+Dx/jrfbYks5Obzxfgff3115X93uX3Ti8x52Wu\nnj17Kl9xxRXKPlzjlVdeUX7yySeT448//njloj18a5XfU/Lz6ist+L+V2uv3H6VmpZb6mf8otQLH\nfffdp+x7+Pr3m39vVvuC2H6+33///eTfNtxwQ2U/R/797DPFd9ttN+V58+Yl7+XDNXzYh7eX32u9\nDxGRDhfwtmiL763qbmEAAAC0KDqHAAAAkIp+3u+Pri+88EJlnwn52GOPKXtZKyJ9dNy+fXtlL5P5\nApg+2zJ/tO/lS/+9fDaXL1jpOeLfJe9/+CPsk08+WTmfPViN/NG3P54vKivnimbM+mxvX9h1+PDh\nyrW6eGtb8c+vlyvzPc691LLffvspe3v7nqQ+3MLLXNVeploV8nvUgw8+qOylKi97HXTQQcqlznnR\nLFm/R/mivvfee29y/MCBA5V9iEe9DR/Ivwe+/vpr5S5duij77FM/9+XMcI0or8yYf158ofMXX3xR\n2Wfr+v7n5e4xXw38+sj/Xz70pWgYgK+ecvrppysPGzYseS9vFx8O5fdR/27LS/8+HK6th2dwRwYA\nAIDQOQQAAIDQOQQAAIBU9JhD17FjR+ULLrhA2cfAjB8/Pjlm+vTpyj6Gz2v3zzzzjLIvfZNPKS9a\ncuWHH35QLrUavS8b4dPgfayOLx9Ra2NzmjumzMdl+LgdXw7lhBNOaLGf1xQ+PqUa288/4z5G9v77\n71fOxzb50ia+w5C3kV8jO++8s/LQoUML3xf/LR8rNXfuXGVf9qRoLGhT+Pv6fSzfCcR/F9+Rqhqv\ni+bw+1NEuiuNXy/9+/dX9u+q5i7D5fck3xUkImLs2LHKPs6uX79+yocddphytbedn69ddtlFefDg\nwcnrvH/gYwP9XPq152NyvQ8REfHEE08o+/J4PvbXx+TmY+VX5VhsnhwCAABA6BwCAABAqqas7I+E\n/bGvlyzOPvvs5JgzzjhD2R8P+zR2n6rvq/z78h0RxcvXeGllwIAByn369EmO9zKzr3ReL7sHNLds\n6Offl4fwEoivJr8qVPtyLF42ef755xvMv//+e3KML+ngpTEvS3uZzEvJO+ywgzJl5cbLP299+/ZV\n9l2YFi5cqPz9998rFw21iShuD793+rJE+f3Sd2LxpXR86aOin5GXy6v5s+HnKyJi8uTJyuPGjVMe\nOXKksl8jPlSp1HeFX7teyr799tuVvU0i0vbr2rWr8q233qpcaleWaubl2/y72ndNe/TRR5X32Wcf\nZS/DT5w4UTkf2rZgwYIGf86RRx6p7H2DSvqsV/e3GQAAAFoUnUMAAABITdU080ey/kjes9t///2V\nfVZfS/x8NI+XZLxs0qFDB2WfTdfWJZBaKn9FpDPCfWail/F95nFEuvvJnDlzlHfaaSflY445Rvno\no49WrtWSVVvJy8qjRo1S9l2kJkyYoHzJJZco77777so+PCMiHS7jJTgvk1177bXKM2fOTI4fNGiQ\nspeSixStBlGN/PfPd7ry68dnD59//vnKd955p7LP7s9L1H69zZ49W9l3APNhIPnnxd/bVyTwYU+1\nykv0vspFRHrOfMUTb5elS5cqL1myRDnfQWqDDTZQPuqoo5R9hnKlft55cggAAAChcwgAAABp14ab\nadfOrt2VqyWeT1dMOxXNwPMN471ktffeeysXDSOoABXbRn6+vXw4depUZV80PiItDe+6667KXqrx\nBewruF1cxbZRyR9o93Ivb3355ZfKTz/9tPK0adOUfXhARDqswGdTfvLJJw3+PF/APyIdSnDmmWcq\nlzNbuRGa+wat3kZ+TUWkpeSLLrpIecqUKQ0e46XgvGTp589L/36Ou3Tpouyl/oiIMWPGKLfiQuUV\n30Y5P/8+u99n5PsKAD7MyYdzRKSbBPjrKmxliwbbqKJ+QwAAAKxadA4BAAAglJVrS1WWw4r47Dyf\nMeslMJ8NuO666ypX8OLiNdVGNaqm2sivI5+RPm/ePGVf6Dwi4v3331f2fZK33HJLZR9GkJfJunXr\npuzlTC+F1kNZ+V8/sKD0/8033yh/8cUXyj4L2f8+ImLHHXdUPuSQQ5S9fOkzj/M2aqNZslXXRnWI\nsjIAAABKo3MIAAAAoaxcW2qqHOb8c1rOgrmVurBo1HAb1ZC6a6P8eyBfcLkhpUrElCzx/2ijykdZ\nGQAAAKXROQQAAIDQOQQAAIBU7HofqC3ljBMsVxWOMwQqWn7ttPAuGQCqDE8OAQAAIHQOAQAAIJSV\n0SaKSr7llpspGQMA0DZ4cggAAAChcwgAAABpyx1SAAAAUOF4cggAAAChcwgAAAChcwgAAAChcwgA\nAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAACh\ncwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgAAAChcwgA\nAAD5H+lOvc4XsKLzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc712bce4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGHlJREFUeJzt3XnQ1eP/x/G3Xfa9bEX2ULZhEH2JQpYQxr5FM2SXfSyD\n7MvY9y2GYizTEJFClrJl10IqITsp2X9//H7fl9d1/c7nON1L7nPO8/HX6+4+S/fnOp/zuebzvpa5\n/vrrrwAAAAAiIub+t/8DAAAAaDnoHAIAAEDoHAIAAEDoHAIAAEDoHAIAAEDoHAIAAEDoHAIAAEDo\nHAIAAEDoHAIAAEDoHAIAAEDoHAIAAEDmnYPvxSbOzW+uJngN2ql50UYtH21UHRrbTrRR86ONWr6S\nbcSdQwAAAAidQwAAAAidQwAAAAidQwAAAMicnJBSV/7666/Cn//44w/leeaZR3nuuemrAwBQj/J+\ng5trrqaYJ1c5eiMAAAAQOocAAAAQOocAAAAQxhw2gI8L+PPPP0v+e7nxAb/88otyq1atmvh/h6bg\nbfnbb78pzzfffMpzegxIvfK2YOwugJbEv5NmzZqV/M6vHePGjVOeOHFiyX/v0qVL8vwlllhCebXV\nVlNecMEFlZvrOsS3KAAAAITOIQAAAGSuclOnm1hN7pHox+/3339X/vXXX5PHffvtt8peil5qqaWU\nvcQ877wNqvizJ+xs8rb48ccflV9++WXlV199VfnEE09UXnjhhZPXqvD2Pm30f4pK9xERH3/8sfKN\nN96oPHz4cOWll15a+ZxzzlH20kwDSy411UYN+Y6vkiETVb1vr7fLzz//rPzee+8pDx06VPm1115L\nnu/lyLZt2yofdthhyjvvvLOylyIj5lgbz/E2qnR4V2P4deOzzz5THjhwYPK4d999V9n7BN99952y\nD4PxYUsRaVn5gAMOUN54442VF1lkkZL/x3x4TZnhNuytDAAAgPLoHAIAAECYrdxIRTOVHnjggeRx\nN998s/K6666rfOqppyq3b99e2WdfVkmJp2r4LNcffvhBedSoUcrPPPOMcseOHZXn4DCMqlY0o9+z\nl1m8/BIRcdVVVykPHjxYeebMmcrrr7++8pJLLtnI/3H1yT+LvgrCCy+8oDxgwADlt99+W9nLjF6+\niojo0KGD8oYbbqi89dZbK7du3Vp5/vnnT57PjPHSvM3eeecd5eOPP1555MiRyn6+5Md4mWWWUf7w\nww+V/Zry8MMPK1999dXJ85dddlnlWmqv5rpe+nXD2+iyyy5Tzs+jvfbaS7lTp07KCyywgLIPR7vt\nttuS5w8bNkz53nvvVfZScrt27ZQXX3zxku/RELXziQAAAECj0TkEAACA1E1ZeXbLgeVuTRctyvvT\nTz8pjx8/PnmOl3CmTZum7LeUa+nWfkuSt70f/7PPPlu5TZs2yr1791b2cn9jb9XXMi+B+efayzH+\nmKJ/j4hYdNFFlb1kvPzyyysPGjRIea211lKul2EY+QzvK664Qrl///7KPhPWV0HwMqV/9iMipkyZ\novzkk08q+6xwnzHp7xcRseqqq5Z8z3rnpf/jjjtO+fnnn1f2c2GNNdZQvvzyy5PX2nzzzZW//PJL\n5ZNPPll5yJAhyvmGCzfccINyPpMZ/8vb4o033lDu27dvycf36dMn+Xn77bdX9vPNv6P8+uSrYUSk\n5+Xrr7+u7O3l35X+Ho3tT9AbAQAAgNA5BAAAgNTU/f68fOhlF58Z6WUOX3TSb8OWK0MXLV7qCyeP\nHTs2eY7PivXbwP78ei8r58e8qHzvx6mSEuL06dOTn7385rfqL730UuXVV19dOV+YFH/z0vCnn36q\n7Iu++3noZRL/9zfffDN5XS9L+6LWZ5xxhnI9lpK9zPXSSy8lv7vmmmuU/fitssoqyj179lReeeWV\nlX3GY0Q663LEiBHKDz30kPJTTz2lPGPGjOT51113Xcn3r7fvuPw7zVex8OPqC+rffffdyj169FDO\nZyv7Z943U/DPwXbbbafsM9Uj0kX/KSv/zdvsk08+UT7mmGNKPv7II49U9uMdMftl3oUWWij5edNN\nN1X2z4jPNPfrU1N+D9bXmQoAAICy6BwCAABAaqqs7KWUiIgJEyYoT548WdkXofbbs15uLnd71mec\nff7558o+CzZflHennXZS9n0w1157bWUv0c1u6bQW5DNWv/nmG2W/de6314tKvl7W7NWrV/K7Dz74\nQNnLNj77jxmWpeVt5PuK+oxLbztfRNnLj/6Y0aNHJ6/rwzLOPPNM5XosJTsfhpIvmOvHdptttlH2\nYRSrrbaash+/vF39d95+Plv8oosuUvbhGRHpDGcvu9VbWdk3RohIP8vOZyLvscceypV+xv24+p7j\nfr74tSoiHW7j18F6PK+cX99POeUUZS/Dn3XWWcp+fcmvR5UcS39Mfn54+/lQJ78GNldfob7OVAAA\nAJRF5xAAAABC5xAAAABS9QOrfJzexIkTk9/5BuQ+psLHCGyxxRbKvpl1vgSBj2f0sT3ff/+9so8v\n2WijjZLn77jjjsrLLbecctEG3PmyBbXKj7OPE4xINzT3MRa77rqrsu8gMHPmTGVfwf6tt95KXned\nddZRvvLKK0u+R72Puyni51tEugTNww8/rLzVVlsp+1gZHx/jS2t88cUXyev6khBdu3ZVrsexoH6O\n+E4Y7733XvI4X87Ed9/wc6RojG65ZaScj8v18VA+9jQionXr1srzzDNPydeqVeWWG/LrhS9Tcvjh\nhys39rvHl23z5Yrya5KPTazn77t8vO0999yj7N9vft3ZfffdlRu7lEy5ZfOKlhgqGmfImEMAAAA0\nCzqHAAAAkKqs0fht2EmTJin369cvedwrr7yi7EvL+LIyfku4aEeOiLRk7KU1v+3boUMHZS+lRUSs\nuOKKyl4a89fyW8I+nb6WV6/3UvrNN9+c/O7+++9X9iU4fCkiX1F+2LBhyr5sg5fVIiKuvfZa5cUW\nW0y5nksrDeVDATwXLbvgS7F4OcfLXxERBx54oLIPvah3XlbOS/w+FMV3OGnI8jHeNt5mw4cPL/l4\nX+4mIh0WUG/nlbfLwIEDk9/5Z9nLlw3ZhcmvUf6e48ePV27fvr1yt27dkuf7MKp648cuHxLhywr5\n9emoo45S9nZsys93/lpFr12uFP1Pz60Udw4BAAAgdA4BAAAgVVlW/u6775QPPfRQ5XxWqt/63Xff\nfZU7d+6sXFRyyW/b/vbbb8pTp05V9vKLl3LatGmTPL+obOC3fn0Fdv8bfReV/DnVyI/tp59+qnzX\nXXclj/Pj4TMxfZcGL7m3atVKedFFF1XeZ599ktfdYIMNlKv9WM5p+czTfIb5f/3www/KPkTCd0Xx\nf99ll12S57dr10653nbVyPln1GcI+8zhiIhx48YpDxkyRNlL9r5SQtHwloi0lHz77bcrP/HEE8o+\npOPSSy9Nnu/DNeqND0HynbkiIrbffntlL/kWKVc+9GuPD6MZNWqUsreDf3Yi6m8WufMZ3V5Gjoj4\n6aeflA866CBlP48ae93wds13dnOV9E/yc/e/8pUdZvd7tL6/dQEAAJCgcwgAAACpmrKy33r1zct9\nw/f8tqkvdn3CCScoVzL7N7/V67ehx44dq+xltW233Va53KwjvyXspQFf1NbfwxdtrgVeoh8wYICy\nLxAbkS7Suuaaayr7MZsyZYqyz6T0kmXPnj2T1633MmVj5J/rTp06KQ8ePFjZP78+k9X/3YdhbLnl\nlsnr1ssi8LPLy4QXXXRR8jsvJT/++OPKPvPbFyc/4ogjlPPy5VNPPaX86KOPKrdt21b5tNNOU/ah\nHhH1N1zDj9/XX3+tPH369ORx//nPf5SLyolFZcL8tfz70ld68McdfPDByuVm/fv/v1bbzv/GCRMm\nKL///vvJ4/y7yFdAKfpOqmTmcETarl66/uSTT0o+JiJilVVWUfahaf5/8euhP78hM+AdV0kAAAAI\nnUMAAABI1ZSVfYHjxx57TNlvgftMsIi07FG04Kff2vcZeqNHj04e5+Vrv6XrJV/fv7nSW7peAnjo\noYeUvaRaC4pK6V5a7N69e/IcL2H577y0dt999ym/+OKLJR9Tz4u9NrW85OQlYz8X/PzxRXl9FvOe\ne+6pnLdRrZa2GsuHRPjC/hHp8fSZir7PuLeLl6F92EZEutj2+uuvr3z66acr+8L+tNfffKWJr776\nKvndyJEjlX1Ihn8/jhgxQtlXXfAZ/BHptcPbyzcM8FUe8tnJc2Ih5ZbES65Dhw5VzocZ+comRZsk\n+Gv5MKl8n2b/nZevvQ/z0UcfKa+33nrJ83fbbTfllVZa6R/f0zccaCzuHAIAAEDoHAIAAEDoHAIA\nAECqZszhAw88oOxT+H3F/0suuSR5zgorrKDs4wp8rIXX7t955x3lCy+8MHktf5yP7fHxOD4+JB/H\nUPSePh7Lx6P4khO1xsdu7LXXXsr5Lhk+Xsaf8/HHHyu/++67ypMmTVL2sXBoPj5WcPfdd1fu3bu3\nsi/t4btqdOnSRZnlhWZfPh7MlyrZYYcdlPfee2/l+++/X3nMmDHKfh5FpGOejz76aOXWrVsr02Z/\n87ZYfPHFlX0cekQ6HvHJJ58s+RwfG+jnl3+/RaTLqHXs2FHZxzL684uWyMl/59cqH19f7e3ty9G9\n/fbbyvl4W1+Szv9mH9v32WefKfuyT74cXUTEtGnTSj7f29g/I/kuNs53l/Jz13fH6dOnT8n3y9+z\nEtXd2gAAAGhSdA4BAAAgVVNW9k3lffcLv53vZciIyjat9hL1Pffco+zL2kSkU8x9erkvxeK3bfNl\nAnzJHC8lX3/99cp+q9r/xvy1qn15AT9OXr7Kb3sXLR3gQwm8tOK30X3D9Govh7Rkfmy9rOlL1syc\nOVPZy1xe+pzdkgf+Pz9fWrVqpbzffvuVfMxzzz2n7Ls0RKQ7OPjON1tvvbWyL5dD+/3Nv598qEVE\nupPMZpttply0LJCfR4MGDUpeq2h4lZf+fRiHD2eKSMusRbt3+DXVP1PVeA3y5X58CFnXrl2Tx3kp\n3a+9M2bMUPa+gudZs2Ylr+Xt6p+FLbbYQnnttddWznex8bL0qFGjlB955BFl/w4+/PDDlb29GoKr\nJgAAAITOIQAAAKRFl5W9TOjZb/X6rWJfWT4iYuedd1b2EojfXn/llVeUfTPufGaXrzrvr1XJLOiI\niC+++EL5rrvuUn755ZeV/VZ9PtOo2hVt7O47yVRaqvASvR8nv43uJZtqLIG0VPnn0odf+LnkFlxw\nQWVKkc3HzzEvDU6ZMkXZS47dunVTXmONNZLXevbZZ5V9V5VTTz1V+aSTTlL23Rsi0p0a6u3889Kg\nl+Ej0pmxRatbeDt66XeDDTZIXqtDhw4ln7PMMsuUfA9/rYj0e9RLphMnTiz5fD+Pq6VN/bjceeed\nyr5bSf639OvXT9mvT/5aU6dOVfZjl5eFe/TooewrOPjOK972eb/BV3rw71ffVcWHjTRlG3HnEAAA\nAELnEAAAANKiy8p+W9TLFj5rx2f1nnzyycnz/XdeZiya9eMlal+sNCLdQN1LNl4+8VvQvjBlRDqj\nadiwYcp+q9pnnF1wwQXK1XILvxz/OyvZ8L0cn4Hnt/S9rOyzxNA4XurwhVgj0oV8X3zxReVdd91V\n+ccff1T2UtbkyZOV88Vf8/IM/pmfCzfccIOyL8zrZc0jjzxS2cvNEWk5zBf5ffrpp5Xvu+8+ZS9R\nR6SL0Pv5Wg8rB/j3dX5cfShFJceiaAhTRNqWvoi2H+9y7+dDRHy2+tixY5V9Jm01Xof8b/Trvg+H\n8RJtRPr3b7LJJiVft23btspFmzVERPTq1UvZ26io7fPhbN4/efPNN5V9YfujjjpK2WdaN1btn6kA\nAACoGJ1DAAAASNWUlfv27av8zDPPKPsioT7LKiKdHdSmTRtlvx3vM7i8LOx7VUak5bRbb71VefXV\nV1f2Mrbv/xsR8frrryt7idrLDt27d1f2EnO18vKx3y4vmomd32r39vfX8kWVvWTppeQll1yy5Oug\nMt5ePnP1jjvuSB7ns/58ofjzzz9f2YdY+N6+Xq7MZ2IWLQqMv+XDM7wE5cNY/LvPZzauuuqqyv7d\nF5GWwA455BBl3wvd2zXfU9a/L/2zVG7/+Vrhn9f8b/Q28+9Bf45fk3z2qe+ZHJGWEMuVn4t4adXP\naz/3fGMGnwVdjbxEPnz4cGUfJhYR8dhjjymvs846yt5efrz9MT6cIiLdN7loxRUftuPDNiIiXn31\nVWVfMeXYY49V9qFtTak2z04AAAA0CJ1DAAAASIsuK7vNN99c+ZZbblHu37+/cr4/qN+69dmPXnL0\nmV1Fz42IeOONN5S9rOnlXy8BtGvXLnm+zxrzWYIHHHCAst/O99vWtaCoROyzV/PSVtFisAMGDFD2\n8pWXNb0tMPt80XZfBcCHcUREbLnllspXXHGFspegfP9xLzEOGTJEuWfPnsnrMtv8n02fPj352c8L\nX23Bh6tss802ypUuQO/fRV5u9nKWL9YbETFy5EhlL2v758W/k2tp6ID/LR9++GHyOy8hdu7cWblo\n8eKixZLLvafz71AfghMRcdlllyn7PtuHHnqosg89qEZ+zE444QRl/3vz2cqvvfaasg+p8SFkPlt5\nhRVWUPZVUSLSFQT8u9NnRPtwkLwP4yXriy++WLm5SsmOO4cAAAAQOocAAAAQOocAAACQqhnY5tP2\n99xzT+XddttNOZ+SPm3aNGUf2+bjPp544gllX+n8gw8+SF7Lxwz6WBkfW7XRRhsp77///snzfUr7\nIossopyPs6tVfvwq5ePTfGzhCy+8oOztevDBByvX6jIZzcnHJ40YMUJ50qRJhc/ZaqutlH1sofPz\nzZc/8aU08jG+/nMtjUdrSvkx8x2efNkMX8KjaGxbQ5TbCcTHcflyRz52vEj+d1Vz+8+aNSv52c8r\nH6vmbeTXukr/9qIdqHzZtEsuuSR5ji/n4ueujzn0pYeqsR2KdlkbNGiQsi9NF5F+L/lSPn5+TZky\nRdnHhfqSdRHprlHeH/Ex9H7u9OvXL3m+93Xm9K5RXEEBAAAgdA4BAAAgVVNWdl4y9FvwXhaOSJes\n8TKLlyvbt29f8t+///775LW8LOrv6SVin15ebrePelH0N/uxLCqHRKQl42+//VbZb++vvPLKyl26\ndPnH90YxP/4TJkxQ9jJLvkSQl4m9hOXn0sCBA5V9SSg/R7zkE0H7VaJVq1bJzx06dFD2ElZRW/rz\nK10mxT8jvlzO4MGDk+d4ybJr164VvU+t8OOVL0nmS/z4clHnnnuusi+T4vKdpXxplHHjxin7zkMv\nvfSSsn8mItJz7qyzzlJefvnllWvpPPTrju82cuaZZyaPmzp1qrIPA/ChEmPGjFH2ErN/H0ak55sP\nIfNl13zXqHzpoIYMx2oqtX+mAgAAoGJ0DgEAACBz5aW8ZjTH3qiONUUNoNnbqWjzeZ/VGpGWQd56\n6y3l22+/XXnTTTdV9p085vTMrtnQYtvI28KPd69evZTzXRa8NLn00ksreznNy82//PKL8g477KB8\n0003Ja/7L7dfi22j5A2y724/zuedd56yl3g7duyovOGGGyrn5UMfouNt4asGePkyn9Huw3WuvfZa\nZd85qglKzI1tp2Zvo3xHoW7duimPHz9e2Y+xz1z2mbA+PCAiYvLkycq+UocPe/Kdhnzma0TEscce\nq+wzp5u4lNni2yjn34N+XH1XFd/hxGcoe5tGpEPbtttuO+W+ffsq+y5t/1IZv+SbcucQAAAAQucQ\nAAAAQlm5tlRFOSx5M/v85WVlnzHuM8V8lmSnTp2UvTTSgmfZVUUbeVv47PB8Vuro0aOVvazpi8N7\nmaZ79+7Ku+yyi7LP+o/419uvKtoo58fZ2+zuu+9WfvDBB5W9NJZvIOB8hrp/Lrwc1rlz5+Q5Bx10\nkHKPHj1KPqcJypctvmSZzzD2RZXPPvts5aFDhyrPmDFD2b8f8zbysryXj3fccUflPn36KHu5OiKd\nPduMs8hbfBs1VqV9qGq7JnHnEAAAAELnEAAAAEJZubZUZTlMb1xmf92iz6mXQ1rwbXtX1W1UJ6q+\njfx88cXkvfR/7733KvtivxHpnsBeVt54442VfYiAL0Yfke7J6zM2m/gcrZmSZWOvwy34u69m2qiG\nUVYGAABAeXQOAQAAIJSVa0vVl8PqAG3U8tFG1YGSZctHG7V8lJUBAABQHp1DAAAACJ1DAAAACJ1D\nAAAACJ1DAAAACJ1DAAAACJ1DAAAACJ1DAAAACJ1DAAAAyJzcIQUAAAAtHHcOAQAAIHQOAQAAIHQO\nAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAA\nIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQOAQAAIHQO\nAQAAIHQOAQAAIP8Dqv9eGoxqIewAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc712a58a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGLBJREFUeJzt3WmsXVUZh/EXRRAUsMxKLVSmShkUZJKGIUJLkSFIsQoq\ngUpQAkJMrKgQhDAoIH4gUoxA1apIRGYEDVUoUNpiWwoqpQOzILQWFRkEFT+Yvj5rcff19k6cc8/z\n+/Sn9wz37nX2Pov9rmG1119/PSRJkqSIiLe82b+AJEmSWoedQ0mSJCU7h5IkSUp2DiVJkpTsHEqS\nJCnZOZQkSVKycyhJkqRk51CSJEnJzqEkSZKSnUNJkiQlO4eSJElKqw/ie7mJ88BbrR9ew3YaWLZR\n67ON2kNf28k2Gni2Uevrso28cyhJkqRk51CSJEnJzqEkSZKSnUNJkiSlwZyQIkmS1NFef/1/82z+\n/e9/Z37uueeKx73jHe/oMr/1rW8dwN/uv7xzKEmSpGTnUJIkScnOoSRJkpJjDt8EHG+w2mpdrxHa\nk8eof3Hsxz/+8Y8u/33ttdfObLtIagV+X7QGflfU//3ss89mnj59euZZs2ZlXnfddYvnDx8+PPOn\nPvWpzOutt17mgWpv7xxKkiQp2TmUJElSsqz8JmM5oKePsWywaurjx5LxwoULM5977rmZlyxZknnb\nbbfNfOWVV2ZmiblTNS3JwPyvf/0r81ve8r//H+3uc8znvPLKK5mff/75zO985zszs8yyxhpr9Oh3\nH8p4/P/5z392maluC7bT2972ti4f53Wo/7z66quZ582bl/nSSy/NPHv27OI5y5cvz8zP/GabbZb5\nM5/5TJc5olwahW2sVcNz6qWXXip+9uSTT2ZevHhx5vnz52e+++67My9btqx4/iabbJKZ17gJEyZk\nfvvb3565P89J7xxKkiQp2TmUJElSWq0nZc1+Mmhv1OWbN/ydg1UaaSqz8f2bZpyxxNPVf0N//DFv\najv1Fx5LlpEjIubMmZP5G9/4Rua5c+dmZin5ggsuyLzrrrtm7uUq9W3dRvzsRpRllEcffTTza6+9\nlpmlEpaC+RlfffVyhAt3Crj55psz//nPf848fvz4zIcffnjmYcOGdfkeq6At24htcf7552e++uqr\nM7Mt+PldZ511itfaaqutMo8aNSrzRz7ykcz77rtvZpa8IgbtutrXN3lTr3U8l6ZMmZL5y1/+cuaX\nX345cz2MhZ//0aNHZ37hhRcyr7XWWpnXX3/94vkHHHBAZl7v+nlYRlu3EfE7hcMAnnrqqcwrVqwo\nnsM2/stf/pKZ30FLly7NfNtttxXPf/HFFzPvvvvuma+66qrMG2ywQeb+/E7yzqEkSZKSnUNJkiSl\nIT1bmbeBm2ZMUn+WQuoyNt+fs5vqctpKLEMPxibbQwGPOXN9q/8HP/hB5gceeCAzS8ZTp07NvOGG\nG2buxBma/Oxyll1EWbJcsGBBZs6y22OPPTLvuOOOmXlcn3nmmeJ177zzzswswdDWW2+deTAWhW01\ndYn/xz/+ceaLL744M0uTvN6wzMh2iYh417velfnee+/N/P3vfz8zhwiw9B8R8cEPfjBzN8NgOtrv\nfve7zOecc05mDg/YYostMvOciChnJfMzz+8XlixvueWW4vm8LnIYSNPs9E7UNBP5/vvvz8zr3jbb\nbFM8f/PNN8/MBa132223zBzq8b73va94/k033ZSZbVQvtj0QPGslSZKU7BxKkiQpDemyMm8Jc/Hc\nNddcMzNn6fX1Fjpv9fL9IiKeeOKJzCy5sXzD8jFLRnXpudNv9fdksWXODONs44iyLfbbb7/MF154\nYeZOLyWzLM+yx+TJk4vHsczMz+nEiRMzb7zxxplHjBiRmYvwsr0iyvOH5/HYsWMz77LLLpk7pXTJ\ndvnTn/5U/Oyss87KzBn6I0eOzMzy5ZgxYzLXe7qyLVn+/PznP5+ZQzJOO+204vnXXXddZpafOx1L\ng5/+9Kczc3b+uHHjMvM4crHjiObrEsvCTYsoR0T87W9/y8zvnk683hHPHe6B/Otf/zoz22LTTTft\n8t8jyu93Hn9er/icT37yk8XzuR8zV2rg9XKjjTZq+lP6pDOuqJIkSeoRO4eSJElKQ6qsXM/g4b6G\n06ZNy7zPPvtk5kxKlpt7emudZQIuYMk9MSPKmZXc45K3nVk+683vMtSwhMZjw3IaZ7myHPCb3/wm\nM2eTRZRlgDPOOCMzy5+desxX4rG/5pprMt93333F43jOHXTQQZlZMmMpmeUrPveOO+4oXveuu+7K\nzFm1kyZNyswFfjulvXi9ueyyy4qfsdTEkjsXzOXs154eM848PumkkzKfeuqpmTl7M6KcJcuZ0J3S\nTk04DIPlerbLjTfemLmvC1LzeNebAbDM2ckrYtT7jXNWN88xDr3gYvAcBsDvkIiyZNw09IVDOHit\njIj40pe+lJn9Gc58pqaNNHrDO4eSJElKdg4lSZKU7BxKkiQpDakxh/WYCu4SMHPmzMx/+MMfMnPJ\nEq5OzuUAIspaPseKfPWrX818zz33ZK43Sf/Qhz6U+f3vf39mjjfge3TyGJCVOBaEY924vMP222+f\nmcf8r3/9a2Yu5RFRjvnccsstM3fKcig90TTek0s/RZTLlBx66KGZt9tuu8w8lzhmbtGiRZlnzJhR\nvC6fw3GG3IGgE9uLS57ccMMNxc94LeNYKY5n6+kxY/vzOVtttVVmLqHBcbwR/btEWDurx8GfffbZ\nmbmUE3fC6Os4w6Yl1UaPHl08jtfOTvu+4bJn9XhZjvNjWxxxxBGZDzvssMwcZ1ifX6v62a/b4T3v\neU+X79O0s1p/nmudd3WVJElSIzuHkiRJSm1fVn711VczX3/99cXPuATHCy+8kHmDDTbIzFXHu7u1\nztXRr7jiisyPPvpo5mHDhmVmSTsi4uCDD87cVDZouiXMEk93j2t39d/58MMPZ/7iF7+YmeX/HXbY\nITPbj+Wv+nW5NEenlVN6iuUR7nYye/bs4nEsLbJ8ybIwX4sl6oULF2bmMICIcmP6Qw45JHNTOWUo\n4+e3u2PGoSujRo3K3JNScn2O8L9ffPHFzFwSjKXkCRMmFM9vWmqj03AXkohyVxl+rtlevcH2YsmU\nQ614TYwoh4QM1e8U4jHi8AwODat/xqVlmnZ96s3uMvX5tlJ9rvK1ee3jdZTDCPj+fR12451DSZIk\nJTuHkiRJSm1Zo+GMx9/+9reZ61LuK6+8kpnlx9NPPz0zZw7zlmw9y2z58uWZeat+7733zvztb387\nM0ts9WszN91eZmmgE275R7xxtjl3L2H5fs8998zMWbHcOYWr/7ONI94441ZvxM/cJptskvnkk08u\nHnfeeedl5jAOlpu5C9Djjz+emSXquqx24oknZq53HehkvPbxMx7RXJqsr2Vdqa9Df//73zNfdNFF\nmR966KHMHJ5x/PHHF8+vV3voJDyWPF4REe9+97szc1Zsb67xTSsKrFixIjN3zeF5HNE8pKY/d9lo\nJTx3TjnllMz1rk/8fjj//PMzc6WE3pSSmzSVhbvDNnr55Zczc0eWvs56986hJEmSkp1DSZIkpbYp\nK/PW6zPPPJOZs+eefvrp4jnc8P2b3/xmZi4GytvDnN3Dskr935wty1m0m2++eZev1R3eRmZp4KWX\nXsq81lprFc8ZSjNseXv8kUceKX7GxUk5w5y397mQ7BNPPJGZZfl6ll4nLp7cFzxenBEbUZaPf/az\nn2Xm+TJ+/PjMS5cuzcwy13777Ve8LoeB2F7/w9mT9XAJzl6eM2dOZl4/WDJbf/31M3PVhoiIa6+9\nNvPtt9+emcM4Jk+enJnnYcTQKkeuKn5XcaH3iPJzzePPkic/78x16Z/fF9yY4a677sq87rrrZub3\nU0Q5W5m/c9PGDO3Ypvz9ueLIL37xi8z19eUTn/hE5nHjxmXuTZm2aRY5S8H89/p3WXPNNTOzvbmC\nAF+LnynLypIkSeo3dg4lSZKU2qaszNuzXFiUM1T32GOP4jmc8ch9JDmTjrdqeXv2hz/8YfFat912\nW2bOnmSZpTflr6YZZ5y5y9v/Qw3LGZxdHhExduzYzLvuumtmlmbuvvvuzD/96U8zs73rsrx6ry5V\nnHPOOZlZPmZb/PKXv8zM0tbHPvaxzDvvvHPxup0827XGch73WuXwioiIJUuWZP7617+e+bHHHsvM\n6w3Po3roBRcCZttwX3JeB9ux5DhQeE1jWTCinGHOmcQ8r+bPn5+Zq2R84AMfKF6LGztwn21+j7z3\nve/NXA9HYjmTZW2ee/y9+P3WLu3NoRZf+cpXMnPzjHplkZNOOilz03WI5xHbm8c+ovxO44obXMye\nQ3O23Xbb4vlc6YFtxNficCquntLXVTm8cyhJkqRk51CSJEnJzqEkSZJS24w55PgITtvneDJuZB5R\nLrvBsQMcL8AlN2bNmpW53m2FYz96M56taYwCxxZyU3aupM9xBxHNu620Ix4Xjo+JiDjzzDMzc6kM\n7jpw3XXXZZ45c2ZmtpEGDpej2WuvvTJfdtllmTm2iudx0zjgiPb/XA8ULk3y2c9+tvgZx3lyOROO\nDeS4al7TOHY7ImLttdfOPGbMmC5fq7tlVjq5/XgsNttss+JnTcvHcBk2jtFlu3A3sIjymsgxo1tu\nuWVmjnGsxxxyDBu/B/m5GD58eOZhw4ZFu5k+fXpmjtPj53PSpEnFc/g91LSbGc+jZ599NjN3iYqI\nmDdvXmYux8aluzbccMPM3OGkfk/OtbjpppsyP/jgg5m5zF69I86q8s6hJEmSkp1DSZIkpbYpK3PH\nEO7EwNu73BEloiyNsATStHzNnXfemZm3cCPKEsAuu+ySuWkF9Hqze5bWuGr+Lbfc0uW/H3jggZkP\nO+yw4rU49b3d8bZ9d+VzlkCeeuqpzDNmzMjM4//xj3+8y9dR/+Ix59IaxOPPsglLpLZRz7A0WJcs\nTzjhhMxHHnlk5qZSJkteF154YfFaLJWxRM2liNiW3Jkhohwm0FR+Hqptzr+LZd2Iss3YLnzOxIkT\nM3PYDHcGq1+bS5hsuummmeuldIjfqVzapd4tZyVen9ul7ViK564i/HzuueeexXOalqTjMeLyUJdf\nfnnmP/7xj8VzOOyMy0BtvfXWmTk8oC79s6/Cpaq4dBG/D3lOs5/SG945lCRJUrJzKEmSpNSWZeWl\nS5dmZvmDpdsabxVzY3He9uXMMM4irp+/ePHizE8++WSXr8tNviPK28Cc9cX3YRmcJbe6ZNPumkpL\nPH7143j8n3/++cxcBb5pw3r1H5b3I8qy16233pqZZRt+flmG5jCO0aNHF69bz9rTG9WfcV7LWL5k\nqYrHn7utTJ48uXgtrtzAVRTuuOOOzJwhy9muEWVpk78Xz/d23HGjJ3iOcKeZiHJmatNMYpbuucMH\nh0BFlCtacCYsv0d4TWXbRzQP2+H368iRI6Pd8Htj2bJlmeu/fyVehyLK3bh4/NgH4XAmln532mmn\n4rU4vIkzv5t2nqln/fO/uRsY+x38u/h56+sKAn6DSpIkKdk5lCRJUmqbsvKKFSsyc3YOZ0hy4d2I\nsuzB2+68Dctbwtwkuy6fsczFW/1z587NzJmA3Fg7ImLEiBGZuYH6hz/84cxctJIzkuvyUbuXYOrb\n3U3/3rRYOEtmLFnyVn3TTECtOs7Su+SSS4qfTZkyJTOP/7nnnpuZn2sOt7j//vsz33777cXrjhs3\nLnO9QLb+qz5fWHZkW/Dzz+sdr4ksA0eUGwjwGstrJ9uVJbeIcogPV37gMJCheo5yViwXKI4o/35e\n4/n3syzMmaycNR4RcdBBB3X5HJZC+d1Rr6DBkiu/x/i9yaEH7ThUp14BYyWeB9OmTSt+xs8+F+pf\nsGBBZs5K5rnD7/mI8ruKM8ebPu/1Of34449n5sLXbEu20THHHPN/36On2q+1JUmSNGDsHEqSJCm1\nTVmZ+x1y70jOpLv++uuL53AmMxeF5YKhLMWwRMlbtRHl7X2WY7iYJW8hjx8/vng+Z6Mx83VZDhhK\nZZaI5sXCWb7vrqzM0hRncvNY8t85I3yoHcvBwLa4+uqrM1900UXF4zhL9Wtf+1rmL3zhC5l5/Hfb\nbbfMV155ZeZ6dv/OO++cmWWbdixtDRSWxiIiHn744cwsM/J6xfON/17vEc824+Oa9pqtS/9cUYD7\nze6zzz6ZWb7k79vu5yuv4xwCFVGeL7y+8W/mMA6eL3Xpn2XlppIl26gu/bNMzX2H+d3V7kM6Dj/8\n8MxTp07NzGEPnJ0dEXHKKadkPuSQQzJzFjmHDvCatMMOOxSvxeFQTatpsL0feuih4vm8pvL3ZB/o\n2GOPzdzX/ZTJK60kSZKSnUNJkiSltikrc5YXF9797ne/m3n69OnFc3gblrfQOZOP5QyWOzljKaK8\n9cuZlJzNxBnN9R6JTTN0WQJo93JKT7GUz1lf9T6kLLnzNjxvvbM8wOPP53bKce1PPK6nn356Zq4a\nEFGWCU899dTMPMeI5RCWfK699tricffcc09m7jM+VGe49hSvI1y0PyLiRz/6UWaeSxMmTMjcVMrt\nTbm+u2sXZ79yWALLzSeeeGKXv0u747HkbNOIso248DuHNHGGLc+RepOApgXF+RnheXzWWWcVz1+0\naFFmDgNp9z3P+Tvze/zss8/O/K1vfSvz8uXLi+ez9M8VFXjtY7n4kUceyVzPCG9aaJzvcd9992Wu\n97Vm++21116ZzzzzzMxc/aT+jPSFdw4lSZKU7BxKkiQp2TmUJElSapsxhxxTwQ3HzzjjjMz15vHc\nyYQ4tpDL3YwdOzZzPQZqiy22yMxxhlwCoh5nSH0Zu9HXDbRbAf8GLkvD8YMcdxNRjltjW3KjdI4F\nPfjggzNz3Ix6hm3Ejd3rcYY0ceLEzD1Z9qLpPK4/4zfeeGPmbbbZJjOXimjH86A/1eObOMaaY/jY\nLhyX29fjxzbjOOKIcswol3Nhm3Mc+VDCZWW41FlEOU6U10EuvdW0pFl9jnApI14HubzbxRdf3OVj\nIiImTZqUef/998/c3fdYu+F3yOc+97nMRx11VGZ+n0REzJw5MzOPMcf/zZ8/PzPbccaMGcVrcdc1\n9jvYrvwd688Lx3sfccQRmQdj7LV3DiVJkpTsHEqSJCm1TVmZeBuVt+DradxczoS35HmruGlXFO4K\nEFGuPD7YOzYMtfIZb68vWbIkM5e1iSjLZmy/e++9NzPbgpuONy2lomb8nG288caZWaKvd1lguZ8l\nFB5/tjd3FuCORr/61a+K12XJke/RtCRUp2AbsRQZUS6v8dhjj2VmmWzkyJGZ2Ub1NabpOPOcfPrp\npzNPmTKleByHBfCzxKVZunv/dsYy/o477lj8jO3CJX6OPvrozPzs83ybNWtW8Vo/+clPMi9cuDAz\nzzEuf8ISZUQ5PKrdd0Jpws8Vy+U8dw499NDiOVw6i0MiOGyDr8ulaHgNjCiPK6+jXFKKywhxR5aI\nctjaYJ8j3jmUJElSsnMoSZKktNoglmlash7Ev58lE5bCarw93WIzu/rjvvOAtBOPLWd3fe9738v8\n85//vHjO73//+8wcMsCyy3HHHZf5tNNOy8xZmS2mZduI+PmfNm1a5muuuaZ4HGfFjho1KjM3qeeq\n/wsWLMg8e/bszNwxIKIsh11wwQWZWRYdwCEdbddGERFXXHFF5ksuuSQzz7cxY8Zk5s4K9ZAOruLA\nHSD4npy9WX+PDB8+PPN5552Xeffdd8/cD7s59LWdBryNWOKNKK9R3LWr3u1mJZYp63OE11Qey49+\n9KOZL7300swcGhUxOEOiog3aqNs370H/gG3Ecy2i7B/we6unq5wMki7byDuHkiRJSnYOJUmSlDq+\nrDzEtGw5jJ8z3pJnqeSqq64qnjNnzpzMvA3PWcmcDcjHtLCWbaMmnN3/3HPPFT+bO3duZs6mXLRo\nUWYuGMvX4mLBI0aMKF6XC9rvvffemQdphmtbtFF97WZ5izNZv/Od72Tm4uYsedaLWBOPOUuR6623\nXuZ6luXJJ5+cmYuYN71WL7V8ybK7Npo6dWrmm2++OfOyZcu6fHw9jGCnnXbKzPNlu+22y9wCM8Jb\nvo1kWVmSJEn/h51DSZIkJcvKQ0tblMM63JBqI14/mLmP62uvvdblYzj7jyXmiHLxWJYfWRrr9LLy\nG96wYegGZ1DOmzcv8w033JB58eLFxWtxZiXLl1wgmAv5ctZ6RDl7toVnlbfMeTSE2Uatz7KyJEmS\numfnUJIkScmy8tDSluWwDmMbtT7bqD1Ysmx9tlHrs6wsSZKk7tk5lCRJUrJzKEmSpGTnUJIkScnO\noSRJkpKdQ0mSJCU7h5IkSUp2DiVJkpTsHEqSJCkN5g4pkiRJanHeOZQkSVKycyhJkqRk51CSJEnJ\nzqEkSZKSnUNJkiQlO4eSJElKdg4lSZKU7BxKkiQp2TmUJElSsnMoSZKkZOdQkiRJyc6hJEmSkp1D\nSZIkJTuHkiRJSnYOJUmSlOwcSpIkKdk5lCRJUrJzKEmSpGTnUJIkScnOoSRJkpKdQ0mSJCU7h5Ik\nSUp2DiVJkpT+AzHLe9IKI7g7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7126fff60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF7pJREFUeJzt3XnQlfP/x/G3fUsl0qZNyhZSioRMthiKMGYYe0OEMcYk\nW4YZwxj+CNNCyVJNmVFjHctkkmVka6iI0CIqhUT25ffHb7x6XZ/vfd3OvR1neT7+enGfc+6763Ou\nc6653p/P+7PF33//HQAAAEBExJb/9R8AAACA0sHFIQAAAISLQwAAAAgXhwAAABAuDgEAACBcHAIA\nAEC4OAQAAIBwcQgAAADh4hAAAADCxSEAAACEi0MAAADI1kX8XWzi3PS2aITXYJyaFmNU+hij8tDQ\ncWKMmh5jVPpqHCPuHAIAAEC4OAQAAIBwcQgAAADh4hAAAABSzAUpAAAAVe3vvzevs/nzzz+VFy1a\nlHncbrvtptymTRvlbbbZpgn/uv/HnUMAAAAIF4cAAAAQLg4BAAAgzDlE1fH5Hr///nuNj/H/7/M7\ntt1226b7wwAgx19//ZX73999953y999/X+PzO3bsmPnvrbfe/PW/5Zab7xNtsUVj9IBHRPa75tdf\nf1V+++23lRcuXFjj/4+I6Nevn/LgwYOVu3TpotxU48WdQwAAAAgXhwAAABDKyqh4P/30U+a/77rr\nLuWXXnpJ+bffflM+9thjlS+55BLlYtzOBxrK22P4+/rnn39W/uGHH2rMERE//vijco8ePZRbtGih\nvNVWWzXOH4sMLxd7yfHee+/NPO7VV19V9s84LxEPHDhQuX///pnn9+zZU3nfffdV3nnnnZV9So1P\ntUnHvhitVcqRl5X9PPSWNWPHjlVet25d5vkfffSR8g477KDcoUMH5aaa6sSdQwAAAAgXhwAAABDK\nyqgYfgv/22+/VR41alTmcU8++aSyl2O8ZNa1a1fldu3aKVdLKTmvg/+mTZsyj1u9erXyxx9/rLzT\nTjspe5lr1113Vfbj7f8/ImL77bdX9hKWH/9qGYtCecn4zjvvVH7ssceU16xZo/zHH38o+/GOiGjW\nrJly+/btlQ888EDlW265RblTp06Z5zM2dePn2EMPPaTsxzgt3Z5yyinKZ555prKP1y677KL89ddf\nZ57vJcxVq1Yp++edn6OMac38s9JzRHZ6xqOPPqp89913K/t3VboifcmSJcrvvPOO8tlnn92Av7gw\n3DkEAACAcHEIAAAAqeiyst/i9du1eaUpbpuXHx9jb/7qK5LTzcz79u2r7Bubn3rqqcpesqmWxtd+\njnj5eNy4cTXmiOwKvLxzabvttlP2MlXr1q2V07Kkl8l8vLxc7eNSjeduWoJ67rnnlO+55x5lH0sv\n8bdt21Z5wIABmdfycXrxxReVH374YeWZM2cqp817ffVrNY5NIfyz6/XXX1e++uqrlf3cmTFjRub5\nhx9+uHLe1Av/HS1btsw8v02bNsr+2bnjjjvW+Lp5TbOrUd4qZG90HRGxfPly5Q8//FDZS/fNmzdX\n9ukFKR+XYpxT1T3CAAAAyODiEAAAALJFurqmCRXtF/3Db/euXbtW2UtTeQ0/6yOvjB1RtFJ2Y7xY\n0ceprvw4+2rZyy67TNlXeXkT34hs2cZ/5mW2JmzqWrJj5Ktdb7/9duVJkyYp+zkVEdGqVStlb6rr\nJV8vUXoj1w0bNih/+eWXmdf1UouvkD3jjDOU99lnH2UveTXCOVWyY+Tvfd9PNyK7D+vnn3+u7Mdp\n6tSpyvvtt5+yH7+Uj83pp5+uvGDBAuWLLroo85zx48cX9NoN1NBx+k8/67755htlP3d8heszzzyj\nfNRRR2WeX9f3efpd/8svvyj7ee0r1xth/+WSH6P0uPh3t6/oz/v//jm2dOnSzGt5KdlXJfs1iH/u\nrl+/PvN8X0Xu30+33nqrctrpoR5qHCPuHAIAAEC4OAQAAIBU1Grl9PbwihUrlCdMmKDspbDzzjtP\n2VdP+iqxiPwVYH572ZuMpmVl/515DX4L5b+/GlcCbty4UdnLXD514Oijj1Y++eSTM8/v1q2b8u67\n767chOWvkuXvJW+QPHfuXGVf7eol3ohseSOvrOzlK3+/elntlVdeybzuyy+/rDxv3jxlX2HZvXt3\n5WoZO/+8eeSRRzI/++qrr5R9nPxYelPkQj879thjD+WrrrpKecSIEcpz5szJPMffMz6toNr5+Xb5\n5Zcreyl5+vTpyl5Kbuhnffr96GVin+6RN0Wjkr5r/Fj4RggR2RXGPi6+t7Tzx6Srlfv06aPs3/t+\nTvjUNp8mFRExf/58ZT+//W/2VeiNud85dw4BAAAgXBwCAABAuDgEAACAVNREnbTe/+yzzyq/8cYb\nyl6X987wPifAu8+nP/P2Au+//75yx44dlW+66abM8/13NrS7fCXN/SiUt1oYOXKk8sqVK5XPOuss\n5aFDhyp7C4CI7NJ/b1lTjcfV+TxBn8fiu8gcc8wxmef06tWrxuf4sfS2ND7Xx9s5pK/rc298w3mf\nq1ONY+dtM3yHkojsfMxZs2Yp12eeocsbSx+/vffeO/OcIrZIKys+V8zf1yeddJLyCSecoNyU72v/\nTqrreKWPL4fzz/9mXx/g6xEisvOfDzjgAOUOHToo+1x1Hy+fP5j+zrxj5I/x142IOOKII5S9LY5/\nH/pcSOYcAgAAoElwcQgAAAApWlm5qdqv+EbVn332WeZneeUo3xnA29csW7ZMedq0aZnX8vKlbzLv\npbjBgwcre4k5otF3cKgK/p7xVhm++8mAAQOUvazspeR0Jwk2kK+ZnyN+LL21hreficjv+u/nRV4b\nKM/epiH9bz/HBw4cqFyNY+efUX7sI7Itmzp37qxcn8+bvFYf9913n3KzZs2UvSwakX0vYLPXXntN\n2dsSXXHFFcqNWRr0lmppezX/77zzstAWUeXQXs1bat15553Ks2fPzjzOy7T+2bPXXnspn3baacre\nBigdu7xjkbfbSnre+Gv7+eatovxvbMxxqL5PVwAAAOTi4hAAAABStLJyU91q9lU7Xu6NyN5GPvTQ\nQ5W9ZOari7xk9sMPP2Rey1f/eWnnxBNPVL7yyiuVfSUl6sdvnT/88MPKvjLSS/l5qzLTsWBsNvPj\n5CUN33nGj+vEiRMzz/cdMy6++GJlH5e8VZHeQSCdxuFl5bFjxyq3b9++xr+9kvkx87Jk2p3BV3wX\nUs6qbYWqf3bec889yqtWrVL2KR0XXHBB5vnVsmPNv0lLuYsWLVLu3bu3sq+Ebej72n+nj2O6w4ef\nl/6cvPPVx9RLmenPSum89L9/6tSpylOmTFFO/y2dOnVSvvnmm5W9g4lPm6nPNACfEuPZu6JEZP9+\nf5xPKfnkk0+UfWekhk5P4M4hAAAAhItDAAAASFne+/dbrX571W/ZR0TsuOOOyqeccopyq1atlP12\nuDegTFdCevNZf/6NN96o7LeaUXdpmctvl3vJc//991fu2bOnsk8F8JXrhxxySOZ1G3M1YCXxcpCf\nO74S1Vf3R0RceOGFytddd53yp59+quwrjL259eOPP66cllN8NaGPcSmVrIrFy14+debHH3/MPG7N\nmjXKeU1y169fr+xl6fSzc/78+cpz585V7tOnj/Jtt92mnK6yrMZxqklasvTzZ9iwYcreaD5v9Wn6\nneQ/8/Fet26dsk/d8FWxEdmOGr4q1lfl5pWY06k5pTrevtLeS8T+XeGfdRERl156qfJRRx2lnFc6\nz+vGkP7MS/c+XnmvFZEdcx9Xn4bjjel97NKG3HXFnUMAAAAIF4cAAACQsi8r+63ybt26ZR63zz77\nKPtqMOelFb+F7iXNiP9tsP0P38+0qZTjPpb1ka7s81v/vircV5OtWLFC2cuUXr4ZNGhQ5nUr9fg1\nFS9t9O3bN/Mz35fUVy4/8MADypMnT1b2Rq6+unbMmDGZ1/WSSLWPl5egvvjiC2XfFz4iW4r3x23c\nuFHZS89eykzLhN65wadxeKNtf361j1Ge9DOtR48eyl4O9OPvn3s+dt41ICL73eX77vo0gI8//lh5\nzz33zDzfy9c+dcNf18vN5TjGPqXF91N2vilGRMQ555yjXEhnCx/jdBqB82sVP6d9Sk065SmvOfnC\nhQuV/Zw+88wzldP9zuuKO4cAAAAQLg4BAAAgXBwCAABAynLOofN5GN4dPCJijz32UPbl6nmtTH7+\n+Wdlr+NHRLz77rs1vi5tURpPOl/D58j4vBBvX/TMM88ov/7668pDhw5VTltAoP7SY9mvXz9l34XI\n5xn6uHbu3Fn53HPPVU7bLjBmm3mLLG9V8d5772Ue53PVfK6ZH3Of57nrrrsqv//++5nX8s+1du3a\nKfu8M28TkrYiytshxedNVepnp/8b08+0Ll261Pg4bzf07LPPKk+aNEk5nVPvcz79PeLzBP0z9OCD\nD859vp9/eeO63XbbRTnw4+pzMfNa8dx1112Z57ds2fJff4ePl88fXLt2beZxPv6+u5OPUW3ngf/N\nvtvNd999p+zzg/33M+cQAAAAjYaLQwAAAEjZl5W9fJGWlf1WeyFL0v22edomwm+1e1uVhpa/0jY1\nNUnbIVRSOaa2HQB8xxq/db9s2TLll156SdlLXr4jDiXKxpO+X33HDW//lLexvD//gw8+UPZWGulz\nqp2/r6+++mrldFeSww47THnw4MHKPvXGX8vPqbSUtmDBAmUvZ3nZytvleJktIqJFixbK/tnr7Tz8\n76+k8fb3+KZNmzI/83+/fz95m5VZs2Yp+444aXs1/x4aOXKksk8D8OlU6bmbVz7275dybGWTtxOJ\n83+vl/prey0vEfvr+k5F/r6PyD+W9Xm/L1myRNl3oPL3mLcx8t1d6qNyzkgAAAA0GBeHAAAAkLIs\nK/vtbS8rp6XjvI2yXd7KMi+XRWRvzx9//PF1/IsLk7eBd7nczq+P2srqeav5/Ja6797gx8lXNFdy\nWb7YfJVcRMSQIUOUvezhuxMNHz5c+a233lKeMWOGcloi9Z04vPxWyedCHn+/+vt99OjRmcf5bk3+\nWZj3ueKv27t378xreanNVzL7CldfOe2ltYhsqct/p+++4Z+plcTLtb7yOCK74rh79+7Kfiz8GNe2\nqtWnYvj55tOjfOy9pB2RHYu871E/98qFT4Mo5Ht/5syZmZ/5Dil+zPPKx57TVfr+/IZ+dvkubd5Z\nxb8bV69e3aDf4bhzCAAAAOHiEAAAAFKWZeU86Qqgut7G9VvNaYNZv+3vjUULuW2d8ufkPb+Qx5Qr\nPzZe6khXPHpp35uZ+upLb/Drt9R9RXNaVkbd+BidddZZmZ95ybFt27bKzz//vHKrVq2UvQw9YcIE\n5bQRra+Wvf7665XT8nO18TKVlx8j6r4C0h+flnj9OHfo0KHG3+/lx3RcvLT6zjvvKHs5zLsRlPu4\n+mfawoULlWfPnp15XMeOHZVvu+02ZT+WF154ofKGDRuU582bl3ktX5Xsz897H6Sfg9482T9Hy3GF\nsvP3qDd6X7dunbKvNh43blzm+b6xgp8XPm3j2muvVfapAg2dwpReN/j4v/zyy8r+mexj7x0LGoo7\nhwAAABAuDgEAACAVVVZOb8nWdcWvlzy8LBIR0aNHD2XfW7mQsjCy/Na77wfrq7EiIjZu3KjsK5SP\nPPLIGp/jj/fVkqxObhhvvvvmm29mfuYlrAcffFDZS8z+mE6dOimPGTNG2adqRES88soryk888YTy\n2WefrZy3f28l88+VtHzo3RbyGo+ne/3+o7ZVqbX9zn/UtkrzoYceUvY9fL0Rse/5XO4Nsf14L168\nOPOz5s2bK/sx8hXGvlK8V69eyqtWrcq8lk/X8DKjj5d/1voq3ojsdAEfi3L/7vL3spfo/fPJv9+9\nmXtExJdffqns0x38PerNyb2Dg5fnI7LvZT/Gnn1cvPQdEXH77bcrv/HGG1ETf+8cd9xxNT6mPsr7\nLAQAAECj4uIQAAAAUpZ1Gb/t7bfmaysr5/1/vx3vq4HSpq6nn366cqU2by0Wb77qJct0H0y/XX7E\nEUco+5j7nqR+S97LXOVepvovePnxkUceUU5X43nZa9CgQcp5xzxvhez555+feZw39Z08ebLywIED\nlX16RzVOHfDzKCLiq6++Um7durVy3p6uhaxwjSiszJiWq73Z+QsvvKDsq3V9//NC9pgvF36OpP8u\nn/qSNw3AV9hefPHFysOGDcu8lo+LT4nyz1H/fktL/14yraQpGn4sR40apexdLqZPn67sU5YismPW\nrFkzZZ8C5c3NvZNGeh54+dr/Ll+p7+dxek6nJe9/+PQE/+z0qQYNxbcmAAAAhItDAAAACBeHAAAA\nkLKfaNDQ+WQ+J8Pn7HgrlIjsZtzFnsOWzmMox/lVPo/D51FMmTJFOZ3b5O1NvAu9j9O3336rfMAB\nBygPHTo093Xx73y8VqxYoZy2PMmbC1pX6et6Ow/fCcT/Ft+xoBzPiYZKP6N8Vxo/X/r166fsc5Jq\nOy/qOs/QdwWJiLjuuuuUfZ5dnz59lI8//njlch8/P14HHnig8sknn5x5nM/t87mBfiz93PN5ud5i\nJSLi8ccfV/YWKj731+flpnPlq2EudsuWLZWvueYaZZ+nN2vWrMxz5s+fr+zni4/d008/rezjkrYL\nymun599bte005C3B/DPR52F7e7DGPI8q/90BAACAgnFxCAAAACn7snJDS4Z+a92XkXv5IyLbTb7Y\nKuH2v5dNnnvuuRrz77//nnmOt3Tw0piXpf22v5eS9913X2XKynXn77nevXsrp136165dq/zNN98o\n55Vj8sYibZHjbYm83ZHvcuBtdLztUW3jXdddk0pZeszmzp2rPGPGDOWRI0cq+zni5azaWpn4ueul\n7PHjxyv7uERkx69du3bK999/v3Jtu7KUMy/f9uzZM/Mz31lj2rRpyv3791f2MvycOXOU0/LnmjVr\navw9J510krKXKcv9/V4f/m/297hPSRkxYkTmOcOHD1f20r+fb96GyXdw8s+qiPz2NV4iHjBggHL6\nfvHx811VitF6qPyvOgAAANBouDgEAACAlH1ZuT789rCXTFq0aKHsK+kiil8CqaTyV0R2VbivqPJS\nvq/gisjufrJs2TLl/fffX/m0005TPvXUU5UrtWRVLF4Oueqqq5R9l4GIiNmzZyvfcMMNygcffLCy\nT9HwcoqX37xEFhFxxx13KC9YsEB58ODByl5KzpPuUFHu55L//eluCH7++OrhK6+8UnnixInKvro/\nLVH7+bZkyRJl3yXCp4GkU1/8tb0jgZfGKpWX/LzLRUT2mPmqWB+X77//XnnDhg3K6Q5SO++8s/KQ\nIUOUfYVyub/fiyE9Rj7dwrM78sgjlb1jQ2P8/lLBnUMAAAAIF4cAAACQLYq44XnJ7Kyet/rON4tP\nS1aHHXaYct6t5hLQGPenm2Sc/Jh7CfHVV19V9saiEdnS8EEHHaTspRpvclrC4+JKdoxyf5l9RqSl\nrZUrVyo/9dRTyq+99pqyTw/wKQW+Em/p0qW5v9ObvPo0gksuuUS50NXKBSqLMUqb43spefTo0crz\n5s2r8TleCk7H1Y+hl//9OLdu3VrZy/0REddee61yEzYrb+g4Ff07yY+/r+73Va7eAcCnOqVTOnyT\nAH9ciXW3KLsxqkI1jlFJvYsAAADw3+LiEAAAAFKVZWVfmeerZb38la4E3GGHHZSL0YCynsqiHFbl\nKmqM/FzyFemrVq1S9kbnixcvVvZ9krt27Zp5XZ9G4GWy9u3bK3sp08ug1VJW/p9fmFP+X716tfLy\n5cuVfRWy//+IiP3220/52GOPVfbypa88TkuZRVqBScmy9DFGpY+yMgAAAGrHxSEAAACkKsvKzv/9\nhTaeLtWmlVGm5bAqU3Vj5OdV2mw5T16ZuEzKlRFlNkZlipJl6WOMSh9lZQAAANSOi0MAAAAIF4cA\nAACQku3JUqhC5wkWokznGQIlzc+dRt4hAwDQBLhzCAAAAOHiEAAAAFL2ZeXayr2FlJwpFwMAAGzG\nnUMAAAAIF4cAAACQYu6QAgAAgBLHnUMAAAAIF4cAAAAQLg4BAAAgXBwCAABAuDgEAACAcHEIAAAA\n4eIQAAAAwsUhAAAAhItDAAAACBeHAAAAEC4OAQAAIFwcAgAAQLg4BAAAgHBxCAAAAOHiEAAAAMLF\nIQAAAISLQwAAAAgXhwAAABAuDgEAACBcHAIAAEC4OAQAAIBwcQgAAADh4hAAAADyf+yAvc6y2N/2\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc71278bba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Coming soon..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  },
  "nav_menu": {
   "height": "381px",
   "width": "453px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}