15_autoencoders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 15 – Autoencoders**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 15._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"autoencoders\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "A couple utility functions to plot grayscale 28x28 image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
    "    images = images - images.min()  # make the minimum == 0, so the padding looks white\n",
    "    w,h = images.shape[1:]\n",
    "    image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# PCA with a linear Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Build 3D dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "rnd.seed(4)\n",
    "m = 100\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "X_train = np.empty((m, 3))\n",
    "X_train[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 2] = X_train[:, 0] * w1 + X_train[:, 1] * w2 + noise * rnd.randn(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Normalize the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Going to need TensorFlow..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's build the Autoencoder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2  # codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden = fully_connected(X, n_hidden, activation_fn=None)\n",
    "outputs = fully_connected(hidden, n_outputs, activation_fn=None)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_iterations = 10000\n",
    "codings = hidden\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run(feed_dict={X: X_train})\n",
    "    codings_val = codings.eval(feed_dict={X: X_train})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure linear_autoencoder_pca_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFzlJREFUeJzt3X+sXGWdx/H3t/dWoHAbalSSgrSU8qP0miKurQGES1gs\nsFmMxI0SF424upruGnBVNguxLRhYd038tbrCCijgKqYLBZcSQzfeotwumGWptrQlaH/oxUUNpXNL\nrXLLd/94znDPHc7MnDN3zpw5M59XMpk5Z56ZeWbmzHee5znPD3N3RETyMKvoDIhI71KAEZHcKMCI\nSG4UYEQkNwowIpIbBRgRyY0CjIjkpvAAY2arzOwnZnbIzG5vkO4DZjZpZhUzm4iuz+tkXkUkm8Gi\nMwCMAzcCK4GjmqQdc3cFFZGSKDzAuPt6ADN7K3B8wdkRkTYqvIqU0ZvN7DdmtsPMrjezsuVfpK8U\nXoLJYBMw7O57zGwp8D3gJeBzxWZLROopTYBx992x29vM7AbgkyQEGDPTCE6RnLm7NUtT9ipG3Tfo\n7qW4rF69uvA89Gp+ldf8LmkVHmDMbMDMjgQGgEEzO8LMBhLSXWxmb4hunw5cD6zvbG5FJIvCAwwh\nUBwErgXeF92+zszeGPV3OSFKdyHwUzObAP4TWAfcXESGRSSdwttg3H0tsLbO3UOxdJ8CPtWRTHXQ\nyMhI0VnIpEz5VV6LZ1nqU2VhZt6L70ukW5gZ3geNvCLSxRRgRCQ3CjAikhsFGBHJjQKMiORGAUZE\ncqMAIyK5UYARkdwowIhIbhRgRCQ3CjAikhsFGBHJjQKMiORGAUZEcqMAIyK5UYARkdwowIhIbhRg\nRCQ3CjDSMyYmYPPmcC3dofAAY2arzOwnZnbIzG5vkvYaM/u1me0zs2+Y2exO5VOKkTZoTEzA298O\n550XrhVkukPhAQYYB24EbmuUyMxWAp8GLgAWAidTfzUC6QFZgsbWrbBtG0xOwlNPhdtSvMIDjLuv\nd/cHgOebJH0/cJu773D3/YSg9MHcMyhtk7UK89hj6YPG8DAsXQqzZ8MZZ4TbUrzCA0wGS4Etse0t\nwBvMbF5B+ZEMslZhJibgmmtCcAE45RQ4cKD+44aG4Ec/gkceCddDQ8nppLMKX3gtg2OA/bHt/YS1\nqYeAfbWJ16xZ88rtkZGRnl3YqiySqjBve1vj9Dt2hNsDA3DoEFxySSiZ1AsgQ0ONn1NaNzo6yujo\naObHdc3Ca2Z2I3C8u19V5/4ngc+6+7po+7XAb4HXufu+mrRaeK3LVEswTz0VqjDNShnx9AsWwK5d\ncPhwqAI98ogCSdF6ceG1bcCy2PaZwHO1wUWKVa+dJWsVJp5+06bQxqL2lfIpvARjZgPAbOAzwAnA\nh4FJdz9ck24lcAdwIfB/wDrgv939uoTnVAmmANVSx7ZtjasyrT539XnVvlK8tCWYbggwq4HVQDwj\nawnB5Clgibv/Kkp7NfD3wJGEAPMxd38p4TkVYDpoYiK0mbz4YmgnmZxUVabXlSbA5EEBpnPipZbT\nTw/7du5M184i5ZU2wJTpLJJ0ofjZoZ074aGH4OijVZWRQAFGZqTawa16dmj5cgUWmaIqksyYGmD7\nj9pgevB9FanakDs83Lz/Spp0Um692A9GCpK2m79GNEstBRhpKu1IZY1olloKMNJU2pHKGtEstdQG\nI6mkbchVg29/UCNvD76vTlJjrTSiRl5pmRprpV0UYHpcKxNhq7FW2kUBpoe1WhJRY620iwJMD0tT\nEkkq4Wj6SWkXBZge1qwk0qiEU51+UsFFZkJnkXpco9PGmzeH4KL5WyQrnUXqQfUabBs15DYqiait\nRfKmEkxJVKszW7fCSSeFeWrnz5/5NJXqGCetUEe7HntfmzeHQHI4mql48WJ44okQcFTNkU5TFanH\nDA+HkkvV7t3w+OP1qzlaCF6yyOt4KTzAmNk8M7vPzA6Y2S4zu6JOutVm9kczq5jZRHS9sLO5Lc7Q\nUKgWLVoUticn4eqrw+3aU8rqiStZ5Hm8FB5ggK8Bh4DXA38J/KuZLamT9rvuPtfdh6Lr3Z3KZDeY\nPx9uuSWsdAhhDtxt217dkJvU/0UlGqln69ZwmZwMx0o7e24XGmDMbA5wOXC9u//e3R8FHgCuLDJf\n3WzFiuaLkNVWm048USUaqW/BgnCsAAwOhuOlXYouwZxKWGTt57F9WwgL3Sf5czP7nZn9zMw+mn/2\nuk+aXra1afbs0diiMsta+syafs+ecGxAOImwd29r+UxSdICpXdCeaDvphOk9wBJCVeojwGfM7D35\nZq87pellG0+j/i7llbV9pJX2lDyPj6KXLTkAzK3ZNxd41cfi7jtim5vN7EvAuwmB51XWrFnzyu2R\nkRFGRkZmmNXyqpZo1N+lfJLa0xp1Q8iaHtIdH6Ojo4yOjmbOf6H9YKI2mOeBpdVqkpl9Cxh3939o\n8thPA8vd/d0J9/VcPxjpT9USSXXdqWYdKbOmb1Xb+8GY2RwzW2dmL5jZqmjflWZ2VquZdPeDwL3A\nDdHznwNcBtyV8PqXmdmx0e3lwMeB9a2+drfRWR5JknVke7eNhE9dgjGztYT2kV3AW4GFwOPu/sUZ\nZcBsHnA7cBHwO+Bad7/HzM4FNrj73CjdvwPvAF4D/Ar4qrt/tc5zlqoEMzEBZ58NO3aE9Z3Hxoo/\nMEQaaftQATM71t1fiG6/BVjp7jdF25cDC4AVwHZ3X9tyztugbAFm40a46KLp2xdeWFx+RJpJG2BS\nN/LGgstFwJJYcFkEHOvuXzCzI4GdZva0u3+nxbyLSI/IdJo66sZ/nLt/ObZ7GFgD4O6HgMeBc9qV\nwX5Q7Tw3MBCuly8vOkci7ZGlkfcDwAF3vzu272ZgA3BpLOnxwA4ktaGh0O7y4x+r/UV6S6o2GDN7\nF6H7/m7gaaASbX/d3e+PpXsz8F3gTHf/fR4ZTqNsbTDS23pxjam2naY2s+OA09z9cuArwMeAfwbu\nqgkuRxCqSu8oMrh0g1ZPOetUde/Jc6RyKY4Xd2/LBbgROCG6vbhdz9tiXrwolYr7smXug4PhulLJ\n93HS3cbGwncK7rNnu2/e3PwxlUp4XKNjoFJxHx4Ozz083PnjJfqNNf0ttmUskpl9hNAW85KZHQ/8\naTuet4xaXbRMi531pqzjfNKWeB57bGqKha1bw+Rj3WjGAcbMzibM6fJj4NfAXuA3M33eMkgqorY6\ncEwDEsslbfUka8/aXvuj0Zy8LWo02XarE2lrAu5ymOlE62meu9lYomrv7+3bYcmSzp991KTfOb8v\nrSnUv/L+7tP+0RT5h6QA06ESTN6jVqX76LtXgMk9wICqNP2s3797BZgefF8i3ULrIokUJO85dMtE\nAaZDevkgkimdmEO3TBRgZihN4Oj1g0imxPuxpOkA12v9XmopwMxA2sDR6weRTBkeDrMSQlgC5Oqr\nG/+h9HoHSwWYGUgbOHr9IJIpQ0PwhS+EBcwgdIRrVor5/OfhoYd683S3ziLNQJb+EP1+WrOfVHvZ\nbt0atoeHk3va5tkjOG86Td2h96XAIUk2boRLLmnc27fMvcFLc5razOaZ2X1mdsDMdkXTctZL+7lo\n6djfmtnnOpnPetKssij9Z8WK5tXifqg6F72yI4SR2IcIS8KeBTxoZk+6+/Z4IjP7a8KaSW+Kdm00\ns5+7+60dzW1OenHWs35V/S43bAhtMPUK0/2w4mY3rOy4DzjDp1Z2vBP4ldes7GhmjwJ3uPs3ou2r\ngL9y97MTnrdUPXnLXBeX6eLfZfVs0vbtcNJJsGkTzJ9fbP7apSxVpFOByWpwiWwBkgqLS6P7mqUr\nHZ3G7h3x73L79nA5fBieeQbOP7//+kAVHWCOIawWGbcfSPr/rk27P9rXlbL03O2Huni/iH+XS5bA\nggVT9+3Z039/HkW3wRwA5tbsmwsk/Sxr086N9iVas2bNK7dHRkYYGRlpNY+ZZa3y9ENdvF/UfpcT\nE6HksmdPuf88RkdHGR0dzfy4bmiDeR5YGmuD+RYwXqcN5nZ3vy3a7to2mDKffpRs0jTO92JXhlK0\nwbj7QeBe4AYzm2Nm5xDOFN2VkPxO4BNmNt/M5gOfAO7oXG7TU5WnP6QdKtLPXRmKboMBWAXMIUwU\n/m3go+6+3czONbNKNZG73wJ8H/gZ8FPg++7+b0VkuJmsEz1LOalxvjn15J2htEVk9XHpHdXvc8EC\nuPTS/pw6M20VqehG3tKamAhr01xzDezYUb8xV31cekvt97lhA+zd21vtK+3UDVWkrlXvVHP1ILvk\nkqnFr+oVkbMUozUpVfer/T737u3f9pU0FGDqaNSAFz/IAAYGGo83Of30kOa00+o3+GpSqnJQA342\nCjB1NCp5xA+y4WH4wQ+aV32sSW1VDYbdo1FJUg342SjA1NHonyp+kI2NwYUX1j/Qtm4NbTSTk7Bz\npyal6ga1ASS+naYk2c+nnTNz9567hLc1c5WK++bN4Xomz7Fsmfvs2eG60XO14/Wkser3MTgYrsfH\np28//HC4DeE727y56Bx3p+g31vS3qBJMA0n/VEnF53YVqfXP2D71vpPaquiDD07fNlNJsq3SRKGy\nXWhTCaZW7b9fpZL8jzg2plJIkSoV9+Hh8J0MD0//LmpLlNUSTLyEqZJkc6QswaijXQZJY4zcp/YN\nDsLChbB7t/q8FGnjRrjoounby5dPdXaE6WODenGsUN5KMRapbJIaYuP7qsFlcjIcsM1mk5fOOHhw\nesMtTK+KqmqaHwWYDOLtKRs2TM0aX923adPULGaTk83XxJF8rFgRAv/AQLg+6ih1ASiKqkgtaNT9\nf+NGuPjiMItZ1qkaNGapfeLVHki/vIyko2VLcnxfjeZ7efbZMMFQ1nYYjVnKl9pZ2kttMDmq1ylu\nYiKMrt29O7THbNiQ/mBWT958qZ2lGAowLajXtyUeJPbsCQPh0qoNWieeqIGPUn49W0WqVLzj7RlZ\nlpKt9/ht20JwufTS9lWX1LYj7db3VaROjkyu9hqFmQ2Eqxbjq7PPN6supZneodtHaad9DyrNlVSa\n3nhluwBtG09SqTTumZvUu3em0oxfSvu6Dz/sPjAQPovBQfeNG2eet1Z6Kic9Ls17yOPzlZkjZU/e\nwoNBHhcg9QDDRtIc3GNj+QyOGx93v/XWcJ0kzetWu8yH/sbhUtt1PotGn0ejwFPvcWneQ16fr8xM\n3weYdownSfsjbkcwS3rONP/sjV53bGyq9FK9DA5m+5HGA0ft57FxY9hXOyK5Ni/1PscsJbXZs90X\nL64fcKWzShFggHnAfYQF1HYBVzRIuxr4I1AhLMxWARbWSduWDzFt8Gj34Li0/9rNXrdSCT/KeIBZ\nvLhxyWN83P1LX3K///5XB46dO90XLQpBa8mSqQGFixdPBbKk/Db6HNN8duPjU6+halJ3KEuA+U50\nOQo4B3gBWFIn7WrgzpTP27YPsoiRte0sFVV/nEklgNqS0s6d7kccMRWMFi6cHugWLZp+XzyoVF+j\nUUmk1c9R1aTu0/UBhrAW0h+Ak2P77gRuqpO+kABTlHYGtnrtObU/3GuvnV7amTVrenCaNWvqvmrJ\nJT7tQV6BOI9qqMxMGQLMmcCLNfv+Dri/TvrVwD7gd4TF1z7a4Lnb+VmWWrOG2fgPt7YEc8YZU4Fj\nfHx6g/HwcL5BJel9aI6W7pE2wBS5LtIxwP6affuBer1H7gFuAZ4D3gb8h5ntc/d7khKvWbPmldsj\nIyOMjIzMMLvt0elOb0lDEKrjpmoXah8agl/8Au69NywqNjIS9s2fH9KPjU1NQbF8+fT78lbtIyTF\nGB0dZXR0NPPjcuvJa2Y/BM4Hkl7gUeDjwKPufnTsMZ8Aznf3d6Z4/muBP3H3v0i4z/N6X1nFAwp0\nfkBjK72L1fNXmim8J6+7X+Dus9x9IOFyHvA0MGBmJ8cetgxIO8zPgaZvsCgTE2HqhrPPnupF+9hj\nnR/QmHWZjW7v+SvlUthQAXc/CNwL3GBmc8zsHOAy4K6k9GZ2mZkdG91eTigBre9UfrN49lk46yxY\nuXL6yo9FTSidZSSxRnVLOxU9FmkV4WzSb4BvExputwOY2blmVomlfS/wTLTvm8DN7n53h/P7Kklr\n7Jx/PjzzDLz8ctg3OBgCyvLl3bdoV23+tT6TtFPPjqbuxPtKmiRq69apyagAFi2CW2+dahTNOz9Z\n2k7qTXKlyZmkmcLbYPpBUnUiXgJYvDj8aBut/NgurbSd1KsOaXImaRcFmBlIqk7EG1WfeKJzp3Fb\naTtRdUjypirSDExMhDNDZp2pAjXLSyuTXak6JK3QpN8tvK8sbRjdOEm3goV0itpgMsrahtGNp3PV\ndiLdRgEmkjVgqP1CpDlVkSKtdqlXlUT6kdpgWmyDUcAQaU4Bpgffl0i3UCOviBROAUZEcqMAIyK5\nUYARkdwowIhIbhRgRCQ3CjAikhsFGBHJjQKMiORGAUZEclNYgDGzVWb2EzM7ZGa3p0h/jZn92sz2\nmdk3zGx2J/IpIq0rsgQzDtwI3NYsoZmtBD4NXAAsBE4G1uaZuU5pZbW8IpUpv8pr8YpcF2m9uz8A\nPJ8i+fuB29x9h7vvJwSmD+aawQ4p24FVpvwqr8UrSxvMUmBLbHsL8AYzm1dQfkQkhbIEmGOA/bHt\n/YRlYzVri0gXy2U+mGYL30drU1fT3ggc7+5XNXi+J4HPuvu6aPu1wG+B17n7voT0mgxGJGdp5oMZ\nzOmFL2jzU24DlgHrou0zgeeSgkv0+k3fuIjkr8jT1ANmdiQwAAya2RFmNlAn+Z3Ah8xsSdTuch1w\nR6fyKiKtKbIN5nrgIHAt8L7o9nUAZvZGM6uY2QkA7v4D4J+AHwK7osuaAvIsIhn05Jy8ItIdynIW\nSURKqGcDjJm9JhpSsNvM9pvZ/5jZxUXnq56sQyc6zczmmdl9ZnbAzHaZ2RVF56mebv8s40p4nN5l\nZs9Ged1hZh9qlD6Xs0hdYhDYC7zd3X9pZn8GfM/Mht19b8F5S1IdOrESOKrgvCT5GnAIeD1wFvCg\nmT3p7tuLzVaibv8s48p2nN4EXOXuL5nZqcAmM3vC3f83KXHPlmDc/aC73+Duv4y2HyQ0Dr+l2Jwl\nyzh0oqPMbA5wOXC9u//e3R8FHgCuLDZnybr5s6xVwuN0u7u/FG0aoa/byfXS92yAqWVmxwGnEPrU\nSDanApPu/vPYvi2EIRzSRmU4Ts3sq2b2IrAdeBbYUC9tXwQYMxsE7ga+6e5PF52fEqodqkG0raEa\nbVSW49TdVxGOiXOBe4E/1Etb2gBjZj80s5fN7HDC5ZFYOiN8aX8A/rab89rFDgBza/bNBSYKyEtP\n6objNAsPxoA3Ah+rl660jbwZhiPcBrwOuNTdD+eYpbpyGDrRaU8TelufHKsmLaOLi/ElVPhx2qJB\n+rUNxsy+DpwOXObufyw6P41kHDrRUe5+kFAUvsHM5pjZOcBlwF3F5ixZN3+WScpynJrZ683sPWZ2\ntJnNiiaCey/wX3Uf5O49eQFOBF4mDEGYiC4V4Iqi81Ynv6uj/B6OXT5TdL5i+ZsH3EeoLu0G3lN0\nnsr6WdbktTTHKaGENUo4O/cCoaH/qkaP0VABEclNT1eRRKRYCjAikhsFGBHJjQKMiORGAUZEcqMA\nIyK5UYARkdwowIhIbhRgRCQ3CjAikhsFGBHJjQKMdFw0Inudmb1gZquifVea2VlF503aS4MdpePM\nbC1hRrxdwFuBhcDj7v7FIvMl7acAIx1nZse6+wvR7bcAK939ptj9iwgTjF9VVB6lPUo7o52UVyy4\nXAQsqQkuf0OYUX9BQdmTNlIbjBQiWrjtOHf/cny/u/8L8M1CMiVtpwAjHWdmHwAOuPvdsX03F5gl\nyYmqSNJRZvYu4J3AbjM7njA95JXA1wvNmORCAUY6JlpU7DR3v9zMTgLWE+Z5/ZS7319s7iQPCjDS\nMe7+HPCP0e1dhKVPpIepDUa6kUUXKTkFGOkqZvZh4JPAm8zss2Z2StF5ktapo52I5EYlGBHJjQKM\niORGAUZEcqMAIyK5UYARkdwowIhIbhRgRCQ3CjAikpv/B608kTQ3sV8EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85c49d15c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked Autoencoders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's use MNIST:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train all layers at once"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders). We will use ELU activation, He initialization and L2 regularization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import fully_connected\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer() # He initialization\n",
    "#Equivalent to:\n",
    "#initializer = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "with tf.contrib.framework.arg_scope(\n",
    "        [fully_connected],\n",
    "        activation_fn=tf.nn.elu,\n",
    "        weights_initializer=initializer,\n",
    "        weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n",
    "    hidden1 = fully_connected(X, n_hidden1)\n",
    "    hidden2 = fully_connected(hidden1, n_hidden2)\n",
    "    hidden3 = fully_connected(hidden2, n_hidden3)\n",
    "    outputs = fully_connected(hidden3, n_outputs, activation_fn=None)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = tf.add_n([mse] + reg_losses)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train it! Note that we don't feed target values (`y_batch` is not used). This is unsupervised training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0245285\n",
      "1 Train MSE: 0.0127265\n",
      "2 Train MSE: 0.0108945\n",
      "3 Train MSE: 0.010546\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_all_layers.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This function loads the model, evaluates it on the test set (it measures the reconstruction error), then it displays the original image and its reconstruction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure reconstruction_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAGoCAYAAAAHJ+8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHaNJREFUeJzt3VuMXmX1OOC3h+l0Oj3QKe0wQ6Eg5VAMmkIQkAKCCTck\nGjRcIBcYNJGYaCKJGkhM1Bu9U24MGg0omGg0UdSkCIgQJCgVKIciUEARaCv0QDu0nR6m/G/+N7/k\nXa/uj4/Vafs8l2vPevf+DvOt7GSt/c549913CwCQZ+bhvgAAONYovgCQTPEFgGSKLwAkU3wBIJni\nCwDJFF8ASKb4AkAyxRcAks1OPJdHaXE0mXG4L+BotXnz5s6/FYf7SX0zZtS/Dq3rinJaenmdXc/T\n72vu5TzT9fwt0bWNjY1VD7jzBYBkii8AJFN8ASCZ4gsAyTIbrgD+q6xGnH423PTzmg8dOtR5rZkz\n4/uoXtbrqt9NWv1sEutFRgOdO18ASKb4AkAyxRcAkim+AJBM8QWAZIovACQzagQc8Y7E5wT3cwSo\ntVbX8/TyvrRGnaJjvVxz6zz9FH2erfem6/vszhcAkim+AJBM8QWAZIovACRTfAEgmW5n4IjQ6jTt\npTu1F127mlsdsLNmzarGp6amOp+/9Tqj9fbv31+NtzqK58yZ0zmnn5seZGwS0SsbKwDANKf4AkAy\nxRcAkim+AJBM8QWAZIovACQzagQcEVojK9GYRz/HXFqi87RGcKJrHhgYCHOi9VrjSQcPHqzGBwcH\nq/EDBw6Ea01OTlbje/bsCXOi19N6nfPmzavGo1Gj1uvv50hXP7nzBYBkii8AJFN8ASCZ4gsAyRRf\nAEim2xmYVnrpXO5lw4FeHtIfdcj20h07d+7czmtFGxu0RJ3I0flb54g6oVsd0tF5ongpcVd39N5E\nHc2tY/3udraxAgBMc4ovACRTfAEgmeILAMkUXwBIpvgCQDKjRsBRqzUyEo0aRZsHlBKPp+zdu7dT\nvJR4BGZkZCTMmT27/pMdxUspZXR0tBqPNi9ojcxEr6c1thOt1zpPtBlE9J61Xn/0Obc2vehlDK3r\neJI7XwBIpvgCQDLFFwCSKb4AkEzxBYBkup1LKX/961+r8VtvvbUaP/HEE8O1hoaGqvHrr78+zIm6\nG1tdj3C06vqA+pZWB2rUUbt79+4w5/XXX6/GN2/eXI2/8sor4VpR5+6SJUvCnAULFlTjrW7fxYsX\ndzrP4OBguFbU7bx///4wZ2BgoBpvbcYwPDxcjUe/iccdd1y4VvR9anVoR1rfJxsrAMA0p/gCQDLF\nFwCSKb4AkEzxBYBkM7o+j/I9SDtRV2eeeWY1vnHjxpTzL1q0qBq/8MILU86f5ZRTTqnGb7755mr8\n5JNPfh+v5j3rX0su/8emTZs6/1ZEv2NRR3MpcYfujh07wpyXXnqpGn/ooYeq8aeeeipcK+rQXbhw\nYZgTXfNzzz0X5syfP78aj7qgW6//7bffrsZXrlwZ5px99tnV+LZt28Kc6DfxkksuqcbPPffccK3o\nGdat70bUiR7FS4m/g+Pj49XfCne+AJBM8QWAZIovACRTfAEgmeILAMkUXwBIZmOFUspvf/vbanz9\n+vXV+Ac/+MFwrQ0bNlTjf/vb38Kcu+++uxr/4x//GOaceuqp1fg///nPMKer1sPax8bGqvHXXnut\n83miEaSvf/3rndfiyNfLxgrRmEdrlPLQoUOdc6KH8Uf/K2eccUa4VjTqM3fu3DDnX//6VzXeGk+K\njkWjNps2bQrXmjmzfr92/PHHhznReNYjjzwS5kS/sdFYaOs7Ex2LNnwopbfvk40VAGCaU3wBIJni\nCwDJFF8ASKb4AkAy3c6llFWrVnWKt3zoQx+qxq+99tow57vf/W41HnU2lhJ3O7/yyivxxXU0Z86c\n8FjU7RxdVymlvPXWW9X4WWed1e3COKr1c7OXVgdq1FUcbXhQSty9PDw8XI23HsQfdTvv27cvzNmy\nZUs1Pjk5GeaMjo5W49u3b+90jlJKWbZsWad4KfE0R2tjgz179lTj0SYJg4OD4VrR5xxtUlFKb93O\nXb+37nwBIJniCwDJFF8ASKb4AkAyxRcAkim+AJDMqNE0ELXC9zKC08t4VC+ijSK2bt0a5lxwwQXV\n+JVXXtmXa+Lo0MvGCpHWqE+0GUJrxC4adYnGeYaGhsK1ovGY1qhR9Jswf/78MCfaQGJiYqIaP3Dg\nQLhW9Dp37doV5qxdu7Yaj97LUkpZsmRJNb5y5cpqPNrwoXWs9d1ovQf94s4XAJIpvgCQTPEFgGSK\nLwAkU3wBIJluZ0K7d+8Oj1199dXVeNRZWUop3//+96vxVkcox57oAfW9dEH30u3cyomORR210TlK\nKWVqaqoab22SEP2vtK65a7fvwoULw7WiDSTWr18f5kSbvbQ2sLjooouq8UWLFlXj0cRIKe3fpMPJ\nnS8AJFN8ASCZ4gsAyRRfAEim+AJAMsUXAJIZNSJ0xx13hMe2bNlSjUcPRC+llBUrVrzXS+IYEI0U\nRSNIvYrWaz2kP7q2aGxo79694VrRw/v7/VD/6BoOHjxYjbfGo1577bVq/Fe/+lWY8/TTT1fj0ThR\nKaVcdtll1fiyZcuq8dY4UT+/T61xt67rufMFgGSKLwAkU3wBIJniCwDJFF8ASKbbmfLyyy9X4zfd\ndFPntR599NHw2AknnNB5PXgvWt2p0bGoC7iUuKM1ymltThJ1NQ8MDIQ5USf2vn37Op8n2kBhz549\n4Vr3339/Nb527dowJ5qAuPDCC8OcE088sRqPXn/Ubd7SS4e0bmcAOIIpvgCQTPEFgGSKLwAkU3wB\nIJluZ8rvf//7arz1jNlrrrmmGv/ABz7Ql2vi2BV1jbY6TXsRdci2uo2jruL9+/d3OkcppQwODlbj\nQ0NDYU7X6yqllEWLFnU6z/bt28O1nnjiiWp8YmIizLnkkkuq8TVr1oQ5IyMj1XjUodzqUJ81a1an\ntXrN6cqdLwAkU3wBIJniCwDJFF8ASKb4AkAyxRcAkhk1Oka0xoZ+85vfVOPRKEQppXznO9+pxqMW\nfXivWg+uj8aQWjnRd3VycjLMif6PovMvWLAgXGvevHnhsUg00hNtOFBK+/+45rHHHguPRaNG0WhQ\nKfFI0VlnndXpukppb/rQVeu7EY0U9bJRR8SdLwAkU3wBIJniCwDJFF8ASKb4AkAy3c7HiJ/85Cfh\nsYcffrga/8xnPhPm2ECBI120GUKrCzbadCHqjm11NEfdsa1NAvbu3VuNtzqaow7t//znP9X42rVr\nw7VefvnlavzKK68Mcy644IJqvLWBRfQ6o9fS6vbuZaOO6Fgrp/W9qXHnCwDJFF8ASKb4AkAyxRcA\nkim+AJBM8QWAZEaNjjLr16+vxr/0pS+FOccdd1w1/u1vf7sv1wRddH1AfUtrbCcaD2qZPbv+kzln\nzpxqPNoIoZT42lojK8PDw9X4okWLwpzt27dX43fffXc1/oc//CFca2hoqBqPNk8opZTR0dFqvPXZ\nRBsoRJ9ZL6NWrU1gjBoBwFFI8QWAZIovACRTfAEgmeILAMl0Ox+hogePX3vttdX41NRUuNZ1111X\njds8gcOhlwfhRzmtB+5Heum2jrpwW/93vWzssHjx4k7nL6WUdevWVeN33XVXNR51R5dSyvXXX1+N\nf+QjHwlzoq7iqAu5lPg9iDZjmDt3brhWL3rphO/KnS8AJFN8ASCZ4gsAyRRfAEim+AJAMsUXAJIZ\nNZrGWu3uV111VTX+wgsvVOOrVq0K1/rWt77V7cLgMOj64PpS2qM+0Xqth/RHYzPReVqbB0QWLlwY\nHotGap5//vkw5xe/+EU1/uKLL1bjl19+ebjWNddcU40ff/zxYU40UtT6bKIRsegzizZiKCX+zHr5\nPrVyuo6oufMFgGSKLwAkU3wBIJniCwDJFF8ASKbbeRprPeD8wQcf7LTWnXfeGR4bGRnptBa8n3rZ\n2KCXzRiiaYLWA//37dtXjfeygcPw8HA1Pm/evDBn27Zt1fgjjzwS5jzwwAPV+NKlS6vxqKO5lFLO\nP//8ajzqKC6llImJiWq8l885+mxa54++G61pkl4+z66bMbjzBYBkii8AJFN8ASCZ4gsAyRRfAEim\n23ka2LlzZzV+4YUXdl7rrrvuqsZXr17deS04HHp55m7UOdvqgo1yWp2uAwMD1XjUhbtgwYJwreja\nou7gUkp58sknq/F77rknzIk6pK+77rpq/JJLLgnXil5P65qjZzjPnh2Xn9bnVtP6zKIu5Fa3dT+/\ngxF3vgCQTPEFgGSKLwAkU3wBIJniCwDJFF8ASGbUaBq4/fbbq/FXXnml81pr1qypxnt5iDlMJ72M\nhrRy+vnw/MHBwWq8NTKzf//+avz1118Pcx566KFq/N577w1z5syZU42feuqp1XhrY4fomlujRtH7\nHG0sUUopk5OT1Xj0frY+yyjn4MGDYU70vWmNIHUdT3LnCwDJFF8ASKb4AkAyxRcAkim+AJBMt3OS\njRs3hse++c1v5l0IHKF6edh9P7tTS+k+NRB1R5cSd/Q+9thjYc66deuq8Vbn7tjYWDW+YsWKMCey\nZ8+earzVIR29B9FmFKXEmy5En1nrfW4di0TnaX1nbKwAANOc4gsAyRRfAEim+AJAMsUXAJIpvgCQ\nzKhRkocffjg8tmvXrs7rrVq1qhofGhrqvBYc6fq5cUhrnCR6gH90/taYS/R/H43zlFLKSSedVI1/\n/OMfD3M+/OEPV+Onn356Nd56L6PX08qJjk1NTYU5keiz6WWTjBYbKwDAUUjxBYBkii8AJFN8ASCZ\n4gsAyXQ7T2Mf/ehHw2P33XdfNa7bmSNd1Gna6qjt5UH4Xc/fWq+1SUAk6tCNupNLKWV8fLwab3VI\nn3LKKdX40qVLq/HWexZ1KPe7Q7qfXeX93JCjl+9GxJ0vACRTfAEgmeILAMkUXwBIpvgCQDLFFwCS\nzeilDbtHaSeCBP17kj//x6ZNm/r2W9HLhguJv4lVrQ0HopGaVs7AwEA1PmvWrG4XVnrbcCAaG2qN\nB/VTdP6sz3lsbKz6prnzBYBkii8AJFN8ASCZ4gsAyRRfAEiW2e0MABR3vgCQTvEFgGSKLwAkU3wB\nIJniCwDJFF8ASKb4AkAyxRcAkim+AJBM8QWAZIovACRTfAEgmeILAMkUXwBIpvgCQDLFFwCSKb4A\nkEzxBYBkii8AJFN8ASCZ4gsAyRRfAEim+AJAMsUXAJIpvgCQTPEFgGSKLwAkU3wBIJniCwDJFF8A\nSKb4AkCy2YnnejfxXPB+m3G4L+BotWnTJr8VHDXGx8ervxXufAEgmeILAMkUXwBIpvgCQLLMhisA\n/r8ZM+o9e+++G/eb9ZLTT4f7/EcTd74AkEzxBYBkii8AJFN8ASCZ4gsAyRRfAEhm1Ajgf9TPUZso\nZ2pqKsw5ePBg5/MPDAx0uq6uf19KKYcOHQqP9XMMaebM+v1i9Ln0+/z95M4XAJIpvgCQTPEFgGSK\nLwAkU3wBIJlu5yQ///nPw2O7d++uxh9//PEw50c/+lGn83/jG98Ij11xxRXV+Mc+9rFO54CjXdTV\nG3Uo79+/P1xr37591fg777wT5uzatasa3759e+fzDA0NVeMjIyPhWsPDw9X4vHnzwpw5c+ZU41Hn\ncimlDA4OVuPR+9zq0J6um0G48wWAZIovACRTfAEgmeILAMkUXwBIpvgCQLIZie3W0/Pp1n32xS9+\nsRr/4Q9/mHwl/7uzzz67Gv/LX/4S5ixatOj9upwjRfwkd96TTZs2HdbfitbGBtGxHTt2VONbt24N\n13rxxRer8SeeeCLMWbduXTX+j3/8I8yJLF++vBpftmxZmLNq1apqfM2aNWHOueeeW40vXbo0zJk7\nd241Hm0s0dpYoTXSFInO04vx8fHqxbnzBYBkii8AJFN8ASCZ4gsAyRRfAEhmY4UeRB3NpfS3q3n1\n6tXhsU9/+tPV+MaNG6vxn/70p+Fazz33XDX+61//Osz53Oc+Fx6DI1mr2znqat68eXM13upc3rBh\nQzX+wAMPhDl79+6txk8//fQw55xzzqnGow0P1q9fH64VXfOKFSvCnKgTesmSJWFO9BlEr781tRNt\nhtHKiY61uqq7cucLAMkUXwBIpvgCQDLFFwCSKb4AkEzxBYBkRo0a/v3vf1fjP/7xjzuvdf7554fH\n7rnnnmp83rx5YU40JhC16L/00kvhWo888kg13nooPBytDhw4EB6LHrg/OTlZjU9MTHQ+z6WXXhrm\nXHbZZdX4FVdcEeaMjIxU488880w13tqIIPpNbP1WzZ5dLzOt9zkaD4pGjVqitaJ4KaUMDAxU49Fr\n6YU7XwBIpvgCQDLFFwCSKb4AkEzxBYBkup0bom7f1gO5o67m+++/P8yZP39+twtruOOOO6rxdevW\ndV7rk5/85Hu8Gji8Wp27s2bNqsajSYJS4q7epUuXVuOnnXZauNbo6Gg13tok4YILLqjGly1bFuZE\nHdcvvPBCNd6ajNi9e3c1Pjg4GOZE73PUIV5KKXv27KnGo2mOVud0tBlCdF2tnH5y5wsAyRRfAEim\n+AJAMsUXAJIpvgCQTLdzw7nnnluNt555HHVKDg0N9eWa/pvoudP79+9POT8cDlFXc6vbOfpfbXXB\nRutFnbsnn3xyuFbUOT0+Ph7mDA8PV+M7duwIc55++ulqPJrAePLJJ8O1zjnnnGp8+fLlYU70OqPn\nJ5cS/15Fn1nrc460plaiZ3i3vhtdufMFgGSKLwAkU3wBIJniCwDJFF8ASKb4AkAyo0Y9WLRo0eG+\nhHLnnXdW40899VTnta688spqvPVQeDgStMZJIq2NFSLRxgbHHXdcmBM9vH/hwoWdc7Zv3x7mPPzw\nw9X4xo0bq/GRkZFwrfPOO68aX7lyZZgTjRTt27cvzJk7d241Hn2erY0dItE4USk2VgCAo5LiCwDJ\nFF8ASKb4AkAyxRcAkul2nsZaDzj/whe+UI1HHYRjY2PhWrfeems13nrwOWRrdaD20tUc5Rw6dCjM\niR6sH20e0Oqcnpqa6nRdpcQbKLz66qthzvPPP99prYsuuihc67LLLqvGW78V0Xla3c4LFiyoxqOu\n5tZ7Fl1ba7OZXj6brh3S7nwBIJniCwDJFF8ASKb4AkAyxRcAkim+AJDMqNE09uijj4bHWm36NTfe\neGN47Iwzzui0FhwOvYx5zJwZ31/0Mp4UjSFFa7VGjfbu3VuNHzhwIMzZvHlzNf7nP/85zIk2W4k2\ng2htknDSSSdV463XOTExUY1H40Qts2fXS1Z0jlLiUaPWaFD0Obe+T1258wWAZIovACRTfAEgmeIL\nAMkUXwBIptt5Grjhhhuq8V/+8ped1/rKV75SjX/ta1/rvBYc6XrZjKH1wP0oJ3oQf9SdW0rchfvG\nG2+EOb/73e+q8Va388jISDW+evXqTvFSSlm+fHk1Pjk5GeZEHcKtDunovYm6mltdyNFnE23SUErc\ncd5184QWd74AkEzxBYBkii8AJFN8ASCZ4gsAyRRfAEhm1CjJO++8Ex5bu3ZtNd5q3x8dHa3Gb7nl\nlmq81dYPR7quI0CtnJbogftRPBqZKSUeZ3n66afDnPvuu68ab/2+nH/++dV4NFJ06aWXhmtFY0tb\ntmwJc6L3oPXeRJtOHDx4sBpvjXRFn3P0mbVy+smdLwAkU3wBIJniCwDJFF8ASKb4AkAy3c5Jrrnm\nmvDYm2++2Xm9L3/5y9V41I0Ix6JWR2t0rNU5G4m6cKN4KaVs27atGv/73/8e5jzzzDPV+Pj4eJhz\n5plnVuPnnXdeNT5//vxwrZ07d1bjrcmMJUuWVOOtDSx2795djUefTatzOjpPLxto9JM7XwBIpvgC\nQDLFFwCSKb4AkEzxBYBkup377PHHH6/GH3zwwc5rfepTnwqP3XTTTZ3Xg2PNjBkzwmO9dDu3OmRr\nJiYmwmPPPvtsNR79hpQSv56zzz47zFm1alU1HnUIR13YpZSydevWanzevHlhzsyZ9Xu8qKO5lPgz\nGBoaqsZb3c5RJ3arozl6Fn4rp/Vdq3HnCwDJFF8ASKb4AkAyxRcAkim+AJBM8QWAZEaNerB3797w\n2M0331yNdx1RKCV+8HkpcSs8HIuiMY9eRkNmzZoV5kTHovNs2LAhXOuBBx6oxl988cUwZ+nSpdX4\nVVddFeZEGytE40GtTRKinIULF4Y50W/f3Llzw5xopChy4MCB8Ni+ffuq8damG9GoU+u70ZU7XwBI\npvgCQDLFFwCSKb4AkEzxBYBkup17cNttt4XH/vSnP3Ve74YbbqjGbZ4A75/oYfyth/RH3a5vvvlm\nNf7CCy+Eaz3//PPVeKvT9+qrr67G16xZE+aMjIxU41G3b7QRQimlLFmypBpvdZXv2rWrGm9txjB/\n/vxq/J133qnGo/e/lHg6pdWh3XWThF648wWAZIovACRTfAEgmeILAMkUXwBIpvgCQDKjRj245ZZb\n+rre9773vWrc5gnwv+llNCR6eH4ULyV+SP+2bduq8V5GjVasWBHmXHzxxdX46OhomDMxMVGNHzx4\nsBpvjRpF701r1CjS2sAh2oxh06ZN1Xg0zlRKKXv27KnGh4eHwxyjRgBwFFJ8ASCZ4gsAyRRfAEim\n+AJAMt3O00D0sPBW12E/DQ4OVuPRQ+RLKWVqaqoaj7pBW6IHn996662d12qJXk+re731kH2mj6jb\nttW5HB2LNhwoJf6ubt26tRqPunNLKWXnzp3V+IEDB8KczZs3V+PPPvtsmPP2229X41FHcatzOdqM\noJUTvc7od6eUUl5//fVq/LnnnqvGFy9eHK512mmnVeMnnnhimBN9Bq3f5K4d0u58ASCZ4gsAyRRf\nAEim+AJAMsUXAJIpvgCQzKjRNNBqec9w4403VuPj4+NhzpYtW6rxH/zgB325pkyt9//zn/984pXQ\nb62xnWiMrDVOEo0hRWNs0RhhKfED/6P/rVJKuffee6vx+++/P8yJRo2i1z82NhauFY0StsazopGu\n1lji+vXrq/FoPOnyyy8P14o+m9YYWjQ21HqdrdHMGne+AJBM8QWAZIovACRTfAEgmeILAMl0O/fg\nuuuuC4/dfvvtiVfSH7fddlvKeaLuwq5dgqWU8tnPfrYav+iiizqvdfHFF3fOYXqJHuzfeuB/tDlI\nFC+llMnJyWr8hBNOqMZXr14drhV1zkabB5RSypNPPlmNt7q6u743rU0aopwVK1aEOdFmDG+99VaY\nE3UbH3/88dV49P6XUsrSpUur8dbvTnSsl9+qiDtfAEim+AJAMsUXAJIpvgCQTPEFgGQzWt2AfZZ2\nosPpZz/7WTW+f//+vp7nqaeeqsb7+Wzlr371q+GxlStXdl7vE5/4RDW+bNmyzmtNA/V2TN6zTZs2\n9e23IuqaLSV+hnMrJ/q93LFjRzW+c+fOcK0NGzZU42+88UaY8+qrr1bjrQ7tvXv3dsrZvn17uNby\n5cur8dbzoKPztJ7tPDw8XI1HvxVnnXVWuNaSJUuq8cWLF4c5c+fOrcZb3c7R92ZsbKx6wJ0vACRT\nfAEgmeILAMkUXwBIpvgCQDLFFwCSGTWC3hg1ep/0c9SoJRoNiTYAKSUeNYrGaVpjS9Gx1jhLNDbU\n2lghGneamJioxlujRqOjo9V4NJpTSvx+RqNepcSvMzpP6zObM2dO5/NHOb0YHx83agQA04HiCwDJ\nFF8ASKb4AkAyxRcAksUtYgBHsa6dy6XEHcqDg4N9W6vVubtgwYJqPNqIoJRSTj311Gr84MGD1Xhr\nw4Po9fTSId7abCa6tuj8hw4dCteK3ufESZ8qd74AkEzxBYBkii8AJFN8ASCZ4gsAyRRfAEhm1Ag4\nJvUyghKNtEQjMK1Ro2gzhFZONFLUGrWJNhCIclqvP1qrdc3Req3zdM1pbWAR6SWnn9z5AkAyxRcA\nkim+AJBM8QWAZIovACTT7QzwHkVduLNmzeqcE3UUlxJ3Fbc2Kei6Vi+d063X2bXbupScbufDzZ0v\nACRTfAEgmeILAMkUXwBIpvgCQDLFFwCSGTUCjkmtB/t31cuoy+zZ/fv57WWTgmgEqDXq1IteNnA4\nmkaKIu58ASCZ4gsAyRRfAEim+AJAMsUXAJLN6GfHHwDw37nzBYBkii8AJFN8ASCZ4gsAyRRfAEim\n+AJAMsUXAJIpvgCQTPEFgGSKLwAkU3wBIJniCwDJFF8ASKb4AkAyxRcAkim+AJBM8QWAZIovACRT\nfAEgmeILAMkUXwBIpvgCQDLFFwCS/T+v8qWI4XPZrQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85c083e2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in multiple graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "There are many ways to train one Autoencoder at a time. The first approach it to train each Autoencoder using a different graph, then we create the Stacked Autoencoder by simply initializing it with the weights and biases copied from these Autoencoders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's create a function that will train one autoencoder and return the transformed training set (ie. the output of the hidden layer) and the model parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size, learning_rate = 0.01, l2_reg = 0.0005, activation_fn=tf.nn.elu):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        n_inputs = X_train.shape[1]\n",
    "\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "        with tf.contrib.framework.arg_scope(\n",
    "                [fully_connected],\n",
    "                activation_fn=activation_fn,\n",
    "                weights_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "                weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n",
    "            hidden = fully_connected(X, n_neurons, scope=\"hidden\")\n",
    "            outputs = fully_connected(hidden, n_inputs, activation_fn=None, scope=\"outputs\")\n",
    "\n",
    "        mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = tf.add_n([mse] + reg_losses)\n",
    "\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        init.run()\n",
    "        for epoch in range(n_epochs):\n",
    "            n_batches = len(X_train) // batch_size\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                indices = rnd.permutation(len(X_train))[:batch_size]\n",
    "                X_batch = X_train[indices]\n",
    "                sess.run(training_op, feed_dict={X: X_batch})\n",
    "            mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        hidden_val = hidden.eval(feed_dict={X: X_train})\n",
    "        return hidden_val, params[\"hidden/weights:0\"], params[\"hidden/biases:0\"], params[\"outputs/weights:0\"], params[\"outputs/biases:0\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train two Autoencoders. The first one is trained on the training data, and the second is trained on the previous Autoencoder's hidden layer output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.019391\n",
      "1 Train MSE: 0.0191037\n",
      "2 Train MSE: 0.0188851\n",
      "3 Train MSE: 0.0192504\n",
      "0 Train MSE: 0.00422396\n",
      "1 Train MSE: 0.00435734\n",
      "2 Train MSE: 0.00462614\n",
      "3 %Train MSE: 0.00460534\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150)\n",
    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we can create a Stacked Autoencoder by simply reusing the weights and biases from the Autoencoders we just trained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGtFJREFUeJzt3UvMnWXVP+C7lJ4P9FzaQnmhykHAgEhEcSB+kYEmGiUM\nkIEGTSQmmkiiBhITdaIzZWLQaEDRgYmJGgeKSqxBBUROJiAGaCstBdq+pUAPlLbwjb//vZb//fQ9\n9F3tdQ1X1t772bvsd+XJ/rHuWW+99VYDgApOO9EXAACjMrQAKMPQAqAMQwuAMgwtAMowtAAow9AC\noAxDC4AyDC0Ayjh9Gl/L6g2mw6wTfQGngvHxcd9nptTKlSvD77I7LQDKMLQAKMPQAqCM6fxNC2DG\nmzWr/ynlzTffHLk3kp2mET1+yMkbp50W33dEzzHkGmYyd1oAlGFoAVCGoQVAGYYWAGUYWgCUIT0I\nzBhDknORLAmXpf8mKrreKNE3Z86ckZ8zew9DkoZHjx4duXfIZxNdw3SnD91pAVCGoQVAGYYWAGUY\nWgCUIYgBlDRkVVEUNhgS+pg9e/bI13DkyJGuloUVoufNVjNl9VF7jx07Fva+8cYbI11Xa1MXaBnC\nnRYAZRhaAJRhaAFQhqEFQBmGFgBlSA8C027IgYRDDmCcjAMUTz+9/7MYJexai1N20cqm6Dlbi683\nWsGUvVaW8oteL+uNPsfDhw+HvdH1ZsnIIWnHIdxpAVCGoQVAGYYWAGUYWgCUIYgBzGjZD/rRWqIs\ntDHkfKnoebMgRvQcCxcu7GrZeVrRe8veQxSumDt3btgbhS6y9xCtnco+8yFrnIaENoZwpwVAGYYW\nAGUYWgCUYWgBUIahBUAZ0oPAlBqyWinqPXTo0MiPj5JwreXJucjBgwe72vLly8PeKGUXJRX37dsX\nPn5Icm/+/PldLUsPRvUs+Re93rx588LeaL1TlggccvDmkFShOy0AyjC0ACjD0AKgDEMLgDJOqSDG\nAw880NVuv/32sHfDhg1dbcGCBWHvpz71qa62YsWKsDerw8kq+vE9+0E++qF///79YW8UmMjCBuPj\n4yO9VmtxKGDv3r1h7+LFi7va7t27u9r27dvDx0chk+y1oiBG9jcp+vsV1Vpr7dxzz+1qS5cuDXuj\nVVJDPseMIAYAJyVDC4AyDC0AyjC0ACjD0AKgjFlDVqxM0LS9UOaCCy7oak8//fSUvNYZZ5wR1q+6\n6qopeb2pMjY21tVuvfXWsHfjxo1TfDUjmfgpc/x/jY+Pd9/n7G9JtNYoSv61Fq87ynpfe+21rvbE\nE0+EvQcOHOhqzz//fNg7ZF3SK6+80tWipGD2WlHyLvvbEfVm6cFo7dSll14a9l5zzTVd7cILLwx7\nowMuowMns3p0wGZrcXpw1apV4XfZnRYAZRhaAJRhaAFQhqEFQBmn1BqnX/3qV13tscceC3svvvji\nrpb9yPvggw92tV//+tdh7z333NPVojUqrbW2devWsD6qaOVKa62tW7euq2VrZiJROKO11r761a+O\n/BycfLJVPFGAIApGtBaHNrIf73ft2jXS41uL//uOghytxWdcZeuhonO6hpxPFa1WyoIY0Xqn7G/S\nQw89NPI1vPvd7+5qUcCktThcEQVXst6MNU4AnJQMLQDKMLQAKMPQAqAMQwuAMk6p9OBFF100Ui3z\nzne+M6zfcMMNXe3b3/522Ltt27aulqUHt2zZMvK1RbLVM1F6MLuG6EC7bMULRKIU2aJFi8LeLPEa\nWbJkSVfLDlDctGlTV8sOl4zSg2effXbYG6UHo4MdM9Hqs1WrVoW99957b1f75z//GfZGK7WytGRU\nj95Xa3ESNEsJRinKISnBjDstAMowtAAow9ACoAxDC4AyTqkgxnTKVpsMCTEMCYkMEa2d2rNnT9j7\nnve8p6tde+21k35NnLyicEUWEorq2Y/30Tqx9evXh71RsCALYkQhkWXLloW9UeAheq3s/UbPm62M\n2rx5c1eLghGtxZ9NFrZasWJFV1u6dGnYG/1bTuOZjK01d1oAFGJoAVCGoQVAGYYWAGUYWgCUIT14\nEssO2vv4xz/e1bLE0ne/+92utmDBgoldGKeUaM1PtOKntfigwmitUmvx4ZALFy4Me6P/vrOE3MqV\nK8N6JLre6Lqy70z0+OxA1mjd0uLFi8Pet7/97V3tsssuC3tXr17d1Yb8+wxZvZUlQYckEN1pAVCG\noQVAGYYWAGUYWgCUIYhxErvrrrvC+osvvtjVsh+fzznnnMm8JE5B0Y/vWfBnzpw5XS0LBSxfvnyk\nx2evl/34H4UNst7o2oYEE15//fWutmPHjrB33759XS075yv63karnVqL1zhFZ5W1NnrwJJN9jkPO\n2XKnBUAZhhYAZRhaAJRhaAFQhqEFQBnSgyeJZ599tqvdcsstIz/+/vvvD+tnnnnmcV8TDBWtbMoS\ngVHiLDt8NXreLPUWHeIYpfxai9OD0XVlicIoyfvnP/857N26dWtXiw6sbK21TZs2dbVVq1aFvVFS\nMFsPFaX/skTgVB0O6U4LgDIMLQDKMLQAKMPQAqAMQYyTxG9+85uulp1DdP3113e18847b9KvCTLZ\naqZo3VIWYpg7d25Xi87uai0OXWTfjyiIkV3vqGeFHT58OHx8FKB64YUXwt69e/d2tQ0bNoS90cqm\naF1TVs/e75BAS/TZDFnXlHGnBUAZhhYAZRhaAJRhaAFQhiBGMdmPx7/85S+7WnT2TWutfetb3+pq\n2Q/YMFHRj+/ZD/LRFoXs7K1IFKJoLd5ocejQobD36NGjXS0Lg0Tfsegadu7cGT7+3nvv7WqbN28O\ne6PvfnZG1vr167va6tWrw97o3yL7exBdw5CzxrJ/S+dpAXBSMrQAKMPQAqAMQwuAMgwtAMqQHizm\nRz/6UVi/7777utonP/nJsNfKJqbTkHOVopRelObLZOdeHThwoKtl6cFly5Z1tejMqdbi1Fu0sik7\nr+5vf/tbVxsfHw97L7zwwq52xRVXhL3nn39+V1uwYEHYG63DytZORSubhiSPrXEC4JRiaAFQhqEF\nQBmGFgBlCGLMYI899lhX+8IXvhD2Rj8ef/Ob35z0a4Khoh/fs/OaotBGFuSIQhdDzr3K1pxFYZAs\nbBCtbHrqqae62gMPPBA+ftu2bV0tWsHUWmvXXXddV7v66qvD3ui9DfkcDx48GPZGQYzsPK358+eH\n9YlypwVAGYYWAGUYWgCUYWgBUIahBUAZ0oMzQLZO5oYbbuhqWVLnxhtv7GrWNTETROnB7DDTqDdL\nvQ05vDBKsi1cuDDsXbx4cVfLDoHcvXt3V3vkkUe6WnTYY2vxe7jmmmvC3g9/+MNdbe3atWFvdNhi\n9rdjSAoz+ncbckjnZHCnBUAZhhYAZRhaAJRhaAFQhiDGNIt+tPzIRz4S9v773//uahdddFHY+41v\nfGNiFwYTNOTcrKx3yHOMepZVJgttRPVsrdG//vWvrvb73/++q+3duzd8/GWXXdbVPvShD4W9Uegi\nO58q+hyHhCuy550zZ85Itew5stDGkHO23GkBUIahBUAZhhYAZRhaAJRhaAFQhvTgNItSRJs3bx75\n8XfffXdYX7FixfFeEkyKIQmwrHfIaqboAMa5c+eOfA1ZUnHPnj1dbfv27WHvPffc09UeffTRrhat\nhmqttSuuuKKrXXLJJWFvdLBj9Blk9SHrlrJ/nyiBePTo0ZF7h/w3knGnBUAZhhYAZRhaAJRhaAFQ\nhiDGFHnllVfC+lVXXTXyc/z0pz/tapdffvlxXxOcCENWCk30jKzsnK5IdI5Ua/EZWU8++WTY++CD\nD450DVdeeWX4+Pe///1dbfny5WFvtC4pC0FEn3l2nlYU0MjOD4ueI/v3mSrutAAow9ACoAxDC4Ay\nDC0AyjC0AChDenCK3HnnnWF9y5YtIz9HlCyajDUoMJ0mut4pW7cUpd6yhFyUsssSvtGBj9khjmef\nfXZXi643OuyxtdYuvvjirhalIluLVzNla5yi95v1Rp95lu6M6kMO7pwM7rQAKMPQAqAMQwuAMgwt\nAMoQxJgETz/9dFf7+te/Pv0XAkVkP94P+VE/Cl1kjz9w4EBXiwIXrbX23HPPdbUstLF27dqudtZZ\nZ3W1bP1adM5WtooqCkxk4YpItm4pqmfhmekOXUTcaQFQhqEFQBmGFgBlGFoAlGFoAVCG9OAkuO++\n+7raq6++OvLjL7roorC+YMGC474mmMmGrHbKEmvRqqLDhw+HvdHKp+x5o1RittZobGysq61atWqk\nWib72xGl/ObOnTtyb2ZICnOIqVo5504LgDIMLQDKMLQAKMPQAqAMQYxp9r73va+r/eEPfwh7BTFg\n2Hlap58e/0lbtGhRV5s3b17YGwU8onOzWmtt6dKlXW3I+VRz5szpalmIYshZVkOCJ0PMhPP83GkB\nUIahBUAZhhYAZRhaAJRhaAFQxqxpPNTrxJ8exqngxMebTgHj4+NT8n2O/h4NWbeUGZLoG/K8ox6g\nOOQ9ZL3R82ZpySFJw5lq5cqV4XfZnRYAZRhaAJRhaAFQhqEFQBnTGcQAgAlxpwVAGYYWAGUYWgCU\nYWgBUIahBUAZhhYAZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGYYWAGUYWgCUYWgBUIah\nBUAZhhYAZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGYYWAGUYWgCUYWgBUIahBUAZhhYA\nZZw+ja/11jS+FqeuWSf6Ak4F27Zt831mSo2NjYXfZXdaAJRhaAFQhqEFQBnT+ZsWQEmzZk3sp9K3\n3pr4T4DRNUzG81bjTguAMgwtAMowtAAow9ACoAxDC4AypAeBkqLk3JCUX9b75ptvjvRarbU2e/bs\nkZ43e3xUP3r06MjXdezYsZGvK6pl13DaafH9TFSf7gSjOy0AyjC0ACjD0AKgDEMLgDIEMSbBz372\ns6524MCBsPfhhx/uaj/4wQ9Gfq2vfe1rYf2DH/xgV/vABz4w8vNCNVHgIQortBYHFrLAQ9SbhQ2i\n3uwaIkeOHBnpOVuLr3fevHlh79KlS7taFsSIrjcLqcyEtVHutAAow9ACoAxDC4AyDC0AyjC0AChj\n1jSmQU587GSCPv/5z4f173//+9N8Jb13vOMdXe0vf/lL2HvGGWdM9eWcSBM7rY+RbNu27YR/n6Pk\n3eHDh8PegwcPdrXx8fGw96WXXupqu3btCntfe+21kXqzRGH0XVy4cGHYe9ZZZ3W1c889N+xds2ZN\nV1u2bFnYO2SVVJQqnKqVT2NjY+F32Z0WAGUYWgCUYWgBUIahBUAZ1jglotDFZAQuLr/88q523XXX\nhb1PP/10V/vxj38c9j755JNd7Re/+EXY+5nPfOa/XSKcMENCAYcOHepqWbhix44dXe2pp54Ke3fu\n3NnV9uzZE/a+/PLLXS0KIGThpyjIsWjRorB38eLFXS0Ko7TW2oIFC7rakiVLwt7oOV5//fWwd8iK\nqyHrrIacg+ZOC4AyDC0AyjC0ACjD0AKgDEMLgDJO+fTgc889F9Z/+MMfjvwcV155ZVf73e9+F/ZG\nK1rmzp0b9kZJnWeeeSbs/etf/9rVssQTnGhDkmVRSrC1+KDVbN3Sli1bulqU/Gst/j6uXbs27H3X\nu97V1aL0X5RIbK21ffv2dbXsANnovb3xxhth7+mn93/as88xut7seaN6lh6MriEjPQjAScnQAqAM\nQwuAMgwtAMo45YMYWVgh+nExCly01tof//jHrhatXBnqrrvu6moPPfTQyI//2Mc+NuFrgIkacoZS\n1Jud1zR79uyuNm/evLB35cqVXW3IuVXnnXde2Lt69equFq2S+tOf/hQ+fuvWrV3t+eefD3s3btzY\n1bJ1S/v37+9qWTAieo7seaNzybJVUtHfwOzfZwh3WgCUYWgBUIahBUAZhhYAZRhaAJRxyqcHozUs\nrcWpwmzdUnTg2mSIVkll61VgpsrSf5EoPZglzqLU2vLly8Pe+fPnd7UoJdhaa+vWretq2cGM0TXs\n3r27q+3duzd8/Jw5c7palvKLPpusN1rZlP07RAdURsnM1uK/gVk6NHuOiXKnBUAZhhYAZRhaAJRh\naAFQxikfxMhEP05OlbvvvjusP/744yM/x7XXXtvVNm3adNzXBFMpCwVE52xlP+hHIYYocNFaa+vX\nr+9qa9asCXuj7362qmj79u1d7dFHH+1q2XlaL7zwQlfLAiJjY2NdLXu/R48e7WrZGWaHDx/uakNC\nFFlvVM9CG87TAuCkZGgBUIahBUAZhhYAZRhaAJQhPTjNomTR5z73ubA3SvVEK2Zaa+3222/valG6\nCmaCLMkWpciy3qierXyKDnzMEozRQYcvvfRS2BsdAPuPf/yjq2UHO0bf5wsuuCDsvfTSS7tadths\nlB7M/h5En3mW8pvo35QhKcGMOy0AyjC0ACjD0AKgDEMLgDIEMabZ/fff39WiwEXm5ptvDuvnn3/+\ncV8TTKUh4Ypjx451tSwUED1HduZd1Pv666+HvVEQ45FHHgl7o9DFE0880dU2bNgQPv6cc87pav/z\nP/8T9karqLL3MCSIEZ0HmD1vFF7JzvSaKu60ACjD0AKgDEMLgDIMLQDKMLQAKEN6cIrcdNNNYf3n\nP//5yM/xpS99qat95StfOe5rgpkiSw9GScGsd8mSJV1tSJLtlVdeCeu7du3qan//+9/D3q1bt3a1\nKI23aNGi8PHRe1i6dGnYGyX3Xn311bA3OhwySma2Nuwzj3qz540OgcxWZ2UJ0Yg7LQDKMLQAKMPQ\nAqAMQwuAMgQxJsH+/fu72m9/+9uwN1qPsnbt2rD3tttu62rZmhqYqaIf2bMf5KMf77MzmKKzs6LH\nt9bavn37utr4+HjY++yzz3a17DytM844o6tFoYvVq1eHjz/zzDO7WrbWLQqI7Ny5M+xds2ZNV8s+\n8+jzjdZAtRb/W2ZnmI36+KHcaQFQhqEFQBmGFgBlGFoAlGFoAVCG9OAkuP7667talPTJfPGLXwzr\nK1asOO5rgpkiSoxlicBofVC2mmnIyqYotfviiy+GvXv27OlqWWp34cKFXW3lypVdLVvNtHHjxq6W\nHdb48ssvd7VshVKUdsxWSb3xxhtdLVvjFP27TUYicAh3WgCUYWgBUIahBUAZhhYAZQhiDPDwww+H\n9c2bN4/8HJ/4xCe62i233HK8lwQnlSyEEIkCANkap4MHD3a17du3h71btmzpatlao1WrVnW1aLVT\nFsSIHh+dx9VavC4uC6OsW7curEeiM7mGhF+yzzyrT5Q7LQDKMLQAKMPQAqAMQwuAMgQxEocOHepq\nt956a9gb/R/lmSuuuKKrOSOLU022ESOqZ73R9yb7LkbnaUVbMlqLz7PKAiLR9ouxsbGutmnTpvDx\n0VlUUWgks3jx4rAeberI3m8UMsmCJ9H1zp8//79d4v+R/VsO2arhTguAMgwtAMowtAAow9ACoAxD\nC4AypAcTd9xxR1e79957R378TTfdFNatbIL8HKgopZed7RQl3Hbv3h32Rmdk7dix479d4v+xdu3a\nsH7WWWd1teiMrGi1U2vxWqQs5RclAk87Lb7viNJ4WSoxer0h67SWLFkycu+Qc7oy7rQAKMPQAqAM\nQwuAMgwtAMoQxEjcdtttE3r8d77znbBuZROnmigUkAUIoh/ks94oiBGta2qttf/85z9d7aWXXgp7\no/OsFi1aFPZG52EtW7asq2XnU0VnZEVrpLJryIINzz33XFfLgifRyrrsPK41a9aMfA1D/t2HcKcF\nQBmGFgBlGFoAlGFoAVCGoQVAGdKDUyRKBbU2OemZSHQ42+zZs8PeaIVOlliKRGmj1lq7/fbbR36O\nSHa9UZJzyJoZTqwhicBIdrDjgQMHutquXbvC3igpGCXsWotXM2Xroc4888yuFr3f1157LXz81q1b\nu1qWNIyu68iRI2Hvww8/3NWiz6u1OJWYHS557rnndrXs33Kq/ta50wKgDEMLgDIMLQDKMLQAKEMQ\nY4ps2LBhWl/v5ptv7mrr168Pe1988cWu9r3vfW/Sr2myRJ/lZz/72RNwJRyPKJiQrf6JQghZbxTQ\neP7558PeqP7qq6+GvVu2bBnptVprbefOnV0tChRlrxUFoFauXBn2RmGQLFwRBTyisFb2etm6uei9\nDenN/i2HcKcFQBmGFgBlGFoAlGFoAVCGoQVAGdKDiRtvvLGr3XnnnSfgSkZzxx13TMnzRmmubN1S\n5NOf/nRYf+973zvyc1x99dUj9zLzRIcBDlkxlq01WrhwYVfLUrtRQu6cc84Je6NVUA899FDYmx0k\n+f/KEnbRuqRLLrkk7B2yDmv16tVdLXu/Gzdu7GrReqrW4vcRXddUcqcFQBmGFgBlGFoAlGFoAVDG\nrOhH0ikybS80VX7yk5+E9WzFy6gef/zxsD7R1Upf/vKXw/rb3va2kZ/jox/9aFdbs2bNcV/TNJje\nX4VPUdu2bZvQ9zkLEEQ/9GehjWgF0p49e8LeaDXTjh07Ru6N1iK11tozzzzT1RYsWNDV5s+fHz4+\nCkcsW7Ys7F2xYsXIz3v22Wd3teXLl4/8vFHIpbX43yI7e2ui52mNjY2F32V3WgCUYWgBUIahBUAZ\nhhYAZRhaAJQhPcjJRnpwGkw0PZit/okSZ9nhhdF6pyhR2Nqw1WP79+/valkqcXx8vKsdOnSoq2V/\nZ+fMmdPVsvcbpQqjpGJr8WeTfebRtWXJv+hzzD5b6UEATnmGFgBlGFoAlGFoAVCG87SAaZcFE44e\nPdrV3nzzzbB30aJFXS1bP5StO4pEgYd169aFvdEKt6h28ODB8PFRiGFIgCH7bKLPMfvMo3oW2pju\ns7Mi7rQAKMPQAqAMQwuAMgwtAMowtAAoQ3oQmDGGpNNGTe61Fq93ilYdtdbakSNHRu6NVj5F6b8s\nERil/LLXinqjNVCtxWnJ6H211tqxY8e62jSu9xvMnRYAZRhaAJRhaAFQhqEFQBmCGEBJUYAgW2sU\n9WZnb436+Kwe1bIgRhSkyN5DFI7I3sNMWLc0VdxpAVCGoQVAGYYWAGUYWgCUYWgBUIb0IDBjDEm9\nDVk1FCXyhjw+Sw9G1xvVsteK1k5lScOonj3vkGuoxp0WAGUYWgCUYWgBUIahBUAZs06WH+cAOPm5\n0wKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKgDEML\ngDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyvhfMoUae7K3RTkAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85a0577438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in a single graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Another approach is to use a single graph. To do this, we create the graph for the full Stacked Autoencoder, but then we also add operations to train each Autoencoder independently: phase 1 trains the bottom and top layer (ie. the first Autoencoder) and phase 2 trains the two middle layers (ie. the second Autoencoder)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "weights4_init = initializer([n_hidden3, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "\n",
    "with tf.name_scope(\"phase1\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase1_outputs = tf.matmul(hidden1, weights4) + biases4  # bypass hidden2 and hidden3\n",
    "    phase1_mse = tf.reduce_mean(tf.square(phase1_outputs - X))\n",
    "    phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n",
    "    phase1_loss = phase1_mse + phase1_reg_loss\n",
    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
    "\n",
    "with tf.name_scope(\"phase2\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase2_mse = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
    "    phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n",
    "    phase2_loss = phase2_mse + phase2_reg_loss\n",
    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=[weights2, biases2, weights3, biases3]) # freeze hidden1\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.00782007\n",
      "1 Train MSE: 0.00741086\n",
      "2 Train MSE: 0.007667\n",
      "3 Train MSE: 0.00779957\n",
      "Training phase #2\n",
      "0 Train MSE: 0.00225269\n",
      "1 Train MSE: 0.00255029\n",
      "2 Train MSE: 0.00247932\n",
      "3 Train MSE: 0.00269086\n",
      "Test MSE: 0.00292883\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "mses = [phase1_mse, phase2_mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                sess.run(training_ops[phase], feed_dict={X: X_batch})\n",
    "            mse_train = mses[phase].eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGutJREFUeJzt3VuMnWX1P/C30CMtnXaAUqBSDoK24oECAQSiYMAoCY0S\nLhQTDZhITDSRRA0kJuqNl9obg0YDiF6QmABRRAUVrQarUjlDQU4CtpUyMD1SWsrv+p9nLf/7ZWZ2\nuzqfz+XK2u9+Zw/TlTf7y3pmvPXWWx0AVHDI/r4BABiUoQVAGYYWAGUYWgCUYWgBUIahBUAZhhYA\nZRhaAJRhaAFQxswhvpfVGwzDjP19A9PB+Pi4v2em1MjISPi37EkLgDIMLQDKMLQAKGOY32kBTKkZ\nM+KvNKPTLLLeiYre65BD4ueDPqdsTNWJHNHncCCf/uFJC4AyDC0AyjC0ACjD0AKgDEMLgDKkB4GS\n+qT/ot7s9X3SdIceeuhAvfv27QtfH6UKs94+P++bb7450Htl+qQwh82TFgBlGFoAlGFoAVCGoQVA\nGYIYwEEjCwr0CWJMdA3T3r17B36vKDAx0YBJdg99wiTZdaPPYdirszxpAVCGoQVAGYYWAGUYWgCU\nYWgBUIb0IDB0k7EmaKJJtD5Jw2y1UlTfvXv3QNfM7mHBggVhb5+k4ezZswd6r0zWG6UH+3yOk8GT\nFgBlGFoAlGFoAVCGoQVAGYIYQElRCOKNN94Ie+fMmdPU9uzZM/B7RauOsvphhx3W1KIQRdfFIYYd\nO3YM/F5z584Ne6OfN7uH6HPIft7oM+9z/lfW24cnLQDKMLQAKMPQAqAMQwuAMgwtAMqQHgSGLlv9\nE9Wz1FuURBsfHw97o0MRs4Md582b19QWLlwY9kZmzZo10DW7rl/CLvpsst6ZM9t/2qP7ynqzFGaf\nNU6TkRQM72FKrgoAU8DQAqAMQwuAMgwtAMqYVkGMv/71r01tzZo1Ye9xxx3X1LIvUz/72c82tdHR\n0bA3q0N1fc6MioIRXRevMIrOp+q6rtu0aVNTe/rpp8PesbGxprZz586w95hjjhmo1nVdd/jhhze1\n6DysaLVTVt+1a1fYG31mWW8UpMjCJIsXL25q0XlcXTfxdVhZ+KXP2VuetAAow9ACoAxDC4AyDC0A\nyjC0AChjRraCYwoM7Y0y73rXu5raU089NSXvNTIyEtbPOeecKXm/qXLCCSc0teuuuy7sPf7446f4\nbgYyeAyJt218fHzgv+fo35gsuff66683tZdffjnsfeKJJ5rao48+Gva++OKLTS076DBaa5T1vvLK\nK00tSvllP2+UpnvttdfC3pNOOqmpZYdARvUsPXjeeec1tbPOOivsjdKS2ZqtPqLPYWRkJPxb9qQF\nQBmGFgBlGFoAlGFoAVDGtFrjdPvttze1Bx54IOx9z3ve09SyL3nXrVvX1O64446w9ze/+U1TO/HE\nE8PeZ599NqwPKvpCuevilTQvvPDCwNeNwhld13Vf//rXB74G00f0JXu2zieqR6uDuq7rjjrqqKa2\nfPnysDcKUmQBgujcqWeeeSbs3b59e1OL1jhlXnrppaaWrVCKVlxl5149+eSTTS0Lh0Wf2bnnnhv2\nRp9NFlKJPt/JCP550gKgDEMLgDIMLQDKMLQAKMPQAqCMabXGaZiidTRd13XPPfdcU8vSg1liaVBZ\nCilKD2b3EK3Que2228Le1atX97i7KWON0xBEa5yyg/yi+r59+8LeaAVSdghktEIpSth1XZyymz9/\nftgbHayYJXmXLl3a1KKEXXbo5eOPP97UjjzyyLA3Opj2zjvvDHvvuuuuphatseu6rrvsssua2iWX\nXBL2Llq0qKllMyQ6HDL7vUescQKgPEMLgDIMLQDKMLQAKGNarXEapuycm3e/+90DX2PFihWTdTv/\nj2jt1JYtW8Les88+u6llX9LCoLI1TtFKoGwtUhSkyFYz9TkjKwpNnHbaaQNfN1o7NT4+Hr7+9NNP\nb2pZQCS6r9///vdhb7ReasmSJWHvKaec0tSyEFcUpMjCFVE9C+v0CQR60gKgDEMLgDIMLQDKMLQA\nKMPQAqAM6cGDWLbS5hOf+ERTyxJA3/ve95ravHnzJnZjTCtRMqxPerDPeqjswMjoutmqtei62YGq\n0d9NtDIqWu2U3Vf2Xk8//XRTe/jhh8Pe6DM//vjjw96jjz66qWX3GyUY+/x+HAIJwLRiaAFQhqEF\nQBmGFgBlCGIcxG666aawvmnTpqZ2xBFHhL3Lly+fzFviINbnS/Zs3VIU0MjOoopCDNEZTllvdg9R\ngGCi181eHwUespVqN998c1Nbv3592Hv++ec3tYsuuijsPeqoo5patuKqz++4z9lZfXjSAqAMQwuA\nMgwtAMowtAAow9ACoAzpwYNEtOLl2muvHfj19913X1hfunTp274n6CtKp2Upv6ierR/KVjYNKjsU\nMXq/KCmY3VckSwT+4he/GPi6F198cVM78cQTw94+icA+q5miJGj2u+zDkxYAZRhaAJRhaAFQhqEF\nQBmCGAeJ6EvabHXMFVdc0dROOumkSb8n6Cv6oj8LQUQBgGzlU58gRBQgyFYS7d69u6lF95udOTU2\nNtbU7r///rA3OqfrggsuCHujNU7ZWWPRzzsZK7mscQJg2jO0ACjD0AKgDEMLgDIEMYrJwhW33XZb\nU8u+eP3Od77T1LLzc2CYssDCoL19zoHq8998tlFj0NBFFlZ4/PHHm9q6devC3hNOOKGpXXbZZWFv\ntMkmClx0XRwmyT6bPqGN6BpZOKNP8MOTFgBlGFoAlGFoAVCGoQVAGYYWAGVIDxbz4x//OKyvXbu2\nqX36058Oe61sYn/L0mLRGqZsjVOUyMtSen1WPkUJuSxNN3fu3IFqL7zwQvj622+/vak99thjYW+U\nFFy1atXA97V9+/awN/ocshRnVM9SiZNxdlbEkxYAZRhaAJRhaAFQhqEFQBmCGAewBx54oKl96Utf\nCnsXLVrU1L797W9P+j3BZMi+6I8CE1loo88apygUkJ2xFa0amjlz8H8qo8DDvffeG/b+8Y9/bGqj\no6Nh70c+8pGmtmTJkrB3165dTS37HKOgSxaiiD6HPiuY+vwuM560ACjD0AKgDEMLgDIMLQDKMLQA\nKEN68AAQJX26rus+9alPNbUs1XPllVc2NeuaOFBlhwH2OUAx6s1SftGqoSw9uHPnzoFe33VxGm79\n+vVN7c477wxfv23btqb2mc98Juw944wzmlqWloz+TckOspw/f/7A181WX0Wiz6ZPSjDjSQuAMgwt\nAMowtAAow9ACoAxBjCGLvoC+9NJLw94NGzY0tRUrVoS93/rWtyZ2YzBF+pzBFP199Dn3Kjt76403\n3mhqUeCi6+KARhZM2Lx5c1OLzsj61a9+Fb7+oosuamrRuVld13WLFy9ualmgJfrZojO2MpPx+4mC\nGNnn2IcnLQDKMLQAKMPQAqAMQwuAMgwtAMqQHhyysbGxppYdEBe55ZZbwnp2cBwMS5+DHfteIxIl\n2focMjhnzpwJ38ODDz7Y1KK/51NPPTV8/dVXX93UTjzxxLA3SjVmq5kWLFjQ1LLPJluTFemzmmky\nVjZFPGkBUIahBUAZhhYAZRhaAJQhiDFFxsfHw/o555wz8DV++tOfNrXTTz/9bd8TTKXsi/5oJVCf\nM7L6nGWVrRSK3i+77o4dO5raa6+9Fvb+9re/bWobN25sapdffnn4+g984ANNLQpRdF0cusjWOEWh\njeyzia4x0XBG1+Wf70R50gKgDEMLgDIMLQDKMLQAKMPQAqAM6cEpcuONN4b1Z555ZuBrnH/++U1t\nqlajwFTps/qnj+ga2XWjlU1Z8m737t1NLTvE8dZbb21qM2e2/6xGf8td13VHHHFEU8vSeFE9641+\ntizNFyUF+6zDmqqUYMaTFgBlGFoAlGFoAVCGoQVAGYIYk+Cpp55qat/85jeHfyNwAJqq87SitUTZ\n+qFDDz20qe3cuTPs/dvf/tbU1q5dG/ZG5+N97GMfa2pHHnlk+PpoNdO8efPC3uiziUIfXddvDdOg\n7/W/6sPkSQuAMgwtAMowtAAow9ACoAxDC4AypAcnQZQs2rp168CvX7FiRVjPUkRwsOqzqihKBGYr\nhaLUW5YejA6BzP4WL7744qb28Y9/vKkde+yx4eujny1aI9V1Xbdnz56mlqUEZ8+e3dT6rFvqc/Dm\nsHnSAqAMQwuAMgwtAMowtAAoQxBjyD74wQ82tbvvvjvsFcSAXJ/ztKIQQxYqOOaYY5raBRdcEPYu\nW7asqa1cubKpzZ8/P3x9FCbJRL2zZs0Keyd6htmBELjIeNICoAxDC4AyDC0AyjC0ACjD0AKgjBlD\nTIkcuHEUDib7/5S6aWB8fLzU33OUnNu3b1/YGx3MmKX8Bl0llSX3+qxW6rPiKnq/AzkRGBkZGQk/\nNE9aAJRhaAFQhqEFQBmGFgBlDDOIAQAT4kkLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9AC\noAxDC4AyDC0AyjC0ACjD0AKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4Ay\nDC0AyjC0ACjD0AKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMqYOcT3emuI78X0NWN/38B0MDY2\n5u+ZKTU6Ohr+LXvSAqAMQwuAMgwtAMoY5ndaAJNmxozBv7586632K7g+r5+o6P0nQ/Yz7Nu3b+De\najxpAVCGoQVAGYYWAGUYWgCUYWgBUIb0IHDQi9J7WaIvStllybs+vZE333yzqe3duzfsnTNnzkCv\n77qu2717d1M79NBDw95DDmmfXbLeQV/fdf0+8z48aQFQhqEFQBmGFgBlGFoAlCGIMQl+9rOfNbUd\nO3aEvffff39T++EPfzjwe33jG98I6xdddFFT+/CHPzzwdeFgkIUYomBCFpjYtWtXU8sCD+Pj401t\n69atTa1PAOGwww4L64sXL25qc+fODXuzcERk1qxZTW327NkDX3eqVlRlPGkBUIahBUAZhhYAZRha\nAJRhaAFQxowhJj+GGzGZAl/84hfD+g9+8IMh30lr5cqVTe3Pf/5z2DsyMjLVt7M/HRwn3R3gxsbG\nBv57nujhg9GBhl0XJwKjNF/Xdd3GjRub2iOPPBL2Pvzww01tw4YNYe+//vWvphalD5cvXx6+/qij\njmpqq1atCnvPPffcpnb66aeHvYsWLWpq2e8hSlxm6cPoGn1mSNYb1UdHR8Mb9qQFQBmGFgBlGFoA\nlGFoAVCGNU6JKHQxGYGL6IvTyy+/POx96qmnmtrNN98c9j722GNN7ec//3nYe/XVV/+vW4RJFX3J\nnn3RH4UuosBF18Xrkp5++umwd/369U1t7dq1YW8U0MjWQ73vfe9raqeeempTGxsbC1//3HPPNbUo\n3NF18RqnKIDVdV03b968ppaFIKKfLTtPK1pPl/VG66GyUE2fsI4nLQDKMLQAKMPQAqAMQwuAMgwt\nAMqY9unBf//732H9Rz/60cDXOOuss5rar3/967A3OuAtO3AtOnguSxb95S9/aWpbtmwJe2GYomRY\ndqhilHDrs34oSxpGvfPnzw97L7300qZ24YUXhr3nnXdeU4vu99FHHw1ff8sttzS1nTt3hr3RdV9/\n/fWw99VXX21q2c8bfWbZIbbR7y1LBEarpLLUaJ9VUJ60ACjD0AKgDEMLgDIMLQDKmPZBjCysEH0x\nGAUuuq7r7rnnnqa2YMGCid1Y13U33XRTU/v73/8+8OtXr1494XuAieoTrohCAZOxfmjZsmVNbXR0\nNOyNzq065ZRTwt6FCxc2tSjEsGnTpvD1Dz30UFNbsmRJ2ButcYrev+vizyE656vr+v1+IlkQY8+e\nPU0tWu3U9/08aQFQhqEFQBmGFgBlGFoAlGFoAVDGtE8Prlq1KqxHqcJs3VJ04NpkiFZJvfHGG1Py\nXjBMWSIwWvOT/d294x3vaGqHH3542ButO8quG60fmjNnTtgbJeeiFUp/+MMfwtdv3rx5oPfvuq57\n73vf29Sy1UzRz7Z9+/awN/rM+6T5MtFnk13XGicADkqGFgBlGFoAlGFoAVDGtA9iZEZGRob2XtGZ\nOl3XdQ8++ODA17jkkkua2sknn/y27wmmUvaFfBR4yAITUW8WiooCTNEaqK6LQwHZeqhoNdK6deua\n2vPPPx++PlrZtHLlyrA3WjuVrUWKQhfZWVbRz5addxZdI/vM+5y9ZY0TAAclQwuAMgwtAMowtAAo\nw9ACoAzpwSH75z//2dS+8IUvhL3RgXjHHHNM2LtmzZqmliWLYJiiZFiWFovSaTNnxv9MRem/LPU2\n6Ou7Lj7Addu2bWHvhg0bmtp999038H1Fh1N+9KMfDXujtVXZWrdXXnmlqR1xxBFhb5QezBKB0e8n\nSwT2+V304UkLgDIMLQDKMLQAKMPQAqAMQYwhi76kjQIXmWuuuSasn3rqqW/7nmAyZOGKiZ7NlJ21\nFIUQ9uzZE/ZG9Shw0XVxgODFF18Me3/5y182tfXr1ze1xYsXh69fvXp1UzvttNPC3igEEZ0T1nX5\nmVyR6HPMPvMo3JWtuIoCGllvH560ACjD0AKgDEMLgDIMLQDKMLQAKEN6cIpcddVVYf3WW28d+Bpf\n+cpXmtrXvva1t31PsD9ESbTsQMKoN1sT1Kc3Sr1lvVu3bm1q9957b9h7xx13NLW5c+c2tQsvvDB8\n/dKlS5talvz773//29Syz3H+/PlNLVurFNWzxGeUwswSgdFnnqUSs3rEkxYAZRhaAJRhaAFQhqEF\nQBmCGJNg+/btTe2uu+4Ke6O1K0cffXTYe/311ze12bNn97w7mHzRF/V9vkzPeqPr7tixI+zts1Io\nOpNrbGws7H3mmWea2j/+8Y+wd+HChU3t5JNPbmorV64MX3/mmWc2tewcvOhziAIXXTfxdUnZ66PQ\nRp/ztCa60qvrPGkBUIihBUAZhhYAZRhaAJRhaAFQhvTgJLjiiiuaWrRyJfPlL385rI+Ojr7te4Kp\n1Ccp2EefA1GjFUZZ8i466HDz5s1h75/+9KemtmHDhrD3uOOOa2oXXHBBU3v/+98fvv6www5raq++\n+mrYG33mfX4P27ZtC+vR2qlsPVSUCMwSzdG9SQ8CMK0YWgCUYWgBUIahBUAZghg93H///WE9O2sn\n8slPfrKpXXvttW/3lmC/6LPGKftSf9DrZl/0R9eN1jV1XXwOVPb3HAUxnnvuubB3xYoVTe2ss85q\natG6p+y+MhNdcTVv3rywt8+6pWhlU7SaLtPnv4X0GhO+AgAMiaEFQBmGFgBlGFoAlCGIkdi1a1dT\nu+6668Le6P+2z5xxxhlNzRlZVDPRs7Oy85qiL+qz3r179za1LNjw5JNPNrVHHnkk7H3iiSeaWnbm\n3Yc+9KGmNjIy0tSyTR2ROXPmhPXo36Roo0bWu2jRorA3CmJk4YqoN/vZJhrAyXjSAqAMQwuAMgwt\nAMowtAAow9ACoAzpwcQNN9zQ1H73u98N/PqrrroqrFvZxHQTpQejdUBdFycF+6QHN23aFPbefffd\nTe2ee+4Je6OE3Nlnnx32nnnmmU0tSullSboo7ZglAqNzr3bu3Bn2LliwoKlln3mUNIxqXRcnpbP0\nYPR7m4xz2DxpAVCGoQVAGYYWAGUYWgCUIYiRuP766yf0+u9+97th3compps+K3qiL/WzM7K2bt3a\n1Hbv3h32bt68uallZ1FFK5uiM7K6ruuWLVvW1KJzq7JgQ1aPRGGOLLQRBR5effXVsHdsbKypZQGP\n6Lp9/k2bjH//PGkBUIahBUAZhhYAZRhaAJRhaAFQhvTgFNm+fXtY73MwWh/RwXHZ+ptoTU2Wuopk\niac1a9YMfI1Idr9RkrPPoXrsX9H6oCxRGK01ytKDURItShR2Xddt2bKlqb388sthb7QuKUu9vfTS\nS02tz99XdA/ZobLReqhoXVPXxf/+PPbYY2FvlBTM1i1FycqFCxeGvVGKMruuQyABOCgZWgCUYWgB\nUIahBUAZghhT5Ljjjhvq+11zzTVN7dhjjw17ozOHvv/970/6PU2W6LP8/Oc/vx/uhLcj+pI9CyT1\nOXsrum4W0InWHUWhj67ruo0bNza12267Lexdu3ZtU4tCH4sXLw5fH91vFOToujh0MX/+/LA3+hvf\nsGFD2BsFXVatWhX2LlmypKm9/vrrYW8U2LLGCYBpxdACoAxDC4AyDC0AyjC0AChDejBx5ZVXNrUb\nb7xxP9zJYG644YYpuW6ULMrWLUU+97nPhfVzzz134Gucd955A/ey/2SreKJEYNYbpQr37t078Ptl\na42iFUgrVqwIe//zn/80tWeffTbsfeihh5raa6+91tRGR0fD1/f5GZYuXdrURkZGwt7oM4tWvWXX\nOP7448Ped77znU0tu9/o/fqsa8p40gKgDEMLgDIMLQDKMLQAKGNGdr7JFBjaG02Vn/zkJ2E9O/9m\nUA8++GBYn+hqpa9+9athPfoyNXPZZZc1tWiVywFk4t/08v81NjY28N9znzPk+oR8on+7stDGo48+\n2tQeeeSRsDdaw5Sd6RWtS3r++eebWnbOV7ReKlv5FP3dHXnkkWFvFK7IPpvo/LDTTjst7I3OzsrC\nINmKqUGNjo6Gf8uetAAow9ACoAxDC4AyDC0AyjC0AChDepCDjfTgEPRJD/ZZ3ROlB7NEYXRYYraq\nKDqQMPu3L0oKZgnInTt3NrVojdPu3bvD12/btq2pZYdTRunB7FDFaG1Vdt2onn3m0e8yu4c+v/fo\ndyE9CEB5hhYAZRhaAJRhaAFQhvO0gCnV5zytffv2Ddwb1bOwQbTCKFtrFN1DtG6p6+JzsrK1RpEo\nxJCthYt+3iwgEoVUsutGv5/o9Zks0DJVIT9PWgCUYWgBUIahBUAZhhYAZRhaAJQhPQgMXZ/EWZTm\n67o4OZclDaM0XJ/0YHa/0cqmPquOop+hz2GNmeh++3zmfdKdmegak5Eo9KQFQBmGFgBlGFoAlGFo\nAVCGIAZwQOtzLlPWGwUhshVIfUIM0dlbfVYgRauVsrBD1Bu9//+6xqAm+vpM9vvpE9DwpAVAGYYW\nAGUYWgCUYWgBUIahBUAZ0oPAAa1PsqxP6i1L3vVZrRSl4WbNmjXwPUSpxj4/Q5+kYmaqDmucqvfy\npAVAGYYWAGUYWgCUYWgBUMaMYX4JBwAT4UkLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9AC\noAxDC4AyDC0AyjC0ACjD0AKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4Ay\nDC0AyjC0ACjj/wC99DxGjRI99AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85a013af60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_one_at_a_time.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Cache the frozen layer outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.00724552\n",
      "1 Train MSE: 0.0078875\n",
      "2 Train MSE: 0.0077243\n",
      "3 Train MSE: 0.00779147\n",
      "Training phase #2\n",
      "0 Train MSE: 0.00235939\n",
      "1 Train MSE: 0.00250115\n",
      "2 Train MSE: 0.00260497\n",
      "3 Train MSE: 0.00271682\n",
      "Test MSE: 0.00294311\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op, training_op]\n",
    "mses = [phase1_mse, phase2_mse, mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        if phase == 1:\n",
    "            mnist_hidden1 = hidden1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if phase == 1:\n",
    "                    indices = rnd.permutation(len(mnist_hidden1))\n",
    "                    hidden1_batch = mnist_hidden1[indices[:batch_sizes[phase]]]\n",
    "                    feed_dict = {hidden1: hidden1_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "            mse_train = mses[phase].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGxRJREFUeJzt3VnMnWX1N+C70JGOKW0pRfgXFEqBoGUogx4YIxhHHIJG\nONCgicREE0nUQGKinnionBg0GlD0wMSIxigoCI1AKkOFSgSkUCg4FDrQlpYOTP+j7+D77rX89tN3\noKu9rsOVtff77P12v4sn+8e6p7zxxhsNACo46s2+AAAYlaEFQBmGFgBlGFoAlGFoAVCGoQVAGYYW\nAGUYWgCUYWgBUMbUSfxZVm8wGaa82RdwJNi+fbvPc2LKlP6foM1Dwy1cuDD8LLvTAqAMQwuAMgwt\nAMqYzO+0AA55Y/1O6vXXXx/pOcfDkOc9XL5Xc6cFQBmGFgBlGFoAlGFoAVCGoQVAGdKDwBHpqKPi\n/2aP0n9RrbU4kffaa691talT4z+10TVkKb+jjz66q2Xpweg5hqQHs96JSkEO4U4LgDIMLQDKMLQA\nKMPQAqAMQQxg0g0JEGSyIEUkCkcMuYZXX3017B01mJC9riHXNUQUHMleQ/Q+HgqBi4w7LQDKMLQA\nKMPQAqAMQwuAMgwtAMqQHgQOaVnybsi6pWiN0pDe6dOnj3wNQ0TpwcyBAwe6Wpby27dvX1fLVklF\nzxGtjGpt2Nqpsa6SyrjTAqAMQwuAMgwtAMowtAAoQxADOGQMWR801i/6h6yBykSBheh5sxVKL730\nUlfLXsPMmTO72uzZs8PeGTNmjHwN0WvIghjR72fIiqoh55Jl3GkBUIahBUAZhhYAZRhaAJRhaAFQ\nhvQgcEjLUm/RqqK9e/eGvVGSbdq0aWFvtLIpSu5l9ejx2c+aNWtWV8tWO42aVMx6X3755bA3Mh6r\ns4Y87xDutAAow9ACoAxDC4AyDC0Ayjiighh/+ctfutr1118f9p5wwgldLfrStLXWPvOZz3S1hQsX\nhr1ZHaobsoIpC1dEIYQdO3aEvY888khX27p168g/75VXXgl7jzvuuK42b968sDeqz58/v6vNmTMn\nfHwU2ti9e3fYG618yq4rCmhkrze6tuzvVPT7yX7vYz1rLONOC4AyDC0AyjC0ACjD0AKgDEMLgDKm\njMdajRFN2g/KrFixoqtt2LBhQn5WlCBqrbULL7xwQn7eRFm+fHlXu/baa8Pek046aYKvZiSjR9g4\naNu3bx/T5zlbtxStGtq0aVPY+9hjj430+Nbi5N22bdv+2yX+fx/fWpyci1J6WXowSjXu378/7J06\ntQ97z507N+yNriFLBL73ve/tameddVbYG62tGnK45BALFy4MP8vutAAow9ACoAxDC4AyDC0Ayjii\n1jj9+te/7moPP/xw2HvmmWd2tb///e9h73333dfVfvOb34S9f/jDH7raySefHPY+/fTTYX1U0Re3\nrbV2/PHHd7Xnnntu5OeNwhmttfb1r3995Ofg8DNkjVPWG60fys6yilYYZedLRfVsLduuXbu6WvZZ\ninqjYEJ09ldrcRgke73R5zZ7H6OQSvZ3ZufOnV0tC1dEwb3svYl6h/wbybjTAqAMQwuAMgwtAMow\ntAAow9ACoIwjao3TZMrSQs8880xXy1I9GzduHNM1RAfMtRankLJr2LJlS1e75ZZbwt7LLrtswNVN\nGGucJkG0xilLho266qi1eA1TtkIpOmRwSOotW/kUHXSYrVaKXke0Wik7yDK6hugQytbi1XB33HFH\n2Hv33Xd3tSVLloS9V1xxRVc799xzw97ob0r23oyVNU4AlGdoAVCGoQVAGYYWAGUcUWucJlO2iuX0\n008f+TlWrlw5Xpfzf4nWTm3dujXsveCCC7rapZdeOu7XRC1RuGJIqCtb/ROFGGbMmBH2ZkGjiRCF\nPrJ69N4cOHAgfHy0SipbLxUFHv74xz+GvXv27Olq2dlbixcv7mrZWVhR0CXrzd6zyJB/O+60ACjD\n0AKgDEMLgDIMLQDKMLQAKEN68DAWJYhaa+1jH/tYV8uSPt/73ve6WpZu4sg2ZI3TkGRZlhKM6kNS\nfplojVOWdhzroYbR82arqJ544omu9tRTT4W90Rq5U089NexdtGhRV8te16hpydaGJQKHcKcFQBmG\nFgBlGFoAlGFoAVCGIMZh7Kabbgrrmzdv7mrHHnts2Ps///M/43lJHCYm8Ry+dE1QJFuXFAUesnOg\nop931FHxf99HvdHzZkGQ6H3ctm1b2Hv77bd3tWglW2utnXfeeV1t9erVYe/s2bNHuq6snr22sa76\nyrjTAqAMQwuAMgwtAMowtAAow9ACoAzpwcNEtM7lmmuuGfnxa9euDetLly496GuCoYasCXrllVe6\nWra6bNq0aSNfQ5Q0jH5Wa/HKpegasjVQkU2bNoX1hx56qKvNmzcv7L3kkku62sknnxz2RgnIIenB\nTNSbPX7IOix3WgCUYWgBUIahBUAZhhYAZQhiHCZ++9vfdrXsy+PLL7+8q51yyinjfk2QGfJFf7aa\nKTr3akjgIlvNFMmCAtG5VdE1zJ8/P3z87t27u9qaNWvC3i1btnS1d7zjHWHvOeec09Wyc/CikEj2\nnkey9zH6/Yz1/LHW3GkBUIihBUAZhhYAZRhaAJQhiFFMFq645ZZbutqMGTPC3u985ztdbciZRTBW\nQ76Qj77Qby0OAGRhg+g5sjBI9BxDNkFMnz69q2VhhWj7xcaNG8Pe6NyrD33oQ2Hvcccd19Wyz3gU\nuog2fbQWv4/ZaxuP0EXEnRYAZRhaAJRhaAFQhqEFQBmGFgBlSA8W8+Mf/zis33333V3tiiuuCHut\nbGIiZGmxIcm76Dmyx0frkrKEXJR6yxJy0Zlee/fuDXtHva7NmzeHvb/73e+6WpYejM7IOv3008Pe\nKME45Eyw7H0csvoq6o3e26HcaQFQhqEFQBmGFgBlGFoAlCGIcQh7+OGHu9qXvvSlsHfBggVd7dvf\n/va4XxMMNeQL+agehQpai8+BysIGe/bs6WpZ2CAKJmQBhCh0sX///q62du3a8PH33HNPV8tWUa1e\nvbqrZed0RbJ1WENE7++Q0IbztAA4ohhaAJRhaAFQhqEFQBmGFgBlSA8eArIVMZ/+9Ke7WpYAuvLK\nK7uadU1Mpmzd0pA1TlHy7phjjgl7oyRalvIbcmBklFYcsqIqSv3edttt4eOjQyA/+clPhr0rVqzo\nalmyct++fV0tS2wOOQB2SG/03mTv45D1Tu60ACjD0AKgDEMLgDIMLQDKEMSYZNEXjh/84AfD3n/8\n4x9dbeXKlWHvt771rbFdGBwCosBEdu5VFELIvtCPwhxZQCRaVRStjGqttX/+859d7Ve/+lVXi9Y1\ntdbaeeed19U+/OEPh72LFy/uagcOHAh7o0BLtHKqtTjclQUuonoWroh+b9Y4AXBEMbQAKMPQAqAM\nQwuAMgwtAMqQHpxk27dv72pr1qwZ+fE333xzWF+4cOHBXhJMumwdWZT+yxKB0fqzmTNnjvkaooRb\n1hutbFq/fn1XO/HEE8PHf/SjH+1qp59+etibraiKDElLDjmsccjqrOjnDVnXlHGnBUAZhhYAZRha\nAJRhaAFQhiDGBNm5c2dYv/DCC0d+jp/97GddbdWqVQd9TXCoyL68j1YNZV/eR88RnSOVPe+Qs6Ge\neuqpsH7//fd3tW3btnW1Sy+9NHz8xRdf3NVmzJgR9kZrkYYEG7JVVNHzZiufoiBGdg1REGPI2WoZ\nd1oAlGFoAVCGoQVAGYYWAGUYWgCUIT04QW688cawvnHjxpGf413veldXG49D1ODNlv07HrJSaEg6\nLToUcdasWWFvlP7LDnF88MEHu9qCBQu62tlnnx0+/thjj+1qWaoxSkZmvdF7lh2mGb1n0UGYmSG/\nn/HgTguAMgwtAMowtAAow9ACoAxBjHGwYcOGrvbNb35z8i8EiouCBdkX/VHvkDOyduzYEfbed999\nXe3WW28Ne//85z93tUsuuaSrLV68OHx8dL3Za4hWK2W9UfBkPAITQ1ZfRcbjGtxpAVCGoQVAGYYW\nAGUYWgCUYWgBUIb04Di4++67u9quXbtGfvzKlSvDerZmBg5X0RqnIauKsjTd3r17u9qBAwfC3ijJ\ndsIJJ4S973//+7vaBz7wga62fPnykX9W9hqiwxaz1UzRe5MdvDnkfRwrh0ACcEQxtAAow9ACoAxD\nC4AyBDEm2cUXX9zVbr/99rBXEIMjzZBgwpCwQSRai9RaayeddFJXi862y37emWee2dXmzp0bPj4K\nUkxUECMLQUSvYcj7OFHnZmXcaQFQhqEFQBmGFgBlGFoAlGFoAVDGlElMfkxuxIQjVXzKHONq+/bt\nb/rneUhCbqyyRF/086KUX3b44dSpfYA76x2ybil7jkiUFJzsRGBk4cKF4YtwpwVAGYYWAGUYWgCU\nYWgBUMZkBjEAYEzcaQFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGYYWAGUYWgCUYWgBUIahBUAZhhYA\nZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGYYWAGUYWgCUYWgBUIahBUAZhhYAZRhaAJRh\naAFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGVMn8We9MYk/iyPXlDf7Ao4EW7du9XlOTJnS/xN84w1v\n11CLFi0KP8vutAAow9ACoAxDC4AyJvM7LYBD3lFH9f8tn30nFdWjWvQ913973shYvyvLerNrO1S5\n0wKgDEMLgDIMLQDKMLQAKMPQAqAM6UGgpCjl9/rrr4e9UXIuenxrcZou6923b99I1zBt2rTw8ZHX\nXntt5HqW/Dv66KNHvobovcnex0MhaehOC4AyDC0AyjC0ACjD0AKgDEGMcfDzn/+8q+3ZsyfsXbdu\nXVf74Q9/OPLP+sY3vhHW3/Oe93S1d7/73SM/Lxyqsi//X3nlla6WhRii3gMHDoS9u3fv7movvPBC\n2Ltjx46utnfv3q62ZMmS8PGzZs3qajNmzAh7oyBFFq6YN2/eSD+rtdamT58e1iOHwrEr7rQAKMPQ\nAqAMQwuAMgwtAMowtAAoY8okJj8mN2IyAb74xS+G9R/84AeTfCW9M844o6vdc889Ye/8+fMn+nLe\nTG/+npkjwNatWyfk8zwknRal/1566aWwd9u2bV3t8ccfD3sfeuihrrZ27dqw98UXXwzr/68sobds\n2bKudvHFF4e9q1ev7mrR57611hYsWNDVXn311bA3Sitmic2onj1vtApqyGGYixYtCpvdaQFQhqEF\nQBmGFgBlGFoAlGGNUyIKXYxH4GLVqlVd7ROf+ETYu2HDhq72k5/8JOx99NFHu9ovf/nLsPdzn/vc\nf7tEGFdDwhVDgmHRWVbZuqUHHnigq0WBi9Zae/jhh7tatJqptdbOOuusrrZ48eKuFq2Raq219evX\nd7Unn3xy5J81e/bssDc6/2vq1PjPfXZWWCT6/WTrsKIzvbJzuoZcgzstAMowtAAow9ACoAxDC4Ay\nDC0Ayjji04PPPvtsWP/Rj3408nOcf/75Xe22224Le4855piulq14iQ60y5JF9957b1fbunVr2AuT\naUgiMEqRZcm96N/3U089FfZu3LhxpJ/VWrwu6aKLLgp73/e+93W16ADGNWvWhI/fv39/V3v55ZdH\n7t2+fXvYu3Dhwq6W/Z2JUn7ZYZrZ4bZjla13irjTAqAMQwuAMgwtAMowtAAo44gPYmRhhejL4yhw\n0Vprd9xxR1ebM2fO2C6stXbTTTd1tWgdTeayyy4b8zXAqIaclTTki/esN1rjFJ0j1VprZ555Zleb\nNm1a2Hvaaad1tWiFUmvxGqXNmzd3tU2bNoWP/9vf/tbVojVQ2XNk1xWdcRWFwFqLVytF721r8e8i\nOo+rtfj3Ph7nN7rTAqAMQwuAMgwtAMowtAAow9ACoIwjPj14zjnnhPUoVZitQZk1a9a4XtP/Ea2S\nyg5cg0qyAwmjwxKjNUOttbZ06dKuFq0vaq21t7zlLV0t+9wuWbKkq82cOTPsjT6PUXpw3bp14eOf\ne+65rrZixYqwd/ny5V1t2bJlYW/0ng1JBGaHVkZ/A7PfT/TeDDnsMeNOC4AyDC0AyjC0ACjD0AKg\njCM+iJGZP3/+pP2sm2++OayvX79+5Oe49NJLu9pb3/rWg74mGC9R6CJbzTTki/7oS/0sbBB9nrNg\nVbTuKAsm7Nq1q6tFoYunn3565Os69dRTw94zzjgjrI9q586dYX3u3LldLfv9RGd9DVnJFa2MGvoc\n7rQAKMPQAqAMQwuAMgwtAMowtAAoQ3pwkj300ENd7Qtf+ELYu3///q52/PHHh73XX399V8sOuYPJ\nFCXDskRgJOuNDlrNeqNDEV977bWwNzqoMErNtdbaX//616521113dbUnn3wyfPy5557b1S644IKw\nN/rsZ2uRopRe9vcg6s2eN0o7ZonAqD7koNCMOy0AyjC0ACjD0AKgDEMLgDIEMSbZ2rVru1oUuMhc\nffXVYf2000476GuCiRR9yZ6FIIaENqJ6FiAYEsSIVjY9++yzYe+dd97Z1R588MGudtFFF4WPf/vb\n397Vzj777LA3WjsVva7W4tc2Y8aMsDcKaGThiuiMrOx5s+cYK3daAJRhaAFQhqEFQBmGFgBlGFoA\nlCE9OEGuuuqqsP6LX/xi5Of4yle+0tW+9rWvHfQ1wUTKVvEMSZFF6cHsYMdIlgiMDqLMUolbt27t\nar///e/D3nvvvberRYdIrly5Mnz8+eef39WiQxlbi1ONM2fODHujFGV2kGVUj1KCrcWvLfu9R/Ws\n1yGQAByWDC0AyjC0ACjD0AKgDEGMcbB79+6uduutt4a90ZfKxx13XNh73XXXdbVolQtUk33xHoU2\nhpzXlJ0ZFX1u9u7dG/Zu2LChq61fvz7s/de//tXVVq9e3dWidU2ttXbiiSd2tT179oS9Uchk1qxZ\nYW8UxMhebxSOmD179sjPm62Sip53SOAi404LgDIMLQDKMLQAKMPQAqAMQwuAMqQHx8Hll1/e1V54\n4YWRH//lL385rC9cuPCgrwkOFVGKLEsEZoc4RrI1TJHooNVnnnkm7L3rrru6WrbWaOnSpV1t1apV\nXW3ZsmXh46PPePYeRK8hW1v10ksvdbVsjVO0milLYUaGHLyZ9Q5Z9eVOC4AyDC0AyjC0ACjD0AKg\nDEGMAdatWxfW16xZM/JzfPzjH+9q11xzzcFeEhzyhpyrFMm+pI/OyMq+6N+yZUtXy87Iuv/++7ta\ndMZWa62dddZZXe28887raqeddlr4+BkzZnS1nTt3hr1R8CRb6xatlstEq6Ci97a11nbt2tXVotfQ\n2th/7xl3WgCUYWgBUIahBUAZhhYAZQhiJKKzZ6699tqwN/u/5SPnnntuV3NGFkeabJND9FmaOXNm\n2Bttcsi2PmzevLmrZWdkPfHEE11twYIFYW8UsDj++OO7WhZWiLZGZGdZRbKQShR4mDt37si9WZAj\n2rSRnZEVBUey6x1yzpY7LQDKMLQAKMPQAqAMQwuAMgwtAMqQHkzccMMNXe1Pf/rTyI+/6qqrwrqV\nTRyusgTYoLOSgjVM2blZ+/bt62rZmqCNGzd2tccffzzsjZJ+n/rUp8LeaC1bdMZW9h5EZ2RlaeQo\n/ZetrYoSiFkKM0pyRuuaWovTmVnScM6cOV0tO6dryHond1oAlGFoAVCGoQVAGYYWAGUIYiSuu+66\nMT3+u9/9bli3sonDVfZlehakiEShgCzEsGfPnq62adOmsPeBBx7oas8//3zYG52HtWrVqrD32GOP\n7WpRuOKFF14IHx+tosr+RkTvw8svvxz2Rr+LaAVTa639+9//HqmWOfvss0e+how1TgAclgwtAMow\ntAAow9ACoAxDC4AypAcnSLbaJFu7MlbR6pkstRUltKLEUyY6ILO11q6//vqRnyOSXW+U5MzWwfDm\nyRJgUT3rjf5tTZ0a/5nasmVLV3vkkUfC3qeeeqqrZWm6KJWYpfSigySj15AlAqN/89m/7WiF0qJF\ni8LeF198satlycqnn366q0Urslpr7Z3vfGdX27FjR9gbHZw5Hp9bd1oAlGFoAVCGoQVAGYYWAGUI\nYkyQE044YVJ/3tVXX93Vli1bFvZu3ry5q33/+98f92saL9F7+fnPf/5NuBIORhSkyFYzRYGiLLQx\nf/78rjZv3rywN1qX9Oqrr4a9//nPf7ranXfeGfZGgYUotJG9hlmzZo18XdHKqCzYEAUxssBEdH7X\nKaecEvZG72N2/lcUHMlWO1njBMBhydACoAxDC4AyDC0AyjC0AChDejBx5ZVXdrUbb7zxTbiS0dxw\nww0T8rxR8mvIoX6f/exnw/pFF1008nNEq2OoY8jBjpFs9dncuXO7WnYg4WOPPdbVnn322bA3+je/\nbt26sDdKGkYr3LLPTJR2jFJ3rcWv98QTTwx7o/c3Sma2FqeMV6xYEfbOmTNnpOtqrbXZs2d3tex3\nOeTASHdaAJRhaAFQhqEFQBmGFgBlTBnyBdgYTdoPmig//elPw3q2xmRU0Zk8rY19tdJXv/rVsP62\nt71t5Of4yEc+0tWWLFly0Nc0CUbfB8NB27p1a/d5zlbxRH9jhqztyXpnzpzZ1bLP4qOPPtrVsvOl\nnn/++a6WrUCKniNaw5SdQRcFE7JzvqLVSosXLw57ozBJdL5V9hzLly8Pe6M1TtnzRr+fIb/3RYsW\nhc3utAAow9ACoAxDC4AyDC0AyjC0AChDepDDjfTgJIjSg5lsdU8kWj80ffr0sDf625WtS4rSe1G6\nLXvebO3U9u3bu1q0tipLD0b1hQsXhr3R+xAdOJnJ1jhFScOo1lqc/hvSO4T0IADlGVoAlGFoAVCG\noQVAGc7TAibUkLOzohBDtpopWpeUnUU16uNbi4MFWZhk6dKlXS1adbR///7w8dHrnTZtWtgbXW/2\n3kb1LHQXhUGyEEVUH9KbcZ4WAIclQwuAMgwtAMowtAAow9ACoAzpQWDSZcmyaA1TlpAbkk6L0n/Z\nyqeoN1tVFKXeoqTgrl27wscPeb1D1mFFScMsoZelFSNDkqATtSLQnRYAZRhaAJRhaAFQhqEFQBmC\nGMCkG7S2JwkgRCGGrHdIiCEKeETrllrLV0yN5WdlzxmFIIascRoSJhnrWVgTyZ0WAGUYWgCUYWgB\nUIahBUAZhhYAZUgPAoe0LMkWJeSy1UxD1g8N6R2rISm/KME4JBF4uHCnBUAZhhYAZRhaAJRhaAFQ\nxpTD+Qs7AA4v7rQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKgDEMLgDIM\nLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKgDEMLgDL+F8RGWfKi\nZcbCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8599561358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_cache_frozen.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Tying weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "It is common to tie the weights of the encoder and the decoder (`weights_decoder = tf.transpose(weights_encoder)`). Unfortunately this makes it impossible (or very tricky) to use the `fully_connected()` function, so we need to build the Autoencoder manually:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0153905\n",
      "1 Train MSE: 0.0168082\n",
      "2 Train MSE: 0.0166031\n",
      "3 Train MSE: 0.0175659\n",
      "4 Train MSE: 0.0163731\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGydJREFUeJzt3Vls1PXXx/EvW6GttJSlBQQsZVFkE0VBMf7VRG5MNGq8\nUC80aqIx0UQSNZqYqDd6p9wYNRr3CxMS10SNS4xgEFwApW6ViiACtRW6UFpo4X/75Dmf8zy/H50Z\nejrv1+XJmZnfTGmPv8zH8x118uTJBABABKNP9wUAAJAVQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQ\nBkMLABAGQwsAEAZDCwAQxtgSvharN1AKo073BZSDtrY2fp9RVPX19fJ3mTstAEAYDC0AQBgMLQBA\nGKX8TgsAho08J1yMGqW/KlX1EydODOnx3nWpuve8Q+U973A4FYQ7LQBAGAwtAEAYDC0AQBgMLQBA\nGAwtAEAYpAcBjHgqDecl5AYHB03NS82NHm3/u1/VxowZk/m1vOuqqKiQdeX48eOZexXv/WZNS3oK\nkXbkTgsAEAZDCwAQBkMLABAGQwsAEAZBDAAhqbBAngDB2LH6z58KFqhwRUopjRs3LtNrDQwMyMer\nwEQh1jip583zfr1whfoc8qyoKgTutAAAYTC0AABhMLQAAGEwtAAAYTC0AABhkB4EMGwM9aDDPOuW\nvITc+PHjTc1L/2U9FNFLH1ZXV5uaWu2Ukv4cenp6Mr1+Sjrp6Onr65N1lUD01kuRHgQAlD2GFgAg\nDIYWACAMhhYAIAyCGABKLs+qIi+YoHq9cIV6jmPHjsleFSzwrlcFE1SQo6qqSj5eybPyyVvN5J3f\npaiARk1NjezNcy6Z+ny9QEoe3GkBAMJgaAEAwmBoAQDCYGgBAMJgaAEAwiA9CKDkvBSZlxRUjh49\nampe8q6zszPT41PSibwpU6bIXpWyU+lBL9WoEoEelQj0Uonq9bwEpHrePL3ee1C93ueQZ+UTd1oA\ngDAYWgCAMBhaAIAwGFoAgDDKKojx9ddfm9r69etl75lnnmlqlZWVsvfWW281tcmTJ8terw6UE++L\nfvWFvHe20z///GNq7e3tsnfv3r2m1tHRIXtra2tNzVtrpFY+TZs2zdS893DkyBFT81YwNTQ0mFqe\ndVgTJkyQvRMnTjS1uro62at+Pt55WnlCNQQxAAAjEkMLABAGQwsAEAZDCwAQBkMLABDGKC99UgQl\neyHP2WefbWotLS1FeS2VQEoppdWrVxfl9YqlsbHR1B5++GHZO2fOnCJfTSbZY0g4ZW1tbZl/n9Xf\nGO/vzuHDh03t4MGDsnf//v2m9tNPP8lelTT01jipJFt3d7fs9dZG/W/79u2TdbVeSiWXU0pp3rx5\npua9B7VaSf0up5TSmjVrTG3p0qWyVx0Y6R1EmSsRKNZ61dfXyyfgTgsAEAZDCwAQBkMLABAGQwsA\nEEZZrXF65513TG379u2yd/HixabW3Nwse7ds2WJq7777ruz9+OOPTW3u3Lmy948//pD1rLwvSGfM\nmGFqas2Nx/tC96GHHsr8HBh5vHCF+pK9v78/83OoVUcppXTgwAFT8wITajWSt35IXZu3hunYsWOm\npsIZ3plT6uwtb92SCm3s3r1b9qqA2ZIlS2RvfX29qanQWkr6er3Ahfq551k75eFOCwAQBkMLABAG\nQwsAEAZDCwAQBkMLABBGWaUHFy1alKnmWbZsmazfdNNNpvbUU0/JXpX28dKDra2tma9N8dJRKj3o\nXYNaf3POOecM6bowMg11bU9KKZ1xxhmm5q1Emz59uqmpdJtX91J6PT09ptbb2yt7FZUePHHihOyt\nqqoyNe+gWLW2ylvj9MMPP2S6rpT04bbe3448h0Cq18vzb8TDnRYAIAyGFgAgDIYWACAMhhYAIIyy\nCmKUkvclb54QQ56QSB5q7VR7e7vsXbVqlamtXbu24NeEWNTaHe9LdtXrBTFUrxfEqKmpMbXBwUHZ\nq3gBAnVtal1TSvp8KfUeVF9KKVVXV5uat35NnRWmfpdT0p+NCq6klNL8+fNlXVHvQ63ISkmvriKI\nAQAoKwwtAEAYDC0AQBgMLQBAGAwtAEAYpAdHMO/wvOuuu87UvDUzzzzzjKmptS+AR6XpvH9vKl1W\nV1cne9VhjV6STSUFvZVPKj2YJ/WmEoxemlitcfJWRnV0dJja4cOHZa9KDy5YsED2NjQ0ZLouj3eA\no/rM8vR6uNMCAITB0AIAhMHQAgCEwdACAIRBEGMEe+WVV2T9wIEDpjZlyhTZe9ZZZxXykjBCFGId\nj5JnDZM6e8sLV6jgh7daSfWqlUQp6dCECl14AYS+vj5T27t3r+z95ptvTO3QoUOyd968eaZ23nnn\nyV51fpcXHFE/d2/FlfocvfVdeXCnBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD9OAIsWvXLlNbt25d\n5sdv3rxZ1r2D44CsVOIsT4rMO6xR1b0DFFXCzUsqqvVQnZ2dsjdrQs5bL6VWrX388ceyd9OmTabW\n09Mje6+44gpT8w6VVYnLPId0eklDlYzMs77Lw50WACAMhhYAIAyGFgAgDIYWACAMghgjxPvvv29q\n3uqZG2+80dSampoKfk2AJ88X/XnOvfK+6Fe/C97vR1dXV+bnVedWTZw4UfYqf//9t6lt2bJF9u7c\nudPUli9fLntVXZ2blZI+H89bO6U+h2Kt9PJwpwUACIOhBQAIg6EFAAiDoQUACIMgRjDel8dvv/22\nqXlfYD/55JOm5v0f+0BW3pf36ov6PF/eeyGIgYEBU1PbLLxr886BUr8L1dXVsjfrWVSHDx+Wj//5\n559NTYUzUkpp6tSpprZq1SrZu3TpUlOrra2VvSrQ4v2dUZ+59/MpVmiDOy0AQBgMLQBAGAwtAEAY\nDC0AQBgMLQBAGKQHg3nppZdkfePGjaZ28803y15WNqEYCpEMU+dhqXOZUtJJtjzndHmJWfU+vNVM\n6vXUGVk//vijfLw6I6u7u1v2rly50tTWrFkjexsbG03NSxOrFGVvb6/sVSufjh49KnuHmhr1cKcF\nAAiDoQUACIOhBQAIg6EFAAiDIMYwtn37dlO79957Ze+kSZNM7Yknnij4NQGFkGc10+DgYObnVUEO\n7zm8IIa6Nu96VUjk119/NbUNGzbIx2/dutXUFixYIHuvvPJKU1u2bJnsHTdunKl5q5lU6MLrzfPZ\nVFRUmFqeVV8e7rQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYZAeHAa8NSg33XSTqXlJqltuucXUWNeE\n4UCly7x/xyrR19PTI3vVWqI86TRv5ZNKvXk6OjpM7fPPPze19957Tz5epR3PP/982XvFFVeYmjoY\nMiW9msn7zLu6ukxNpThT0qlEL7GpfhZ51mx5uNMCAITB0AIAhMHQAgCEwdACAIRBEKPE1JfSV199\ntexV62AWLVokex9//PGhXRhQJCpc4QUm1O+H9+W9CleoAEJKOkBQVVUle9X1es/b2tpqal9++aWp\n7dmzRz7+sssuM7ULLrhA9tbX15uat/5IXa93Tpfq9X4+KnTh9Sre9eZ5Du60AABhMLQAAGEwtAAA\nYTC0AABhMLQAAGGQHiyxf//919S++OKLzI9//fXXZX3y5MmneklAyXlrgtQKpcrKysy9/f39slel\n1lSiMCWdVlTrmlJKadu2bab2ww8/mJr3HtTKJu9gR/Uc3gGM6v16a5zU+83z2Xi9Sp6UoIc7LQBA\nGAwtAEAYDC0AQBgMLQBAGAQxiqSzs1PWV69enfk53njjDVNbsWLFKV8TcDqosMCECRNk7/Hjx01N\nrVVKSQcxvDVBireaSYU5duzYIXs3b95saup3f/HixfLxV111lalNmzZN9qrPwQueqKCLOn/M432O\nqp7nMy8E7rQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYZAeLJKXX35Z1tWhcZ5LL73U1Eqd1AGKwVvj\npFKF3iGQeajfG7VSLSV9YOPWrVtlb3Nzs6lVV1ebmlrXlFJKjY2NpuatZlJU2jIl/ZmpAxw9eQ7e\n9FYzFeLnJp+3KM8KAEARMLQAAGEwtAAAYTC0AABhEMQogJaWFlN77LHHSn8hwDCkvrz3vqRXZz55\nwQQV5vDOjDp69Kip7d+/X/bu3LnT1Hbt2iV7a2trTU2tWvvPf/4jH6+CJ3nCFXk+G+8zV0EK73mL\nFa7I4/RfAQAAGTG0AABhMLQAAGEwtAAAYTC0AABhkB4sgI0bN5paV1dX5scvWrRI1isrK0/5moDh\nzFtHlmeFUZ7Um0rkqXVLKenDEhsaGmSveo7p06eb2qxZs+TjVdqxr69P9hYruZcn3Zn18cXEnRYA\nIAyGFgAgDIYWACAMhhYAIAyCGCV2ySWXmNonn3wiewlioNzkOa9J9VZUVMhetW7JC22oYNTMmTNl\nb39/v6lNmzbN1KZOnSofr97DkSNHZK8KiOQJrngrrtQ5W8P53D7utAAAYTC0AABhMLQAAGEwtAAA\nYTC0AABhjPKSOUVQshdCWRu+sacRpK2trWS/z3kScp48qUS18sk7mFEl+saMGZP5tVQ9z2GNhUj5\nDYeDHZX6+nr55obn1QIAIDC0AABhMLQAAGEwtAAAYZQyiAEAwJBwpwUACIOhBQAIg6EFAAiDoQUA\nCIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD\noQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EF\nAAiDoQUACIOhBQAIg6EFAAhjbAlf62QJXwvla9TpvoBy0Nrayu8ziqqpqUn+LnOnBQAIg6EFAAiD\noQUACKOU32kBAP6HMWPGmNrg4KDsPXnSfo04alT5fYXLnRYAIAyGFgAgDIYWACAMhhYAIAyGFgAg\nDNKDAMrS6NH6v9lPnDiR+TlUek+l/LxEoHqtPNdVUVEhe9Xree9LXa9KNaaU/f0WE3daAIAwGFoA\ngDAYWgCAMBhaAIAwCGIUwJtvvmlqR44ckb3fffedqb3wwguZX+vRRx+V9SuvvNLULr/88szPC4wE\nXuChv7/f1LxgQnd3t6kNDAzIXvUcKqxQWVkpH6+e13sPyvjx42VdhTmqqqpkrxfmUEodulC40wIA\nhMHQAgCEwdACAITB0AIAhMHQAgCEMaqEaZDTHzsZonvuuUfWn3/++RJfiXXuueea2qZNm2RvbW1t\nsS/ndCq/U/FOg9bW1pL9PnsHHaq/XZ2dnbJX1X/77TfZu2/fPlP7888/Ze/Ro0dNrbe319S81UwT\nJ040tdmzZ8te9Tvu9arn9RKMKlWo3ldK+mcxbty4zL15NDU1ySfgTgsAEAZDCwAQBkMLABAGQwsA\nEAZrnBwqdFGIwMWKFStM7YYbbpC9LS0tpvbqq6/K3p9++snUNmzYIHvvuOOO/+sSgWElz7qlgwcP\nyt6dO3eamhfE2LVrl6l1dHTIXrUeSq1W8sIKSl9fn6yrAFVjY6PsrampMbXq6mrZe/z4cVPzVj6p\ngEae88e8QEqe0AZ3WgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwyj49uGfPHll/8cUXMz/HhRdeaGof\nffSR7FWpHO8QNnUY3O+//y57v/rqK1Nrb2+XvcBwpZJoedJp3uGrPT09mXvr6upMbfXq1bJ33rx5\npqZ+b1tbW+Xjt23blvm61HvwPpsJEyaY2tix+s+9ul7vGo4dO5b5Gry1UcqYMWMy93KnBQAIg6EF\nAAiDoQUACIOhBQAIo+yDGF5YQZ3VowIXKaX06aefmtoZZ5wxtAtLKb3yyium9s0332R+/LXXXjvk\nawBKSa078s52UudWDQwMyN4pU6aYmvc7es4555ja8uXLZa9a2aRWSR04cEA+Xr0372/S3LlzM/eq\n9ztp0iTZq1ZUeZ+jWp2lVkalpNdDeWEQghgAgBGJoQUACIOhBQAIg6EFAAiDoQUACKPs04Pnn3++\nrKtUjrduKc+6kjzUKim1RgUYztQBfyqdm5I+ANHrVauKvCSbOkDRS9Op9KDXm/UgyubmZvl4dRCl\nd1ijSjt6h0uq1UrqwMqU9M/H61V/A72koerNcximhzstAEAYDC0AQBgMLQBAGAwtAEAYZR/E8Kgv\nbovl9ddfl/UdO3Zkfo61a9eamjrrByg1L0ihqHU+o0fr/7ZWK4G831v1HDNnzpS9KgjhBRNaWlpM\n7YMPPjC1X375RT5end3lhcMuuugiU1u4cKHsVZ+jCrmkpIMY3lol9bNUZwSm5AfXhoo7LQBAGAwt\nAEAYDC0AQBgMLQBAGAwtAEAYpAdLbNu2baZ21113yV6VWJoxY4bsXb9+vakVYmUKUEoq5eelD9Wq\nIi+xplYgeSufBgcHTU2tZkoppU8++cTU1EGt3kGWy5YtM7VVq1bJ3iVLlpiaOuwxpZS6urpMTb2v\nlPRn5qUHVd3rVT9LlVTMizstAEAYDC0AQBgMLQBAGAwtAEAYBDFKbPPmzabmrYhR7r77bln31rkA\nkahwhfflvTpPy+tVq4aOHz8ue9W6o2+//Vb2bt++3dQOHz5sapMnT5aPV6vWVq5cKXvVmV7ee1BB\nDC/Qkmdlnfr55FnTVQjcaQEAwmBoAQDCYGgBAMJgaAEAwmBoAQDCID1YJLfffrusv/XWW5mf4/77\n7ze1Bx988JSvCRju1OoxlVhLaegrhTx//fWXqW3atEn27t69O9M1LFiwQD5e1b00n0oZHzp0SPaq\ntVFqlVVKOoXp6ezsNDXv58MhkACAssfQAgCEwdACAITB0AIAhEEQowB6enpM7cMPP5S9akVMQ0OD\n7H3kkUdMrVhfbgKllCcw4a0JqqysNDUvVKDWHalVRymltHXrVlNT4Qzv9aZOnWpq3jl4dXV1sq60\nt7ebmndOV3V1tal5QQwVfvGeV4UuvHO61M+N87QAAGWFoQUACIOhBQAIg6EFAAiDoQUACIP0YAHc\neOONptbW1pb58ffdd5+sewfHAdF5q38UL2lYU1OTuffYsWOm1tLSInubm5tNrbu7W/aqBGNjY6Op\nnX322fLx8+fPNzXvPfT29pqal9wbO9b+afeSlepn4f181Ot516vqeX7uHu60AABhMLQAAGEwtAAA\nYTC0AABhEMTI4bvvvpP1L774IvNzXH/99aa2bt26U70kICRvnY9a46QCFynpYIEXTNi3b5+peWdk\nff7556amVh2llNKFF15oaitXrjS1FStWyMfPnDnT1NSZVSnpFW7edamAiBeCOHLkiKl5a5xUGES9\nlvd63kquPOuduNMCAITB0AIAhMHQAgCEwdACAIRBEMOhvoh8+OGHZa/6v+09F1xwgalxRhbKjffF\nu/qi3vv9UL1egGD//v2m9v3338vegwcPmtq8efNk75w5czL1etttBgYGTE2FHVLSZ2RVVVXJXrWN\nwvtsDh06ZGodHR2yd+LEiabm/XzUGWZecCQP7rQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYZAedDz3\n3HOm9tlnn2V+/O233y7rrGwC/JVC48ePNzUvnasScn///bfs3bZtm6n9+eefsleth1q4cKHsXbx4\nsamp1UxeWvLAgQOmptYqpaSTe95aJJXcUwnKlFLavXu3qXkJRvU5qNdKSacKWeMEACgrDC0AQBgM\nLQBAGAwtAEAYBDEcjzzyyJAe//TTT8s6K5sA/9wrFdDwghjd3d2mpkIFKaXU3Nxsat65VbW1tabW\n1NQke2fPnm1qKlTgBUT27NmT6fGevr4+WW9vbzc1b22V+hxnzJghe7u6ukxt0qRJsjfP+8iDOy0A\nQBgMLQBAGAwtAEAYDC0AQBgMLQBAGKQHi6Snp0fWR48uzn8nqPU3as1NSjq51d/fn/m1vMPk1q9f\nn/k5FO96VZKzEIfJ4fTx1vmopKC3JkgdVPjbb7/JXlVXhx+mpA9s9FJ6+/bty1T766+/5ONVgtE7\n2FElAlWaL6WUWltbM11XSjrlp1ZZpZTS0qVLTc37vVV/61jjBAAoKwwtAEAYDC0AQBgMLQBAGAQx\niuTMM88s6evdfffdpqbO9UlJn+Hz7LPPFvyaCkV9lnfeeedpuBIUysDAgKyrgI335b3ihYQUL+DR\n1tZmalu2bJG9e/fuNTV1Hpb3fseOtX+CvbVI6jnU66eU0r///mtq3ud48cUXm5q3xilP4EuFKwqx\n2ok7LQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYpAcdt9xyi6m9/PLLp+FKsnnuueeK8rwq3eSlhZTb\nbrtN1lViybNmzZrMvYhBHfaYUr6VQtXV1aY2d+5c2XvJJZeYmrdSTa2Hamlpkb1ff/21qalUovo9\nSimlqVOnmtr06dNlb29vr6mpzyCllOrq6jK9VkopLVy40NRmzZoleysrK00tzxon0oMAgLLC0AIA\nhMHQAgCEwdACAIQxKs+KlCEq2QsVy2uvvSbr6gygPHbs2CHrQ12t9MADD8j6/PnzMz/HNddcY2r1\n9fWnfE0lMPRvevH/am1tHdLvs/eFvPp7VFNTI3vVWiPvfCm17qi5uVn2qnOnfv31V9n7448/mpoK\nmXihDxWCaGhokL3q74wKXKSkV0E1NjbK3gULFphabW2t7M0TxFA/4zznCTY1Ncl/JNxpAQDCYGgB\nAMJgaAEAwmBoAQDCYGgBAMIgPYiRhvRgCQw1PehRhwyqxFpK+sDI/v7+zM/b19cne9W6JO9wyT/+\n+MPUVPrQOwSyoqLC1NT7Skkn+rwDI1Wvl9wbHBw0NW91Vp4Vbnl6FdKDAIDwGFoAgDAYWgCAMBha\nAIAwOE8LwLChzqLKExZTwYaU9Gqlqqoq2avCDV6QQp07pa7Xe7wKg6jPICX9HrznVaELFbjweGu2\nVL0QZ2TlwZ0WACAMhhYAIAyGFgAgDIYWACAMhhYAIAzSgwCGDZWQ81Yz5UnIqQMUvbVGY8faP4ve\n86q6el61Rsp7LW+Nk1ovpR6fUr6UXwlX+RUEd1oAgDAYWgCAMBhaAIAwGFoAgDAIYgAY1vIECLzA\nhApzeL3q9bz1UOrMKPW83momtYbJO4dKXZcKrqSkP5tSr1sqFu60AABhMLQAAGEwtAAAYTC0AABh\nMLQAAGGQHgQwYnhpOrXuKE9Kz6PSf+rx3qokdb1eb57n9VZUjQQj950BAEYchhYAIAyGFgAgDIYW\nACCMUdHOUgEAlC/utAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEw\ntAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQA\nAGEwtAAAYTC0AABh/Bdj0PyoS9o+2QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85a05a36a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Unsupervised pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "logits = tf.matmul(hidden2, weights3) + biases3\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n",
    "loss = cross_entropy + reg_loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Regular training (without pretraining):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.946667 Test accuracy: 0.9231\n",
      "1 Train accuracy: 0.966667 Test accuracy: 0.9257\n",
      "2 Train accuracy: 0.96 Test accuracy: 0.9493\n",
      "3 Train accuracy: 0.973333 Test accuracy: 0.9424\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now reusing the first two layers of the autoencoder we pretrained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.946667\tTest accuracy: 0.9186\n",
      "1 Train accuracy: 0.966667\tTest accuracy: 0.9444\n",
      "2 Train accuracy: 0.953333\tTest accuracy: 0.9489\n",
      "3 Train accuracy: 0.966667\tTest accuracy: 0.9555\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # Freeze layers 1 and 2 (optional)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked denoising Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from tensorflow.contrib.layers import dropout\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.00001\n",
    "keep_prob = 0.7\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "is_training = tf.placeholder_with_default(False, shape=(), name='is_training')\n",
    "\n",
    "X_drop = dropout(X, keep_prob, is_training=is_training)\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X_drop, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0164072\n",
      "1 Train MSE: 0.0119087\n",
      "2 Train MSE: 0.0102273\n",
      "3 Train MSE: 0.00958246\n",
      "4 Train MSE: 0.00910537\n",
      "5 Train MSE: 0.0087664\n",
      "6 Train MSE: 0.00897745\n",
      "7 Train MSE: 0.00899594\n",
      "8 Train MSE: 0.00856953\n",
      "9 Train MSE: 0.0088828\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, is_training: True})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch, is_training: False})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG3lJREFUeJzt3W1s1Xf5x/GrG6XQOwalY9ypkzpWl82xZoiCzhBdjIsS\nNXugM1GniYuJGpeo2RIT9YkPlSdmGs2m0wfGm7mogcUtokOQEAJkIpTuRmG6jnajoze00A0f/3N9\nrn9+3/ac017t+/Xwyvf8+j2nnF775ffZ9W26cuWKAQCQwVVzvQEAAKqiaQEA0qBpAQDSoGkBANKg\naQEA0qBpAQDSoGkBANKgaQEA0qBpAQDSWNLAn8XoDTRC01xvYDEYHh5O/31uatL/VNSUoGhtVSWT\nh0r2NR/UYr/qGl1dXfLC3GkBANKgaQEA0qBpAQDSaOQzLQALWMlzn/nwfOaNN96Q9auuqv7f8vV4\nH9E1Z/usrRbP8NTaWnwGJdfgTgsAkAZNCwCQBk0LAJAGTQsAkAZNCwCQBulBAHU1H5KCKg1Xi5Tg\nkiX+T6hKJUY/S62dnp6uvK9Lly5VXqv2amZ29dVXV75GPaaFlOJOCwCQBk0LAJAGTQsAkAZNCwCQ\nBkEMADVRr8BF9KBf1aPRTEoUjlDv4/XXX6+8h6VLl1ZaZ6aDFNF7UAGNKFyhRGtLfm8qtFHy+lr8\nG+FOCwCQBk0LAJAGTQsAkAZNCwCQBk0LAJAG6UEADVeSCCxZ29zcLNdG6T+lZDSSSvqpPUSpuZIR\nSpOTk5Vf397e7mpRKrEk0ac+82gPJUnOEtxpAQDSoGkBANKgaQEA0qBpAQDSIIgBoK5KzlsqOeOq\nZISRGq0UnVul1kZhBVVXAYQo3HH58mVXi0IjKuARBU9aWlpcLXoPag/RWvWZRb8zVS/5HCPcaQEA\n0qBpAQDSoGkBANKgaQEA0qBpAQDSID0IoOGiRKFK3qk0n5lO2akknJnZ+fPnXS1KvamxRF1dXXJt\nlN6rck0zs9bWVleLUpFVD5yMRAm9sbExVytJMJYckFm6N4U7LQBAGjQtAEAaNC0AQBo0LQBAGosq\niPH3v//d1Xbv3i3Xrl+/3tWWL18u13760592tVWrVsm1UR1YTKKzllRdhSjMzJ5//nlX+8c//iHX\nvvzyy64WfZ+7u7tdbd26dXKt+j6rYEIUrlBBjChcocIcUYBBXXfZsmVybckZWYoaGRWJRmeV4E4L\nAJAGTQsAkAZNCwCQBk0LAJAGTQsAkEZTyfiMWWrYD4ps3rzZ1QYGBurys1asWCHr27Ztq8vPq5e3\nvOUtrvbAAw/ItW9605vqvJtKqs+OwYwNDw+773M0tkfVoxFKFy9edLXTp0/LtQcPHnS1w4cPy7Uv\nvviiq0UJxuuuu87V2tvb5drBwcFK121ra5OvV6LRUJ2dna4WJQ03bNjgatHfpJtvvtnVNm3aJNeq\n9xElI1UCMRr5pPrQ6tWr5T8o7rQAAGnQtAAAadC0AABp0LQAAGksqjFOv/vd71zt2LFjcu1NN93k\naidOnJBrDx065GqPP/64XPvEE0+42vXXXy/XvvDCC7JeVfSAdO3ata529uzZytdV4Qwzs2984xuV\nr4Hc1IPzkjOyImrMz8qVK+VaFRZQQY5obTSq6NKlS66mxkCZ6VFQ6kyvaF9qbRRWUOGV6D309/e7\nWvQ5qoBGT0+PXFtynpbaW/RvhPO0AAALEk0LAJAGTQsAkAZNCwCQBk0LAJDGohrj1EiTk5Oy/q9/\n/cvVovSgOuSuRDTiRaUHoz0MDQ252mOPPSbX7tq1q2B3dcMYpwYYGhpy3+eSvyXRAYwqiabSfGZm\no6Ojrhal6SYmJipfd2xsrNK+zHQqUCUCx8fH5etVmi46rFHtYf/+/XLt3r17XU2NpzIz+8pXvuJq\nfX19cq0aJVWLgx3Vv52uri7GOAEAcqNpAQDSoGkBANKgaQEA0lhUY5waKXqYeuONN1a+Rm9vb622\n83+osVPDw8Ny7Tvf+U5Xu/POO2u+J+RSckaWesgejXZS143GkXV1dbladBaVuq4KTJjpMEe036pn\nRk1NTcnXq/22tLTItSMjI66mvstmOhwR/e1R5wx2dHTIter9Rp+j+sxLRnpFuNMCAKRB0wIApEHT\nAgCkQdMCAKRB0wIApEF6cAGLRsd89KMfdbUo1fP973/f1aIRPFjcSsY41eKgQyUazTTbtGO0B1VX\nr48Sdu3t7ZV/1rPPPutqx48fl2tVsvKWW26Ra9UhkJGS9F/J2uhwSIU7LQBAGjQtAEAaNC0AQBo0\nLQBAGgQxFrBHHnlE1gcHB11NPbg1M3vzm99cyy0BZhYHMaK6ogIPUbhChQJaW1srX3e2IZO2tja5\nVu337Nmzcu3jjz/uaidPnpRrd+zY4Wp33HGHXKvGRkXBiNmOYYquW/L5cqcFAEiDpgUASIOmBQBI\ng6YFAEiDpgUASIP04ALx3HPPudr9999f+fUHDx6U9euuu27Ge8LiUjKKJ0oJliQC1TWiEUjqIMko\nCaf2EL03NZ5JvT46FFatPX36tFz7l7/8pfK+3vOe97jamjVr5FqVHrxw4YJcG6UglZJDIKPfsVxb\neSUAAHOMpgUASIOmBQBIg6YFAEiDIMYC8fvf/97VojN87r77bld761vfWvM9YXEpGcUTrVVBiugh\n/dKlSyuvVfVorQoQ1CI4ooyMjLja4cOH5dpz5865mgpcmJn19fW5WhQGuXjxoqtNTEzItc3Nza5W\nNIKp4LMJrzHrKwAA0CA0LQBAGjQtAEAaNC0AQBoEMZKJwhWPPfaYq6n/093M7Lvf/a6rRZMEgKpK\nJmKUiP5tqlBARAUAVJDDTE9tUGEFM7Px8XFXW7FihatNTk7K1586dcrVjh49Kte2t7e72nvf+165\nVk2yif52qPfb2dkp1166dMnVli9fLteqgMZsz+My404LAJAITQsAkAZNCwCQBk0LAJAGTQsAkAbp\nwWR+8pOfyPrTTz/tap/85CflWkY2oR6icT4qVRgl/9TaKD2o0n/RuKXp6WlXixJ9am2UHlTUfl9+\n+WW5Vo1fe+aZZ+TaD37wg662bds2ubajo8PVonSnShmrz8BMpzCj36VKGkZ7KBoFVXklAABzjKYF\nAEiDpgUASIOmBQBIgyDGPHbs2DFX+9KXviTXXnPNNa72ne98p+Z7AmohGuejQgHRWvWgP1qrQhcl\nY40iK1eurLQvFbiI6moEk5kOYlx//fVyrfoc1b7Mys4wU59NLc5RK8GdFgAgDZoWACANmhYAIA2a\nFgAgDZoWACAN0oPzQDQi5hOf+ISrRWNq7rnnHldjXBPmg5LDIdXaaKSQ+i5EI4VUQi76Lqk9qIMd\nzXTKTo1U+81vfiNfPzo66mqf/exn5dq+vj5Xi96vSkuqAyvN9HtobW2Va1V6sBYHO5bgTgsAkAZN\nCwCQBk0LAJAGTQsAkAZBjAZTDy3vuusuuba/v9/Vent75dpvf/vbs9sY0EBLlug/PSocET3oLwl4\nqLFGURBDhTbU2V1mZufOnXM1NZrp5MmT8vU7d+50tV27dsm1XV1drhaNorpw4YKrRWOc1O8iCnio\nUEzJ2kjJ75I7LQBAGjQtAEAaNC0AQBo0LQBAGjQtAEAapAcb7NVXX3W1ffv2VX79o48+KuurVq2a\n6ZaAhosOAyxJkanUW5SmU0nBaA8qDTc1NSXX7tmzp1JtzZo18vWf+tSnXK2np0euVZ/N2NiYXKve\nW5SALBlxpT7zouRfcLhkyeGQ3GkBANKgaQEA0qBpAQDSoGkBANIgiFEnr732mqxv27at8jV+/vOf\nu9qWLVtmvCdgLqiH7Orhv1lZYKLk4X3JmU/qjKtTp07JtX/4wx9cTYU2PvzhD8vXq78HUbBhYmLC\n1aLPYNmyZa4WhVSWL1/uatHnpX5eLcZsleBOCwCQBk0LAJAGTQsAkAZNCwCQBk0LAJAG6cE6efjh\nh2X9+eefr3yNHTt2uFq9EjlAvcw2cRaNFKr6ejM9Pii6rhq19sc//lGuffLJJ11t9erVrvaOd7xD\nvl6l/KLDGlV6MHq/Kp0ZJTZLfj8lSdCqrzfjEEgAwAJF0wIApEHTAgCkQdMCAKRBEKMGBgYGXO1b\n3/pW4zcCzEPqIXvJaKboDCY1lkidhWWmwwLqbCgzs8HBQVd78cUX5Vp1ntX27dtd7YYbbpCvV59N\nNG6ptbXV1UpCEFHAo+QzV+GVWoxx4jwtAMCCRNMCAKRB0wIApEHTAgCkQdMCAKRBerAGnn76aVe7\ncOFC5df39vbKujqcDcimJBmmlIxmilJvKimoxjWZmR0+fNjV+vv7K1937dq1rqZGO0Wvj96v+hyj\ntSrlFyUNp6enK183+nwbae53AABARTQtAEAaNC0AQBo0LQBAGgQxGuzd7363q/3pT3+SawliYCEo\nGeNU9fVmOligQgVmOpgQjUtSI5d27twp1951112utnXrVldT52aZmU1OTrpayfuNqM83er/quiWj\nmWYbtCnFnRYAIA2aFgAgDZoWACANmhYAIA2aFgAgjaYGJj8aGzHBYlX95DnM2PDw8IL9PqtEn0of\nmpm1tLS4WknKT60tGeNUouTgzVqYbdJw9erV8oPgTgsAkAZNCwCQBk0LAJAGTQsAkEYjgxgAAMwK\nd1oAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGha\nAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCA\nNGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0ljTwZ11p4M/C4tU01xtYDIaGhvg+\no666u7vld5k7LQBAGjQtAEAaNC0AQBqNfKYFACk1NelHpar+xhtvzOpnXbmiHxdGe1Cuusrfj0TX\njerzFXdaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRIDwJYlErSdCWJvtdff93VVJov0tLSUnlfly9f\nlmubm5td7eqrr5Zr1TXUe4jq0XXrhTstAEAaNC0AQBo0LQBAGjQtAEAaBDFq4Be/+IWrjY+Py7VH\njhxxtR/96EeVf9Y3v/lNWd+5c6erve9976t8XSAbFW6Ynp6WaycnJ11tZGRErh0cHHS1gYEBufa1\n115ztZdeesnVoiBGW1ubq73//e+Xa3t6elyttbVVrp2amqq8BxXwiD7HkkBJvcz9DgAAqIimBQBI\ng6YFAEiDpgUASIOmBQBIo6mBB4DlOmlM+OIXvyjrP/zhDxu8E+/tb3+7q+3fv1+uXbFiRb23M5eq\nn5SHGRsaGqrL91mNRYoSa2r80NjYmFz773//29UOHTok1/71r391tRMnTlTeQ2dnp6u98sor8vVq\n3NIHPvABufbjH/+4q916661ybUdHh6tdunRJrlWHVkYHTi5durTS62uhu7tbboI7LQBAGjQtAEAa\nNC0AQBo0LQBAGoxxCqjQRS0CF1u2bHE19YDVTI+O+elPfyrX/vOf/3S1X//613Lt5z73uf9vi8Cc\nUaGLKBSgggVnz56Va5966ilX+9WvflV5X319fbK+YcMGV1PnYZ05c0a+/sCBA6527NgxuXbr1q2u\ndtttt8m1JedeqTBI9JmXnL2lrlsL3GkBANKgaQEA0qBpAQDSoGkBANKgaQEA0lj06cEo1fPjH/+4\n8jVuv/12V9u7d69cqw5tU6NRzHQq59lnn5Vr//a3v7na8PCwXAvMtWg0U5RwUyYmJlxNjWsy00nc\naJyZ+j5/6EMfkmtvueUWV1MJu5MnT8rXq+9+lIBUn02U0FN/O6KDHZcsqd4G1Ni/6HeprhvtoQR3\nWgCANGhaAIA0aFoAgDRoWgCANBZ9ECMKK6gHjuoBrZnZk08+6Wrt7e2z25iZPfLII652+PDhyq/f\ntWvXrPcAzFbJGVlVX2+mz3GKvnebNm1yNRWiMDPbsWOHq0XjktTZWSog0t/fL1//n//8x9Wiv0nq\n/S5fvlyuVSGIyclJuVaFNqIzslTIJNpDvc5q5E4LAJAGTQsAkAZNCwCQBk0LAJAGTQsAkMaiTw9G\nqSCV4InGLUXpmdlSo6TUwXfAfBClxVQSLVqrUm/RmCF12KJKCZqZrVq1ytXWrl0r16oEYjQuaWpq\nytUGBwdd7cSJE/L16u9MlIDs6OiQdUWNfIr+fqnfz8WLF+XakiSoSiWq9KFZ2YGR3GkBANKgaQEA\n0qBpAQDSoGkBANJY9EGMSHTWTj08+uijsn78+PHK17jzzjtdLXooDdRDFK5Q9eiBvHp4H41xWrZs\nmau1tbXJtevWrat8XRX8iMYanT9/3tWeeeYZVzty5Ih8vQpybN26Va7duHFjpdeb6c88OqtM1aNw\nmfocos+xJLRRMvKJOy0AQBo0LQBAGjQtAEAaNC0AQBo0LQBAGqQHG+zo0aOu9oUvfEGuVcmgaPTM\n7t27Xa1kNAowW1GKTCXDohSZukaUelu5cqWrTU9Py7VqLFH0/VCpxPHxcbn2wIEDrrZ3715Xe+GF\nF+TrN2/e7Go333yzXFvyflX6L/oclWhcnPpdqsSnWfzvYbZrudMCAKRB0wIApEHTAgCkQdMCAKRB\nEKPBDh486GrRKBblvvvuk/UbbrhhxnsC6qnkIbsKC0QBgsnJSVeLzoxSo5miIIYaMdXf3y/XHjp0\nyNUGBgZc7dprr5Wv37JlS6WamVlnZ6erRe9XBSai34P6+zM6Olr5uiUBj5K1Ee60AABp0LQAAGnQ\ntAAAadC0AABp0LQAAGmQHqyTe++9V9Z/+ctfVr7GV7/6VVf7+te/PuM9AfVUchhgtDYa71T1ulES\ntySVODw87GrHjh2Ta9VBreo9RAey9vb2utqaNWvk2o6ODllXVAIyOshSrY1+D+ozixKM0Yip2eJO\nCwCQBk0LAJAGTQsAkAZNCwCQBkGMGhgbG3O1PXv2yLVq9Ez04PXBBx90teihJ7AQqHFLJWdkqXOk\noutG/vvf/7qaGtdkpkMb69evd7Xoe7thwwZX6+7ulmtLgifqPKyWlha59sKFC65WctZYSXgmOnur\nZLwTd1oAgDRoWgCANGhaAIA0aFoAgDRoWgCANEgP1sDdd9/taufOnav8+i9/+cuyvmrVqhnvCZjP\nShJnasxQJEqhqfr4+Lhce+bMGVeLDoFU6b2enh5Xu+222+TrVVKwZMRVlIpU14hSmCrZGKX81FqV\nVIz2Fl23BHdaAIA0aFoAgDRoWgCANGhaAIA0CGIUOHLkiKzv27ev8jU+9rGPudr9998/0y0B896V\nK1dcLQobqLXRqCJ1PlT0oH90dNTVBgYG5NonnnhC1pXt27e72rZt21xt8+bN8vUqIBIFG0rerxq3\nFF1X7aGtrU2uVb+3aHRW9PNmizstAEAaNC0AQBo0LQBAGjQtAEAaBDEC6qyeBx54QK4teeDY19fn\napyRhYVMPbxXgYuoHoUNVDBBfW/NzE6dOuVqv/3tb+VaFcS4/fbb5do77rjD1bZu3epqK1eulK9X\n+1UhCjP9dyYKqbzyyiuuFoUrrrnmGleLgjLRVA1FTfCIppuUTEjhTgsAkAZNCwCQBk0LAJAGTQsA\nkAZNCwCQBunBwEMPPeRqTz31VOXX33vvvbLOyCYgptJw0RlZKk13/vx5ufbAgQOutmfPHrlWJeQ2\nbtwo16qkYFdXl6tFqbmWlhZXi5J7HR0drhZ9NpOTk67W3Nws16q9RWk+9fuJzvRSqc8oNVqCOy0A\nQBo0LQBAGjQtAEAaNC0AQBoEMQIPPvjgrF7/ve99T9YZ2YTFRo1bih7eq2BBNMZJjR8aGRmRa196\n6SVXi8IR6uyr3t5euba1tdXVJiYmXC0aL6XeW3t7u1yrzq2KRjOpwIPal5kOV6jzx8zMXn31VVeL\n/qap8Eq03xLcaQEA0qBpAQDSoGkBANKgaQEA0qBpAQDSID1YJ2NjY7JecthZCTUOJhrxohJL0WFy\nSpSE2r17d+VrKNF+VZIzGkmDHKLDBFW9JHE2Pj4u6yWpN5UIjBKMJ0+edDX1HqLk3urVq11NjWsy\n02OrVDLTzOzs2bOudvr0ablWJQXPnDkj16q/XzfeeKNcq8ZZqQSkWfzdl3uovBIAgDlG0wIApEHT\nAgCkQdMCAKRBEKNO1q9f39Cfd99997naunXr5NrBwUFX+8EPflDzPdWK+iw///nPz8FOMBNqpFAU\nxFAP6qMQhBI90FfhiijEcPToUVdTIQgzs3379lVaG41mUgGqtWvXyrVKFIp67rnnXC0KYqj9RiGV\nW2+91dXU2CuzsnO6SnCnBQBIg6YFAEiDpgUASIOmBQBIg6YFAEiD9GDgnnvucbWHH354DnZSzUMP\nPVSX66rD+kpGrnzmM5+R9Xe9612Vr7F9+/bKazH/NDU1VV47OTnpalHiTP3b3LBhg1x70003uZoa\ndWRmtn//flf785//LNcuW7bM1dTYqSh9qA6yVNc005/NihUr5Fo1zipKYW7atKlSzUwnBd/2trfJ\ntdHIptniTgsAkAZNCwCQBk0LAJAGTQsAkEaTGrFSJw37QfXys5/9TNajh6xVHT9+XNZnO1rpa1/7\nmqz39PRUvsZHPvIRV7v22mtnvKcGqP7UHzM2NDRU+fusxiVFf3dU6CIKcqhRQ1FoY2RkxNWi750K\nXQwNDcm1Vf9+loQrovFSnZ2drrZmzRq5Vp1ltXHjRrlWBUdKPvNo7JR6z2psVfTzuru75Sa40wIA\npEHTAgCkQdMCAKRB0wIApEHTAgCkQXoQCw3pwQYoSQ+W/I1RI8LUuKZobZQebG5udrUopafGHUXv\nQe1hamrK1UZHR+XrVfI4er8qYaeSf2Y6uRe9X7WH6HBJtYeS/ZYgPQgASI+mBQBIg6YFAEiDpgUA\nSIPztADUVckD+cuXL1eqmZm1tra6WhTEUMECFZgwMxsbG3O1aAyTOjNKXTc690q9PgpMqHp0RlZJ\n+EV9ZiXndEWfudrvbMMZZtxpAQASoWkBANKgaQEA0qBpAQDSoGkBANIgPQhg3ig5BFIl5KLUnLqG\nGsFkpkcjTU9Py7VqBJJKO54/f16+fnBw0NVK3q9KUJrpsVXRuCVFHU5ppn8/JZ95LXCnBQBIg6YF\nAEiDpgUASIOmBQBIgyAGgHktetAfhSOqisYlqeuqcUtmem8qgKACG2Y6SBGNZlIhiChMot5DvYIR\njcadFgAgDZoWACANmhYAIA2aFgAgDZoWACAN0oMA5o2ShJtK/0UHO5ZcV41AKjmYseo1zXRSMEoE\nloxQin7eQsCdFgAgDZoWACANmhYAIA2aFgAgjaboQR4AAPMNd1oAgDRoWgCANGhaAIA0aFoAgDRo\nWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoA\ngDRoWgCANGhaAIA0aFoAgDRoWgCANGhaAIA0aFoAgDRoWgCANP4HDftFHJ/e8U0AAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85c08d90b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Visualizing the extracted features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_stacked_denoising.ckpt\")\n",
    "    weights1_val = weights1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure extracted_features_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAABYCAYAAABPlQhTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH9FJREFUeJztnUtwXFe1hn89u2XZkWU7dhIljqWQhJgAgfCIeRMHDITn\nAAZhwIAJBVXMGMCAKqqgijGjjHgEikGKV1GkgPBOCM9gYhLA5OHEJE4kx0aWLduyWt19B7rf2rvX\n6bZl7kW9gfVPWuo+fc7pddbe+//XWnvvgXa7rUAgEAgESsRgv28gEAgEAoFeiEEqEAgEAsUiBqlA\nIBAIFIsYpAKBQCBQLGKQCgQCgUCxiEEqEAgEAsUiBqlAIBAIFIsYpAKBQCBQLGKQCgQCgUCxiEEq\nEAgEAsUiBqlAIBAIFIvh9brQnXfe2XWRwMHB1XGy1Wp1/N3rGEkaGhqSJDWbzY7j2u22BgYG7O/8\nWH/O893H+d7L8ZGPfGTgvCf9F+OTn/xkh02532azaXbwn/Fb8mOlVXsND3d3B77j13kcGhrq+az8\nd9dyb5/73Of6as8vfOELHT/Q+9Ja4L/jf3+3Y1dWViTJ7M93vZ38++12254ffu7x8Y9/vG82/eIX\nv7hmw3kb8xu933azp7eL9+1edvTXyvuPXvf34Q9/uG/2vOmmm9pStT3yO4eHh82XRkdHJUmNRkNS\n8o8NGzZIkubn5yVJ4+PjkpIPDg4O2vl4rdfrkqTTp093fIfrLy0t2fWlTnvXajVJ0rlz5+z8+bEP\nPvjgBe25boMU4If5BrkW+EGKB8EPX15ergxcPCSO4bu+UfO9vFFcaGDrN/gNvrPrNnj06ix5f2Rk\n5KJJQn4dru3P69GtEyjNztyj94l/Bt7Pcn/3nWs30tXtXPn7vQanktBr0O32Gf/7QT4nU/5c/rx8\nh/e97/ciufl5eg1k/QR92aZNmySle2QAkKr36X33H//4h6Q00IDcj0ZGRiSlPppzMOAwWF166aUd\n5/D2HB0dtXv2z+9i2nyE+wKBQCBQLNZdSfViMVJvtg87QKp6JcX7rVZLR48elSTNzs52HHPJJZdI\nkjZu3Nhx7uXl5Y5r5+y5NIbv0c2G0vlZig+B8JqH+iYmJiR1hgAkaWxsrOMcIyMjOnv2bMc1sSff\n5fwcVwIjvRBge+cL0fV6n+/4EB6fj46Omq0Ik3AsPooNeYXZ8sr3G43GRUUi1hvnU1CgV1gP8P+J\nEyckJXvy/6lTp+w627dvlyS94AUvkJSYP33BwsKCpNRfYE++v7Ky0jPSUgK4J99ngbw/4DPshXLC\nJtgbFba4uChp1QfpI31fwXdRY5s3b+56n5x7YGDAvuvteTHRiVBSgUAgECgW66akejGlnAleKGnJ\nyA6jh3nCjBYWFoxh8Z2rrrpKkrRz505JKa779NNPS6qykpwFMPp7VfB/yVH8f+J8OSTPgnxCle/A\njur1unbs2CEpsS+O9cn5Z599VpJ0+PBhszP2JV4OM+M1z0XC3vy9lIpuuQrvq/wWr/rxUV4bjYYO\nHTokKfngli1bJFUT4iSst27dKik9B5LejUajwphz/y0FXu3leVCPXrkLfI+2jw8NDQ3p+uuvlyTd\ncsstkqqMH5vgt17J8qzOly8sATzj/LdLndEfn7fkWH7r5ZdfLikpUfq/6elpSdLk5KQpKZQn/vbU\nU09VridJl112Wcf7XKvValWiL+SzuO5aUEZvGwgEAoFAF6ybklprqbKU2LcvW/TMe9u2bZJSDqXR\naBj7hNm/8pWvlJRY6cGDByVJJ0+elCSdOXNGUopnozgWFxeNlcD4SlFQoJcCaTabFWYNo+G3YA/e\nn5yctL+x0eOPPy5JprCwETY7duxY5VnceuutklIO6r777pOU2Fir1VrztID1xoVKlZvNZs+cCWyc\nV9QRtoPdP/roo3riiSckJdU5MzMjKT0Tcij8TxXVn/70p47XM2fOWKSAY2gjJeSqeqmQXAHgj7Bv\nWPzU1JSk5K/8f8UVV3S8f+bMGTsf57j33nslSV/72tckpejJG9/4RkkpZzU3Nycp9QUbNmzoWbZe\ngqLyCoq2nVcwc/8od/pS/PLUqVOSVtu7lGx23XXXSVptx8ePH5eUfJj+FXs9+eSTkpIa4tmg+Hk2\nKysr5ofYmO/Q16wF6144AXzZbbvdto7NJ4xprBgNw9Px0QE0Gg29+MUvlpSMcOTIEUnpofGA+S6N\nm8QiHfPzzz9v16Nz7lUq3C/4cF8eUuN3YjuA3CYUQgik2Wzq+eeflyT95je/kZQ6PAZ6bEvHeM01\n1+gb3/iGJOkrX/mKJOlTn/qUJOm2226TlJ4l57788svteflQZCnoVSbbaDTMfvggA41P6DMQveQl\nL5GUGvrCwoKFVvAvOg580pdj//SnP5Uk/frXv+74v1ar2bN5wxveICl1Pr3mvJWEZrNpHSztjwGE\njpKOmXZKH4F9v/SlL5n9GOzx2/e85z2SpFe84hWSUnuBhDEg8szyY7Cfn8LST3AP/h5pTydPnrT3\naG8UN/Adwn20S0KkDFLPPPOMXQ+yhD/edNNNklJ/+PDDD0uqDvb4dT5I8dz+mTBq/y0fCAQCgUAP\n9I1u+TLfPMmG6iEhBzuE/TPC8z6q4bLLLjN2xCgPM4ClwjxhDqg1RnqYcqPRsM84f0msSqoy/TwM\n4BUUbAflhH1gXrfccosxpDe96U2SEtuirB/7852dO3cag4fxYj8YHMwY+9ZqNWNRnL8Ue14oRNZq\ntSqJaJ+oxs4knY8dOyYp+erY2JixWezNOfFVbIe9iQbAmAmvPP3003Yd7I5iKxF+om6tVrPfBAgl\n7d+/X5L0xz/+UZL05je/WZL0rne9q+O4HTt26IMf/KD9LaU2jW1ot9gRW/FKX3Do0CF7Fl6JlhDu\n8/dE++H9ZrNp7Z4+lDA8fevNN98sKT0D+sNHH33UzoEd77jjDknStddeKympWZTrD37wA0nSXXfd\nJSnZExvyjKTkl/55rwVl9A6BQCAQCHRB35SUX1MqX3cKJgSj5FgS9sScYRLE6K+88kqLh/7973+X\nlBgD78MoOAevhw8f7ri/7du3V5San/BbCvwE0oGBgUpSFVaDKiK+jI2bzaYxe2yEvb3SwS67d++2\nXMJf//pXSSkuzXVIdPM8ms2mqapSFFQv+JzfyMiI/V5shF/hR/wPu/WvS0tLls+iBJ38AWqfc/DM\nyJPeeOONkqQbbrhB0ir75fkSQei2flq/4cviad+XXHKJMX1AlIT2+Le//U1SyoeiLp977jlJq7lB\n7AbDB7QHfJJcK0rhVa96laTOteg4l89XYed+wudKfS53YmLC+iryxvgpSgZlz+8kz0lkYN++fdq9\ne7ekZLef//znklI/iP1QXzyTPXv2SEp+/dRTT1XyZj6KshaU3UsEAoFA4L8a6z6Z16uQfMVdmBCx\nY/InsAEYBLFRRnBw5swZq/phxKaSCsXA57BVv8IvI/5VV11ljAFFAeMtDdgwXy4HBogtYeXE38kl\nwaAOHz5s7MqrAnIoMGDY1w033GDsKmdPUrI77ApGtbi4aLamLLY0ReWrOPNKSb+sDGyWKQ/4k69O\n4/3HH3/c/saefmIlSov2QHXa3r17JSVfvvfee01xYEu/lFUJ8BN0uf8dO3aYLXiPV6rOUOK0xcce\ne6zjtd1uG+PHjvQXqC/yWuQLX/e610lKaiOfRkAeizafV/71G7QhnrFf5GByctLKwLEjfvjII490\nvI+C5ZUq1KmpKcsHcj2UFOemDdxzzz2Sko1e85rXSFqNaPGKT6NW/YKza0E5nhwIBAKBgEPfJvP6\nPUsajYaxF5gCnzEKkwtBYfllTaQUe4UxwIpZigY1BEuFJfsFUGu1WmWxVJjvxcRT/5XopU4bjYbd\no19WH8b0spe9TJL0q1/9StKqWkIxkfPANtgOO2DDu+++W69+9aslpSqh3/72t5JSTgHlRqXm8ePH\ne+5t1W/4PA5MMt+LBx+FlXs/xhdh9ygp/HLDhg3mxzwbVBjnpDLr9ttv7/icqj7yfQMDAx3tJ7/n\nEiZKw/Q9e8Y27XbbfisqHhaOP1HxiB2xPypoZmbGzkFuGhVPbgXmz7lRb76Sb+PGjRZFIBeGKith\nLp/vd7g37Ds6Omr3Sd9IhAP7UiX5lre8RVJSR/zukydPmp3wIezLnECewbe+9S1Jyb4cxz2cO3eu\nUhGMz3O9taCM3iEQCAQCgS7oW3UfqgSVNDIyYswGRgtTgPnA6GEJnIO5OK1Wy3IwnBcmQVULigJW\n55caYYSfn5+vbJnQa8mUfsMvh99ut802fnsT3oeN8/vz1TpQWdj7mmuukZTUJjH9/fv327WZSwFT\nAqhe1MD4+Lg9PxRUCUv4SNW5e74qrVarVVQB6odqMM4B28Te2HBlZcVYut/pFIUBMyUn6G2L/XLF\nDPySOSWAe+G+83lftEvycPv27ZNUzbPCzsmx8Lq0tKS//OUvkqSvf/3rkqS3v/3tkpLvcyxqlrw0\n/o1qm56etnvkFZ8vAX75J78Q8dzcnOWYsCd+yXJlH/rQhyQl27DEFhWkW7ZssefFuWi7V199taTq\nwsioMfwY+x47dszOwfPjulHdFwgEAoH/CKx7dR/wS++Pjo7aSOw/Y+SGEQEYE+d+4oknOuLLUmK0\njODkXagOIs4KW4WdLCws2Hd8dVJp8DP5821GYPIoRFiYX4Puhz/8oV7+8pdLSgzp/vvvl5SYlN8s\nbmpqyhQC7AlWxxyKbnO4fIVcafBrIeYbN2I/7Ik/Ecf3W8WQk9u1a5ck6cCBA5UKU6/qsRW2xe7k\nFWgnO3fuNPVV8koTPGe/eWOtVrNqOnKYVI769Q1RsH7dvdHRUWPyn/jEJzqui8Liu+SxyEfzLHlm\nJ06csGeAgiohFwXyzUal1E/SltvttuWAaGP0a8yLZH7Zgw8+KCnNb8w3kMXfyJ3i2zyjz3/+85JS\nvhC70veilOv1uj0nbO19fC0os5cIBAKBQEB9yEn1qugaHBw0Vg+YcwOD9Mv2MzrnW8YzrwIm4Stg\niENzLViYX6330ksvNeYC2+P/UlaYxpZ+1eyxsTH7G4YNw4J1+ZzV1VdfbUr197//vaQ0+x+W5TdO\nnJmZsWuTW4CpEXMmZwMznpiYKCrOfz7k24pLq7bjPVgm/sOzgKXjd/gOtjx69KgpUxgqagi/eulL\nXyop5Vrz6q383OPj43YfHiVU9wG//ThKcGZmxqo+Dxw40HEs+Q8qxvwcRY7btWuXVarRpn/3u99J\nSmoBO3NdfBTVT7Tl4Ycfthyq31q+hO3kfeWmn9u5srJS2XwTBU/f9aMf/UhSap/kn/MNZelP6WeJ\nBnz5y1+WlPpb1vjkc5QWfcHOnTvtPng2vXZnOB9CSQUCgUCgWPRtnpSv5282m8ZwGKlhPjAFWD/M\nB6bE/1u2bLHRn1goOQMUlN+kzs+jIke1cePGynytUqv7QM7+/NqE/G6YPceyB8+OHTuMcZIHwEbY\nBgaPjaanpy0uTcWaZ/YwqHzdxXzV5pLhVf/y8rL5hJ8zglIlf4odOAdx+tnZWbMvz4BIAarB55ew\nIefA5keOHOm4txwXMw/lXw3aC6/41bXXXmuMmnyob5ewcv5n7tOf//xnSdJDDz1kbR6/wn6s+I3i\n8NVx9CtUth06dKiyCSB9UAltHkXt99nLfw9RChTSC1/4QknJT1l5hqo7fidKdW5urmIv+mPa/2tf\n+9qO+8D+RGDox6empirrNXIfF4NQUoFAIBAoFuueXPGrDeRqhXgwzB12xQoIfs8eGClx7e3btxuj\nJZ/lt+JmZIfpwpSoBOL/5eVlyxX49dBKq0rLt+MGvuIPZg0zpbqRlaAXFxeNTWEbz85RS1ToNJtN\nU6qsPAHLh3WRo4HhXUwsut/o5atS+h2wW45hJXm/9TZsXUrPBHviZ6hczsX7xPpZzYMV5a+77jrL\nOeR7NOXn6Cd8rjSv6pNWVRP2weewG+2Xtk7+g+Py7cg5xuc6UaZUtpH3wmf5nO898sgjVnmJXf38\ntxLgFWFeyUvb5LfxO1BQfI79fvnLX0pSx7qdXtmTx8KHWVcRtcbq8oD+uF6vW5tBQaHcUFtrQd8K\nJ3wjWl5etnCGHwz88kd0BEhZnP706dNW+kzDZvIu8hbnpzH4a+XbVeD8oATJ3w1+IVQp3SthKRwP\nGzKIgPn5eQtV+TJfHJsENQ5+4MABsxuOif0ZFClP5fqtVqtSklwq/LJTg4OD9h4hOBoh5IZCCgZ4\nOjc+n5iYsHP4hWV5JnQGDGIM9HQ0eWfM8l8eJQxSwLebfIFdBlwGaGzDb/R+il/lYWVsQHsl7EeH\nyIRVCBqklyKrhx56SNJqCBGiDPng2ZRErrg3P0G21WpVwu0QT7Z5wa6876dKjI6O6p3vfKek5PcM\nQhRW0S9wTgY6nl2eLqA/oA9ns1XayVpQliQIBAKBQCBD38J9APY/NDRkkxdhL7AaXmGHjOSoIljO\nyspKZcIjST3CUACFhdLIS7Gl1QQ4oz/gOiWxVKnK+FutViVxnttIqpYDz87O2nmQ+dj3ve99r6Q0\nqZcQ6unTpy08gt1gzZyD9wm1HD16tGeBQCno9Xw3b95cKSnH5ih1vzmeX7i4Xq+bf/vwHvbgFZbL\nuZm8DqOt1+vGnPH7ErfqANgGFv/ss8/qF7/4haTVyeSS9NGPflRS8jV+D6FOVDx23717t23FgV0I\n73EsJem0eaZDoLxQWM1ms7LRaUl29IrUhyLr9bptaUI4jVcUKdu+0N+h0ukfd+3aZX0FqoeoAcqJ\nCBaRLz9FAmV75MgRC/NxTuwayyIFAoFA4D8C66ak/NbHHrVazbaRgAH5zQZhSpSffuc735GUWMG+\nffssKQqI+8MoGOWJm3INciowp9nZ2cpS/qUjT1DDboj9wopQTrAgWOSxY8dMOcLGSLpSWkoOCRXR\nbrctT0LZql9w1i/IeurUKbuHUtFL7W/evNlUPSoI9ogyzadUSMke2KfZbBoDJl9HLpDrYh/8GmWA\nb+PT8/PzxoRRHBQOlZQ/9UUHtO/9+/frzjvv7DiGZXqwDfamsInPWQx5z5491rZRA0RP/OIA73vf\n+yQldU8b4Jnt3r3brovqKtGOfhI/KmViYsL8AWAvPwWFvo0NDolubNu2zbbvod3TR1Aogf9Sms41\nfWSk3W5X7pk+JDY9DAQCgcB/BNZNIvjtJPIFR6XVkZbRlgl7MEcq7hjtYUCog7zsF5VFhRQsgOtx\nLAwYdUbuAIW1srJizMCrv5Li1JIqEzqbzaYpRlRgXgEppcox7LCwsGDqx8e6YVt+U775+XljRFQE\nsi09cWueB/dx9OhRY7q+xLs0+K06xsbGKnF3DxiqL8FHfV555ZWWb0HF43s8P9oIdiGS4DfhXFxc\ntHPki4zm91wCfCk6PvPkk0+aHV/0ohdJSu2Tijz6AqY44Jv4kJSUP++RJ4W1U61GBRtqk8ozog23\n3nqrtX+/2WEJC83iFyhR7pG23Ww27W+/lBuqh1f6R1TkbbfdJmnVj+6++25JKQrCtvDkpPBHcoF+\nIW4wPz9v90P7/2fsWFZvGwgEAoFAhnVPtvRasDHfLI/R1sdefR0+k31h9sePH69Us/jtin2smYnD\nMGBY7uDgoF0/30wwv69SgL3yuTk+18YxTOLdu3dvx/tjY2PGqlBMVD56W/L7JycnzV4oBpgTChbV\nC3udmJiobGhXCjzj92rk7Nmzxrr9FvDYke/iT9iQRWPzeVIwYtQEispvasfyX/n8Imk1d+Wv639D\nCfCTebnHI0eOmIJ6//vfLynl4/xmfLfffrukZE8UwYkTJ+x8sHWv0GkX+D65K2yHEqvVapXJ534j\nzH6CduO3uuH/6elpU1CoINqhn7jMuagDYDHjxx57TA888ICkVE3qNzukLdNPcE3mnRGdyqNj3GO3\nhQcuhP5bPhAIBAKBHlh3JeU37so3mGOuByM2lT1+TgrMk2oT1NEDDzxgDAvmynWY88Pn5F+o5mHW\nOcdt3rzZvuvzZ6UBpgiTnJ+ft3v2vxuFxW/BdnNzc8a+sDeM028AB4PatWuXqSwYG8oUBswzRFFN\nT09X5qCUoqi8+vD3tbS0VMkLcQw+Sx6VV7+01sLCglVE4seoMfItMFdWTfEgKrC8vFy55xKVlF9g\nFpucPXvWGDZMH3/El8kvsTQPSgfFtWXLFsuRcF6UEWoWX4ThU/XHcbTz06dPVxaTxkdLsKfP5/ut\ng+bm5kxpYgvyRbRZ7EskgJwf+bnvfve7ev3rXy9J+tjHPiYpVZP+5Cc/6bgP2joqzG9XMzQ0VFkW\nif8vpmI6lFQgEAgEisW6KynP9Bh9z507V1nRAXZO5ZjfqAv2BWt47rnn7D2qy/gMpsBscz/nhGvC\nsoaHhzu2ts/vuTT47TgGBgaMXcKkYDv8JtZMy6sfqayCScG6YEbf/va3JSV1NjQ0VGG2rAwAY+KZ\nog4mJycri6CWEO8/H/DZ+fl58wFYuV+bD4aIOuI4sGnTJjsfNiLHmi9uLKV5J6h+ACtdWFioKLqS\nQa4Nfx0cHLQ80fe+9z1J0h133CEptXFUDmqIHAo+dOrUKfN18oUcg3pHRbDGHAoKNcHzGB8ft+fl\nc+YltH0/xyiPQvE/PsM8sp/97GeSUjtn7iNVuOCee+6RtKrmP/3pT0tK7f/73/++pLRqDHlov208\nVcBEYsbGxixakG/Vc7Eou3cIBAKBwH81+raUgt9CYuvWrZU5CTCdt73tbZISo2S+AyM4I3yr1TLF\n4OdBkXNiJW9YCXkArpXPm0KFeJTO/MfHx60Ch/wH7JJcHVWQsNv5+XlTWbAeGBFszG88NzIyYs8A\nu8HkOAesC0ZVr9crW3OXpgJ87jHPqfgKR34nvoI9UeTYhTk8V1xxhdke38NX+Y6fwY9qQGGRa8ln\n9Pt7LoH5e/h8z6ZNmyyHgsph4038lN+M+kFtkicdGhqyaAg+TpUZeVjaAP/jiyivXP0SiSkx/+xV\nM/0QKml4eNgiR/RnrFaCUqXal9U3WF2CCNO73/1ui3rcddddkqTPfOYz9pmUng1zLf/whz9Iqq7m\nPzAwYO/R5nmNtfsCgUAg8B+BvikpzwqazaZVl/mYPbFPWDnME9YKm63VasZo/cZ9HEtNv98nBlab\n79Xj50iUlkPxLDpf+w22CMuEpbNhGbPI77//fkmra3xRvXP99ddLqm5SxwrK+Sr0VLWxUjcxb8+q\n8nyZV06lsFb/fH3Mf3h42Fg3jB4b+fl3VDXij6iioaEhY+u5IpKScuVZ4df5KtdSUlrNZrMyh69k\n+P3DpqambJ4UvseK5ahNclOsLoM/ESmYnZ01u1AJSC6KXAp2fcc73iEp7SeFkqJvyOdqllZ5KnWu\nLJGD+z98+LCpRXJS3D8RJNrufffdJ6la6XjjjTda/u+rX/2qJOmtb32rpLQqCP0yfUu+IouU8nnn\nzp2zfgB74vMXs59U3wYpGrtfeFBKgxAdAaXphAQIEeB8JE6Hh4etU/ALyzIA8opD8oBxdDqQlZWV\nnmWSJSyRIlUHp7xB0Snyu7ADjoe9OW7btm3as2ePpLQoJSXoJFl90cvS0pLZm04a+3EMHS0NK9/K\norRB399Ht515aYD4hp+g6v0Z++KzrVbLOlhsBKHARoRvOCcdDw07T0L7DqsUW+bwZCovoSd0TKGT\nD/ljC8gVoSja7YEDB6xU/8c//rGkRLwgtZ/97GclSTfffLOkZCM6Y18e3Q0lECnf4eNrtPF8U04K\nowgz0z75DhN2/aTeb37zmxYeZdCZmZmRlMgR/ojdGTx9ufno6GhlqhHnJCy5FpTn0YFAIBAI/C/6\npqQ8i242m5Wl5xl1Ge2RirArRmyS08vLy7acCnIWtuSXsYEF+I0N860XPHsqkaVK3RfAxYa8okpR\nMn6jvb1791rSmtAKk/VgULAi1NLWrVuNJfsFTnvd48DAQGVprNJCVV4x5sw136RTqpYF8x3KyrE3\nW00888wzFiIl7MSxsF4fZuUVELJptVr/Fssh9SrqGB0dtb9RmrRT7EZEBHXPVubYpF6vW5vFLh/4\nwAckpQgAoS+OoyTdL+yb286HekuA3/YGNZ+Xy9Nn8hn9HG0We9IvUliR96V5IZqUwqf0A37Rbj+5\nOC84y0N/+evFoMxeNxAIBAIB9VFJddtYzhdMkIsCvrAClpNPOkVV8QrjInHIsTAJP1HXb8+Qo6Qk\n6lrRjW1J1YKSgwcPmk2IZ7PNBjbEVrChpaWlSjk55/NL4fw7wZfG4xODg4OVz/xWGdjVL47M+8vL\nyzZpmlJpVK7Pa/lJ2n5Jq5zll8T4QS8FledS/W8mX4fN/RJaJPrxwYWFBesnfJkzitVv7MezoD/B\nZ1utVs/pByWAfpH22M0PsIvPs6F+vBqijJz3Dx48aMqeYgpy/t72XLdbwYS0+sz8knIcczF2DSUV\nCAQCgWKxbkqqV0lnN3XCSOzj/L22T+b/8fFxG6mJi1IR5fNdPjfjz5nfc6k4n017bTTnNyHDXpTm\n58dQ/gtgqLC0er1euQdftdntnvlOKVWSvXC+548tfG6q13YfnIvluPLvePWAb/aa8JgzZ87vF0Qt\nARda/Db3kfOpVylNcfCly61Wy367Vw/Yl9xTXmJ+oXsuaYsOwO/BP7zKHBgYqPx2v5Em6p1X+loq\npScnJ03ZU8VHvtAvZ+ajBt2mlqDQiGD5rULWgnKeQCAQCAQCDgMlxrIDgUAgEJBCSQUCgUCgYMQg\nFQgEAoFiEYNUIBAIBIpFDFKBQCAQKBYxSAUCgUCgWMQgFQgEAoFiEYNUIBAIBIpFDFKBQCAQKBYx\nSAUCgUCgWMQgFQgEAoFiEYNUIBAIBIpFDFKBQCAQKBYxSAUCgUCgWMQgFQgEAoFiEYNUIBAIBIpF\nDFKBQCAQKBYxSAUCgUCgWMQgFQgEAoFiEYNUIBAIBIpFDFKBQCAQKBYxSAUCgUCgWMQgFQgEAoFi\nEYNUIBAIBIrF/wCX1IpalbmNdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85c0911828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i + 1)\n",
    "    plot_image(weights1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Sparse Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ageron/dev/py/envs/ml/lib/python3.5/site-packages/ipykernel/__main__.py:3: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sparsity_loss_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcjXX7wPHPNTKYDEaN7Ma+hZBKZUlpoWwpIsnyPHqS\n6kGFNku0/RCylFLTQhKKaLWTtZAlWce+NoxlBrN8f398z8wzM2bGrOc+y/V+vc6rzjn3uc917jHn\nmu96iTEGpZRSytMEOB2AUkoplRZNUEoppTySJiillFIeSROUUkopj6QJSimllEe6xukAcpOI6JRE\npZTyIMYYye5rfa4FZYzRWxq3119/3fEYPPmm18d/rs306YYWLfT6uOOWUz6XoJRSKiPjxsEzzzgd\nhcoMTVBKKb+xZg2cPAlt2jgdicoMTVB+onnz5k6H4NH0+qTPl67N2LHw7LOQL1/undOXro+nkdzo\nJ/QUImJ86fMopXLPgQNQvz5EREBwsNPR+AcRweRgkoRPzeJLT1hYGPv373c6DOWQChUqEBER4XQY\nymHvvw/du2ty8iZ+0YJyZXEHIlKeQH/+6vx5CAuDDRvsf5V75LQFpWNQSimf9+mn0Ly5Jidv4xdd\nfEop/5WQYKeWf/KJ05GorNIWlFLKpy1YAMWKwR13OB2JyipNUEopnzZ2LPz3vyDZHglRTtEE5aPC\nw8Np0qRJ0v3g4GCdyab8zu+/w+7d8MgjTkeiskMTlMMqVqzI4sWLk+5/9dVXFC9enBUrVrB//34C\nAgJISEjI1rkl2Z+M586dI0xHiJWfeftt23rKn9/pSFR2aILyIOHh4fTr148ffvghqfUjHtwvER8f\n73QISqVr925YsgT+9S+nI1HZpQnKQ3z44Ye88MIL/Pzzz9x6661Zfn1kZCRt2rShaNGi3HbbbezZ\nsyfF8wEBAezdu5e1a9dSqlSpFOuC5s6dS7169QC7G/xbb71FlSpVCA0NpXPnzpw5cwYgqUU3bdo0\nKlSowN133w3AZ599RlhYGKGhobzxxhspWoWZOd9nn31GhQoVKFGiBKNGjUqKKyEhgVGjRlGlShWK\nFi1Ko0aNOHz4MAA7duzg3nvv5brrrqNmzZrMmjUry9dM+bbRo+Gpp6BwYacjUdnm9Hbsuby1u0lL\neo97grCwMPPwww+bkiVLmi1btqR4LiIiwgQEBJj4+PirnqdTp06mU6dOJiYmxmzdutWUKVPGNGnS\nJOn5gIAAs2fPHmOMMVWqVDG//vpr0nOPPPKIeeedd4wxxowdO9Y0btzYHDlyxFy+fNk89dRT5rHH\nHkuKR0RM9+7dTXR0tLl48aLZvn27KVy4sPntt99MbGysGThwoAkMDDSLFi3K9Pn+/e9/m0uXLpnN\nmzebAgUKmB07dhhjjHnnnXdM3bp1za5du4wxxvz5558mMjLSXLhwwZQrV86Eh4ebhIQEs3HjRhMa\nGmq2b9+e5rXx5J+/yhvHjhkTEmLM8eNOR+LfXL972f9Oz8mLPe2W3QQFuXPLjrCwMFO0aFHTrl27\nK57LbIKKj483+fPnNzt37kx6bMiQISkSlIgkJahXXnnF9OzZ0xhjzNmzZ821115rDh48aIwxpmbN\nmmbx4sVJrzty5IjJnz+/iY+PT4onIiIi6fnhw4ebLl26JN2Pjo5OkaAyc74jR44kPX/LLbeYmTNn\nGmOMqV69upk/f/4Vn3fmzJmmadOmKR7r06ePGT58eJrXRxOU/xkyxJj//MfpKFROE5R28ZFb6Sn7\n7z9lyhR27txJr169svX6kydPEh8fT9myZZMeq1ChQrrHd+nShblz5xIbG8ucOXNo2LBh0mv3799P\n+/btKV68OMWLF6dWrVrkz5+f48ePJ70++fscOXKEcuXKJd0vVKgQ1113XdL9zJzvhhtuSPr/oKAg\nzp8/D8DBgwepVKnSFfHv37+fNWvWJJ0zJCSE6dOnc+zYsUxdL+Xbzp2DDz6AAQOcjkTllCYoD1Ci\nRAkWLVrEihUrePrpp7P8+tDQUK655hoOHjyY9NiBAwfSPb5mzZpUqFCBhQsXMmPGDLp06ZL0XPny\n5fnhhx+IjIwkMjKS06dPc+HCBUqVKpV0TPKJG6VKleLQoUNJ92NiYvjnn3+ydL70lCtX7oqxtMTH\nmzdvnuKcZ8+eZeLEiVc9p/J9U6fC3XdD5cpOR6JyShOUhyhZsiSLFy/mp59+on///kmPG2O4ePEi\nly5dSrqZVM21gIAAOnTowNChQ4mJiWH79u2Eh4dn+H5dunRh/PjxrFixgkeSLRLp06cPQ4YMSUpw\nJ0+eZN68eSniSa5jx47Mnz+fNWvWEBsby+uvv57i+ayeL7nevXvz6quvsnv3bgC2bNnC6dOnefDB\nB9m5cydffPEFcXFxxMbGsmHDBnbs2JHhZ1a+7/JluzD3xRedjkTlBk1QDkveGilbtiyLFi1i9uzZ\nvPzyy0nPBwcHExQURKFChQgKCmLJkiVXnGfChAmcO3eOUqVK0bNnT3r27Jnu+wB07tyZZcuWcffd\nd1O8ePGkx5977jnatm3LvffeS9GiRbn99ttZt25duuepVasWEyZMoFOnTpQuXZqiRYtSokQJChQo\nkK3zJb/fv39/Hn300aTX9u7dm5iYGAoXLszPP//MV199RenSpSldujSDBg3i8uXLGV9s5fNmzIDq\n1aFhQ6cjUblBy22oXHXhwgWKFSvG7t27MxwHcyf9+fuHhASoUwfeew9atnQ6GgVabkN5gO+//56Y\nmBguXLjAgAEDqFu3rsckJ+U/5s6FoCC45x6nI1G5RROUyrHvvvuO0qVLU7ZsWfbs2cNXX33ldEjK\nzxgDI0bAa6/pprC+RLv4lM/Tn7/vmzcPXn8d/vhDE5Qn0S4+pZRfS2w9vfKKJidfowlKKeXVfvoJ\nYmKgfXunI1G5TROUUsprGQPDh9vWU4B+m/kc/ZEqpbzW4sUQGakFCX2VJiillNcaMQKGDIF8+ZyO\nROUFTVBKKa+0fDkcOgTJtpJUPsaxBCUiISIyV0TOi8g+EXksneMCRWSKiBwTkVMi8p2IXH2nUS8R\nFhZGwYIFiYyMTPH4TTfdREBAAAcOHODw4cN07NiR0NBQQkJCqFevHp999hnwv6J/RYoUoUiRIgQH\nB1OkSBEt4Kd83ogRMHgwXHON05GovOLkj3YScBEIBRoAC0RkkzHmr1THPQ/cCtwInAU+AiYAHd0Y\na54RESpWrMiMGTPo27cvAFu3buXixYtJ+9J169aN+vXrc/DgQQIDA9myZUuK0hIiQlRUlEeXh1cq\nN61YAXv2QLduTkei8pIjLSgRCQI6AK8YY2KMMauAeUBa/9zCgJ+MMaeMMZeBr4DabgvWDbp165Zi\n9/Hw8HC6d+8O2N2+169fT/fu3SlYsCABAQHUq1eP++67L8U5dCGq8hfG2Fl7r70GgYFOR6PSZAxk\no3RQak518VUD4owxyYv9bCbtxPMxcKeIlHIltq7AQjfE6Da33XYb586d4++//yYhIYGvv/6axx9/\nHLCto8aNG/P0008zc+bMFDWfktMEpfzFokVw7Bi4fkWUJ3rvPVi1KsencSpBFQaiUj0WBQSncexO\n4ABwGDgD1ABG5Go0Q4faJeipb0OHZv749I7NpMRW1C+//EKNGjUoXbp0UtKZNWsWTZs25Y033qBS\npUo0aNCADRs2JL3WGENoaGhSddnixYvz999/5ygepTxRYutp2DAde/JYP/wA775r95/KIUf24hOR\nm4CVxpjCyR7rDzQzxrRNdeyXQEGgJxANvAQ8aIy5LY3zmuQF85o3b07z5s09ei+2ihUr8vHHH1Ol\nShWaNm1K48aNefDBB+ncuTP58+cnIiKC8uXLJx0fGRnJgAED+PXXXzl48CD79++nUqVKxMXF6RhU\nOjz556+y5vvv7cSIzZt1Ya4nWvrppyzt2xc6d4Zy5Rg2bFiO9uJzKkEFAZFA7cRuPhEJBw4bY4ak\nOnYLMMQYM991vyhwGrjeGBOZ6liv2yw2MUG1aNGCu+66i40bN3L06FECAwPTTFAA27Zto27dupw6\ndYqzZ89SqVIlYmNjCdDf2DR58s9fZV5CAtx8sx17atfO6WhUmtauhb174TE7KdsrN4s1xkQDc4Dh\nIhIkIncAbYDP0zh8PfCEiBQRkfxAX2wii0zjWK82bdo0Fi9eTKFChVI8PmjQILZt20Z8fDznzp1j\n0qRJVKlShZCQEMB28ekXsPJ1c+faVlPbtlc/Vjnk1luTklNucPJP7r5AEHAC+BJ4yhjzl4jcKSJn\nkx03ELgE7AKOA/cDPrMtZPJuuYoVK9KgQYMrnouOjqZ9+/aEhIRQpUoVDh48yLxk/bsiQkhISIp1\nUO+99577PoRSeSw+3racRozQHcv9idaDUj5Pf/7e78svYdIkWLlSE5Q3yWkXnyYo5fP05+/dYmOh\nVi344ANo0cLpaFQKp07B9den+7RXjkEppVRmffQRVKyoycnj7NwJN94I6azNzA26kkAp5bHOn7f1\nnhb61NJ8HxAZCQ89BG+8AeXK5dnbaAtKKeWxxoyxLaf69Z2ORCW5fBkefhhat4bevfP0rXQMSvk8\n/fl7pxMn7NjT+vW2i095AGOgZ0/bgpoz56qFuHI6BqVdfEopjzRihN1vT5OTBzl+HP75B2bMcEuV\nSL9oQYWFhbF//34HIlKeoEKFCkRERDgdhsqC3bvhttvgr78gNNTpaFR26TTzZNJLUEop79K5M9Sp\nAy+/7HQkKic0QSWjCUop77d+vd1rb+dOuPZap6NROaHroJRSPsMYGDQIXn9dk5NHiI119O01QSml\nPMb338PRo3aimHJYbKydSp4LdZ2yS2fxKaU8wuXLMHCgLcaqxQgdZgw88wzkzw+tWjkWhv4zUEp5\nhMmT7ZTyBx5wOhLFmDGwZo3dndfBvxZ0koRSynGRkVCjBixZArVrOx2Nn5szB559FlavzvE2RjqL\nLxlNUEp5p+ees0MekyY5HYmfi4uDJk3sDyIX9pfSBJWMJiilvM+OHfY7cft2XZTrERISbOniXKDT\nzJVSXm3gQDu1XJOTh8il5JQbdJKEUsoxv/xiW1CzZzsdifJEnpMqlVJ+JS4O+veHd9+FAgWcjsZP\nJSTYKeUeShOUUsoRkyZBiRJ2WyPlAGPg6adhyhSnI0mXTpJQSrnd8eO2WviyZbbmk3LAm2/C11/b\nH0KRInnyFloPSinldQYNgu7dNTk55osv4IMP4Lff8iw55QZNUEopt1q9Gn7+2dZ6Ug5YvBgGDLD/\nLV3a6WgypGNQSim3iY+3W7y9845H/+Huu4yBN96wXXtesGWHjkEppdxmyhSYPt0Oe0i2RyZUjuTi\nQtyr0Z0kktEEpZTnOnXKjjn9+ivUret0NModNEElowlKKc/Vpw8ULAjjxjkdiXIXncWnlPJ4GzbY\nunc6McLNYmNtuQwv7U/VSRJKqTwVH2/Xg44aBcWKOR2NH0lIgCefhPffdzqSbNMEpZTKU5MmQaFC\n9rtSuYkxdir5gQPQu7fT0WSbdvEppfLMoUMwbJgtzOqlvUze6e237WyU5cvtXwdeShOUUirPPPcc\n9O1rq+UqN/nkEzuff9UqCAlxOpoc0QSllMoT8+bB1q3w5ZdOR+JH4uNtyfYff4QyZZyOJsd0mrlS\nKtedO2c3KggPh7vucjoa5RRdB5WMJiilPEP//hAZCZ9+6nQkykm6Dkop5VH++MN2623b5nQkytvp\nNHOlVK6Ji4N//9tuBnv99U5H4wdOnLDjTj5KE5RSKtdMmADBwfDEE05H4gdOnICmTWHhQqcjyTPa\nxaeUyhW7d8PIkbBmja55ynNnzsB990GnTvDQQ05Hk2cca0GJSIiIzBWR8yKyT0Qey+DYBiKyTETO\nichREennzliVUhlLSLAbFrz8MlSp4nQ0Pu7CBWjd2raehg51Opo85WQLahJwEQgFGgALRGSTMSbF\ndpIich3wA/Ac8A1QACjr5liVUhn44AO4dAmefdbpSHzcpUvQvj1UrQpjx/p8U9WRaeYiEgScBmoZ\nY/a4HvsMOGSMGZLq2JFAWWNM90ycV6eZK+Vm+/dDw4awYgXUrOl0ND7u7FkYMwZeecXuUu7hcjrN\n3KkuvmpAXGJyctkMpFWD+DbgtIisEpHjIvKdiJRzS5RKqQwZA//6l92XVJOTGxQpYrv1vCA55Qan\nPmVhICrVY1FAcBrHlgXqA/cAW4F3gRnAnWmdeGiyPtnmzZvTvHnzHAerlErbtGl2Qe4LLzgdifIE\nS5cuZenSpbl2Pqe6+G4CVhpjCid7rD/QzBjTNtWxm4DfjTG9XPeLA6eAosaYc6mO1S4+pdzk8GG4\n6SZYtEhLuKu0eWsX307gGhGpnOyxekBaa8//BFJnHQP49uigUh7MGHjqKbtTuSanPPTNN3D5stNR\nOMaRBGWMiQbmAMNFJEhE7gDaAJ+ncfgnQHsRqSsi+YFXsa2vs+6LWCmV3Gef2ckRQ4Zc/ViVTe+8\nYydDRKUeDfEfTu4k0RcIAk4AXwJPGWP+EpE7RSQp+RhjlgBDgIXAMaAS0MWBeJVS2MQ0cCB8/jkE\nBjodjY8aNw4+/ND2n4aGOh2NY3Q3c6VUpiUkwD33wL33wqBBTkfjo6ZMgbfegmXLoEIFp6PJEd3N\nXCnlNuPH27WiOmsvj8yfD6NGwdKlXp+ccoO2oJRSmbJ9u91dZ+1aqFz56serbLhwAY4fh0qVnI4k\nV2jBwmQ0QSmVN2JjoXFjuyi3Tx+no1HewlunmSulvMgbb0CJErbWk1LuomNQSqkMrVtnx+03bfL5\nvUmVh9EWlFIqXRcu2OKDEyZAqVJOR+NjFi6Enj2djsKjaQtKKZWu55+HW2+FRx91OhIf8/33NjnN\nn+90JB5NE5RSKk3ffANLlsDGjU5H4mPmz4devWySuuUWp6PxaDqLTyl1hQMH4OabYcECaNTI6Wh8\nyLx5tvSwn1xYncWnlMpV8fHw+OO2xpMffIe6jzHw9dd+k5xyg7aglFIpjBhhNzL45RcI0D9hVQ7o\nVkdKqVzz228wcSL8/rsmJ+W8TP8TFJESIjJORHaLyEUROSgiC0TkgZwGISIVRCRBRBrk9FxKqeyJ\nioKuXeGDD6BMGaejUSqTCUpEKgAbgZbAS0AdbAn2hcDkXIhDuLIooVLKTYyxu0Q88AC0bXv141Um\nLF0KMTFOR+HVMtuCmoxNIA2NMbONMbuMMX8bYyZiK+EiIuVEZK6InHXdZotI0t9hIlJWRL4VkX9E\n5IKIbBeRxNUVe13/3eBqSS3OrQ+olLq6yZPh779h9GinI/ER06fDY4/Z4lkq2646BiUiIcB9wBBj\nzBV/DhhjEss9fgdEA81d9ycCc4HEif6TgUCgGXAOqJ7sNLcA64B7sSXe/bfGsVJu9vvv8Prrdvyp\nUCGno/EBH31kL+ivv0KNGk5H49UyM0miCrYLbkd6B4hIS2y3XyVjzEHXY12A3SLSwhizGCgPfGOM\n2ep6WfI/LU66/htpjDmRxc+glMqmqCi7S8TEiVC1qtPR+IDx420zdOlSvaC5IDNdfJmZIlgDOJKY\nnACMMfuAI0At10PjgFdF5DcRGaETIpRyljF2t53779etjHLF7Nk2QS1frskpl2QmQe3Cjj/VzOCY\njCY5GABjzDQgDJgGVAV+E5HXMh2pUipXTZgAEREwZozTkfiI1q1h5UqthJuLMrVQV0QWAnWBasaY\n6FTPFQUaAT8AlY0xB1yPV8Imt3uMMUvSOOeLwLPGmLIiUgo4DNxqjFmf7Q+jC3WVypT16+336erV\nWh1X5R13bXX0NLaVtEFEOopINRGpLiL/ATYbY37FTm74UkQaiMjNwBfAhsTkJCLvich9IlJRRG4C\n7ge2uc5/AogB7nOttyqS3Q+klMrY6dPQqZOduafJSXmyTCUoY0wE0AD4BXgL2AwsAh4EEmtstsVO\ndljieu4I0D7Ve43HJqWfgGPAk67zxwP9gN7YltS32f5ESql0JSRAt27Qpg08/LDT0Xix+HiIjr76\ncSpHdC8+pfzI0KGweDEsWgT58zsdjZe6fNlm+cqVYdQop6PxaLoXn1IqU77/3i7R2bBBk1O2RUdD\nx472Ar6mc7zymm4HqZQf2LXLTimfNQtKlnQ6Gi91+jS0bAmhoXZKecGCTkfk8zRBKeXjzp+HDh1g\n2DBo3NjpaLzUqVPQrJmtgPvJJ3CNdj65g45BKeXDjLFbwhUqBNOmgWR7NMDPxcbaVlOnTnoRsyCn\nY1CaoJTyYWPGwJdf2vWjus+ecjedJKGUStOSJfDOO7B2rSYn5Z10DEopHxQRAV26wOef68472aI9\nMR5BE5RSPubcObsQd9AgO+lMZdHMmXZWiXKcdvEp5UMSEuDxx+HWW+HZZ52OxguNGwf/93+wYIHT\nkSg0QSnlU155xS7XmTVLJ5tlSUICvPSSTUyrVkH58k5HpNAEpZTPmD4dvvrKTooIDHQ6Gi8SGwtP\nPmnLs69cCcWLOx2RctExKKV8wLp18Pzz8N13dqMDlQX58kGjRvDLL5qcPIyug1LKyx0+bMecJk2y\nkyOU8hTuqgellPJA0dHQrh3066fJSfkeTVBpCAgIIF++fAQEBFxxy5cvHz179nQ6RH766ScCAgKI\n1po0fis+3s7Yq1kTXnzR6Wi8iPayeA3HEpSIhIjIXBE5LyL7ROSxqxyfX0R2iMiBvI7t2LFjHD16\nlGPHjjF16lREhOPHjyc9Pm7cuGydNy4uLtdiNMYkNp9z7ZzKu7z4op2x99FHOmMv0+bOhfvu0yTl\nJZxsQU0CLgKhwOPAZBGpmcHxL2Kr8Oa5EiVKJN2KFSsGQGhoaNJjwcHBAAwYMIBq1aoRFBREpUqV\neOWVV1IkocGDB9OoUSOmTp1KpUqVKFiwIAkJCZw7d44uXbpQuHBhypQpw5gxY2jZsiVPP/100msv\nXbrEgAEDKFu2LIULF6Zx48YsWbIEgL///ptWrVoBEBwcTL58+VK8Vvm+99+HhQthzhydsZcpxsDo\n0bYv9K23NKN7CUemmYtIENABqGWMiQFWicg8oBswJI3jKwJdgP7AVHfGmpFixYrxxRdfULJkSbZs\n2UKfPn249tprGTx4cNIxO3bsYN68ecydOzep27Bfv36sW7eOBQsWEBoayquvvsr69eupWrVq0uu6\ndOnCqVOnmDVrFiVLluS7776jVatWbNq0iWrVqjF9+nS6du3Kvn37KFSoEEFBQU5cAuWA77+3hVxX\nrYKQEKej8QJxcTYxrVoFq1dDuXJOR6Qyyxjj9htwE3Ah1WMDgO/SOX4+0AZoBhzI4Lwmt33zzTcm\nICAgU8e+9957pk6dOkn3Bw0aZAoVKmTOnDmT9FhkZKS55pprzLx585Iei4qKMsHBweY///mPMcaY\nbdu2mXz58pkTJ06kOP/9999vBgwYYIwx5scffzQBAQHmwoUL2f5syvts2GBMaKgxa9c6HYmXiIkx\n5v777S0qyulo/I7rOznbucKphbqFgahUj0UBwakPFJH2QD5jzDwRaeaO4DJrxowZvP/+++zdu5fz\n588TFxdHgQIFUhxTsWJFihYtmnR/165dJCQk0KhRo6THihQpQo0aNZLu//HHHyQkJFC5cuUUY0yX\nL1+moFbx9FsHDkDbtvDBB7ZunsqEAgWga1fo3FmLDHohp35i54EiqR4rApxL/oCrK/Bt4IHEh652\n4qFDhyb9f/PmzWnevHkOwkzfsmXLeOKJJxg1ahR33303RYsW5euvv2bEiBEpjrv22mtT3E9MOJJB\nH3hCQgKBgYFs2rTpiudSn0/5h6goaNUKBg6E9u2djsaLiNipjsotli5dytKlS3PtfE4lqJ3ANSJS\n2Rizx/VYPWBbquOqAhWAFWK/0QOBoiJyBLjNGHPFjL7kCSovrVq1iipVqvDCCy8kPbZv376rvq5a\ntWoEBASwbt06HnroIQDOnj3Ljh07uPnmmwFo0KABsbGxnDx5kltvvTXN8wS6Rsbj4+Nz+lGUh7t0\nyW6ufddd8NxzTkejVPpSNwqGDRuWo/M5MovPGBMNzAGGi0iQiNyBHWP6PNWhW4By2DGrekBv7Ey+\nesBB90V8pWrVqrFv3z5mzZrF3r17GT9+PHPmzLnq60JCQnj88cfp378/y5YtY9u2bfTq1Yt8+fIl\ntapuvPFGOnToQNeuXfn222+JiIhgw4YNvPPOOyxw7bIcFhYGwPz58zl16pSuh/JR8fHQrZudDPHe\nezr5LEPGwJkzTkehcpGT08z7AkHACeBL4CljzF8icqeInAUwxiQYY04k3oBIIMEYc9IkH5xxQMeO\nHenXrx99+/alfv36/Pbbb5luvU2YMIFbbrmF1q1bc++993LHHXdQu3btFONL06dPp0uXLgwYMIAa\nNWrQtm1b1q5dS3nXLssVK1bk5ZdfZsCAAZQsWZKBAwfmxcdUDjLGtphOnoQvvrBbxql0XL4MffpA\n795OR6Jyke7F5wEuXrxI2bJlGTFiBP/5z3+cDkd5iJEjbdmMZcsg2TwbldqpU/Dww/YiffklBF8x\n10o5RPfi80Lr16/n66+/Zu/evfzxxx907dqVuLg4Onbs6HRoykNMnQrTpsEPP2hyytCWLXZK4+23\n210iNDn5FJ136QBjDG+//Ta7du0iMDCQ+vXrs3LlSkK1ToICvv0WXn8dli+HUqWcjsaD7d8PLVrA\n2LE6U89HaRefUh5k+XLo2NG2nBo2dDoaL7BvH1Ss6HQUKh057eLTBKWUh9i8Ge691w6j3HOP09Eo\nlXM6BuVHevToQRst+uOTduyABx6AiRM1OSmVSFtQXuTcuXMYYyhSxG7Ccdddd1GnTh3Gjx/vcGQq\nJ/btg2bNYMQI6N7d6Wg81OrVdgLEjTc6HYnKAm1B+YjY2NirHhMcHJyUnJRvOHLEtpheekmTU7o+\n/NBuQnj0qNORKDfTBJWO5cuX07hxY4KDgylWrBiNGzdm+/bthIeHExwczPfff0/16tUpVKgQLVq0\nSLHN0d69e2nXrh2lSpWicOHCNGzYMGkHiEQVK1Zk2LBh9OrVK2l3CYDhw4cTFhZGwYIFKVWqFE8+\n+WTSa5IvtrgsAAAe70lEQVR38fXo0YNly5YxceLEpEq/ERERVK1alTFjxqR4r127dhEQEJDm3n7K\nOSdP2uT0r39B375OR+OBLl60F2fcOFi5Elq2dDoi5WaaoNIQHx9Pu3btaNq0KVu2bGHdunU899xz\n5HMt5b906RLDhw8nPDycNWvWEB8fT4cOHZJef/78eVq1asWiRYv4888/6dixIw8//DA7d+5M8T5j\nx46lZs2a/P7774waNYo5c+YwevRopkyZwu7du1mwYAG3pLNt9bhx42jcuDE9evTg+PHjHD16lPLl\ny9OrVy+mTZuW4thp06ZRv359brrpply+Uiq7zpyxhV3bt4dBg5yOxgMdOmT7Pc+cgTVroFo1pyNS\nTshJrQ5Pu5FL9aAiIyNNQECAWb58+RXPffrppyYgIMCsXr066bH9+/ebfPnymUWLFqV7zttuu82M\nHDky6X5YWJhp06ZNimPGjBljatSoYeLi4tI8x5NPPmkeeuihpPvNmzc3/fr1S3HMsWPHTGBgoFnr\nKhgUHx9vypQpYyZNmpTBJ1budP68MbffbsyzzxqTkOB0NB5q+nRj3npLL5CXI4f1oLQFlYaQkBC6\nd+/Ovffey4MPPsjYsWM5dOhQ0vMBAQEp6jmVL1+e0qVLs337dgCio6N58cUXqV27NsWLFyc4OJjf\nf/+dAwdSbr6euHt5okceeYSYmBjCwsLo3bs333zzDZcvX85S7DfccAOtW7dOakX98MMPREZG0qVL\nlyydR+WN6Gh46CGoXt2uL9XNX9Px2GN2YE4vkF/TBJWOadOmsW7dOpo1a8a8efOoXr06v/zyS6Ze\nO2DAAGbPns3IkSNZvnw5mzdvplGjRlckm9S1ncqWLcvOnTv58MMPKVq0KAMHDqRhw4bExMRkKfbe\nvXszc+ZMLl68yCeffEKHDh1SFE1UzkhMTmXK2K2MAvS3T6kM6a9IBurUqcMLL7zAkiVLaNasGeHh\n4YAtKLh+/fqk4w4cOMCRI0eoVasWYGtFPfHEE7Rr144bb7yR0qVLs2fPnjTfI7XAwEAeeOABRo8e\nzbp169i2bRurVq1K99i06kHdf//9FClShMmTJzN//nx69eqV1Y+ucllMjJ2IVrIkfPqp7kyegpaK\nUenQBJWGiIgIBg8ezOrVqzlw4ABLlizhzz//TEpA+fLl4/nnn2fNmjVs2rSJ7t27U6dOHVq0aAHY\nWlFz585l48aNbNmyhW7dunHp0qWrvm94eDgff/wxW7duJSIigmnTphEYGEjVqlXTPD4sLIx169ax\nf/9+/vnnn6RqvQEBAfTo0YPBgwdTtmxZ7rrrrly6Mio7EpNTaCiEh2tySuHrr6FGDVsyWKlUNEGl\nISgoiJ07d/Loo49SvXp1evToQbdu3XjppZcAKFiwIC+//DJPPPEEjRs3RkSYPXt20uvHjBlDiRIl\naNq0Ka1bt6Zx48Y0adIkxXukVfK9WLFifPzxxzRt2pQ6deowd+5c5s6dS4UKFdKMc+DAgQQGBlKr\nVi1KlCjBwYP/q+HYs2dPLl++TM+ePXPjkqhsunjRztQrXhw++wyu0e2ZrUuXoF8/GDzY7o6rXdAq\nDbqTRBaFh4fTr18/zp49m6fvk1Nr166lSZMm7N27l7Jlyzodjl9KTE7BwTB9uianJPv3wyOP2MG4\nTz6BYsWcjkjlEd1JQqVw+fJlDh06xGuvvUaHDh00OTnk4kVbQ69wYbv5qyYnl8uXbYmMzp1hzhxN\nTipDmqB8zIwZMwgLCyMyMpLRo0c7HY5fio6GNm3g2mttyyl/fqcj8iCBgfD779C/v04hV1elXXxK\n5aJz5+DBB6FCBVsRV1tOyp9pF58HGzp0qNMhKDc6c8ZuF1ejhp1KrslJqZzxuQSVxY0XlMoVp07Z\noZXbboMpU3QRLhcvwjPPwOefOx2J8mI+18UXGWkICXE6EuVPjh2zu5K3aQMjR+rQCn/9BZ062abk\nhx/qRAg/pl18qVy44HQEyp8cPAhNm9pJaaNG+XlyMgY++giaNLFrnGbO1OSkckQTVB7SMSjftmeP\nrQjRpw+88orT0XiAV1+F8eNh+XJbx8mvs7XKDZqglMqGTZtsy+nFF2HAAKej8RB9+sDateDaEkyp\nnPK5MahlywxNmzodifJly5bZjRAmTYKOHZ2ORinPldMxKJ+bCKstKJWXvvvO9l7NmAF33+10NEr5\nNu3iy0M6BuVbpk2Dp56ChQv9ODklJNhxpr59nY5E+QFtQSl1FcbAu+/C5Mm2e69aNacjcsihQ9Cj\nB5w/r+ublFtoCyoPaQvK+yUkwAsv2FIZK1f6cXKaORMaNrTTFlesgCpVnI5I+QGfa0GdP+90BMpX\nXLoEvXrB3r125nTx4k5H5JDwcHjzTViwAG6+2elolB/RFlQe0haU9zp9Gu67z1bDXbTIj5MT2CmL\nf/yhyUm5nSYopVLZtw9uvx0aNLAVyQsVcjoihwUF2ZtSbqYJKg9pC8r7rF8Pd9wBTz8NY8ZAvnxO\nR+Rmp087HYFSSTRBKeUybx60amVn6/Xr53Q0bnbypN1QsFs3pyNRKonPJShPmiShLSjv8f77do3T\nggXQtq3T0bjZN99A3bpQtizMmuV0NEol8blZfGfOOB2B8iZxcTBwIPz4I6xaBRUrOh2RGx05As8+\nC1u3wuzZduBNKQ/ic3vx1a1r2LzZ6UiUNzhzxvZqxcXZyRB+N1Nv7lw7O+/ll6FgQaejUT4op3vx\n+VyCKl3acPiw05EoT7drly0weM89djJE/vxOR6SU7/HagoUiEiIic0XkvIjsE5HH0jluoIhsEZGz\nIrJHRAZmdN5Tp+zWNJ5Ax6A806JFcOed8NxzMGGCJielPJWTkyQmAReBUOBxYLKI1Ezn2G5AMeAB\n4BkReTS9k15zjc7kU+mbNAm6doWvvrKTIvzC4sW20q1SXsaRLj4RCQJOA7WMMXtcj30GHDLGDLnK\na8cBGGOeS+M5U66cYflyCAvL/biV94qNtS2mpUth/nyoXNnpiNzg5Em7keDixXbufOvWTkek/Iy3\ndvFVA+ISk5PLZqB2Jl7bBNiW3pPXXw///JPD6JRPOXHCblu0fz+sXu0HySk+HiZOhNq17cyPbds0\nOSmv5NQ088JAVKrHooDgjF4kIsMAAT5J75gzZ4YydqzdbLl58+Y0b948p7Fm29ChQ3UcymHr1tmq\nt926wfDhfrIzRP/+sHmzbTndeKPT0Sg/snTpUpYuXZpr53MqQZ0HiqR6rAhwLr0XiMgz2LGqO40x\nsekdd+utQ2nVCrp0yZU4lRebOtXOoJ461c8W344YAcHBINnuWVEqW1I3CoYNG5aj8zmVoHYC14hI\n5WTdfPVIp+tORHoCLwJNjDFHMzpxaCgcP56rsWabtp6ccfGi3arot99s6aLq1Z2OyM2KpP7bTynv\n5MgYlDEmGpgDDBeRIBG5A2gDXFGmU0S6AiOBlsaY/Vc7d9my6DooP3bwIDRtahfhrlnj48lp1So7\nsKaUj3JymnlfIAg4AXwJPGWM+UtE7hSRs8mOGwEUB9aLyDnXeqhJ6Z20bFlbmdoTaAvKvRYvhltu\ngUcftTtDBGc4ounFIiKgUye7DYYmKOXDHNuLzxhzGmifxuMrSTY+ZYyplJXzelKCUu4RHw8jR9qZ\n1F9+CS1aOB1RHjl71la2/fBDO2f+k0+0TpPyaT63WawnJShtQeW9Y8fswtuEBPj9dyhd2umI8sil\nS1CvHjRrBn/+CWXKOB2RUnnO5/bii4kxFCliB8oDfK6YiEpu0SJ44gno3Rtee80PppAfOwYlSzod\nhVKZ5q0LdfNMwYJQrJhnzOTTFlTeiI+H11+3a5vCw2HYMD9ITqDJSfkdn0tQYGv67N3rdBQqLxw9\nancgX7nSVoq45x6nI8plBw7Y7dWVUr6ZoKpWhd27nY5CW1C5bcECaNAAmjeHn3/2sQbFP//Yyon1\n60NkpG0mKuXnfG6SBNgEtWuX01Go3BIdbfc8XbAAZs6065x8RnQ0jBtnW00dO9rqtqVKOR2VUh7B\nZ1tQnpCgtAWVcxs3QsOGduHtpk0+lpzA1v/YuNEuup08WZOTUsloC0p5pIQE+L//g3ffhffes1PJ\nfdKAAbpnnlLp8Llp5sYYzp+HG26AqChbwFB5l4MH7fTx+Hj47DMfqe0VF2enGmoyUn5Ep5mnoXBh\n21PiCRMlVOYZA198Ybv0WraEJUt8IDnFx9sPVbMmLF/udDRKeRWfTFAAdevaBfdO0jGozDt6FNq1\ng3fegR9/hCFDvHxtU1ycTUy1a8MHH9jtiZo1czoqpbyKTyeoTZucjkJdTWKrqV49+zPbsMFOJfdq\nO3bYgdCPPoIJE2zL6a67nI5KKa/jk2NQAAsX2kH2xYsdDkql6+hReOopu6j6009t155PuHzZZtrb\nb3c6EqUcpWNQ6bj9dli/HmLTrb2rnGKM3XX8ppugTh37Xe4zyQkgMFCTk1K5wGcTVLFiUKmSXWLi\nFB2DutL+/fDQQ/DWW3bh7RtvQIECTkeVDRER8N//2mmGSqk84bMJCuDOO23Jb+W8uDgYPdq2lBo3\ntqUxbr7Z6aiyYe1aWxHx5pttS8lni08p5TyfHYMCmDHDVladO9fBoBQbNsC//w0hITBlip0/4HVO\nnrTTDI8eheefhx49fLhkr1K5I6djUD6doA4dsntvHj+utaGccO4cvPoqfPWVnT7erZsXr1M1xs68\nuf9+L5//rpT76CSJDJQtC6GhdrKEE/x1DMoYu6lr7dp2D72tW+3OEF6TnBISrnxMBFq31uSklBv5\ndIICaN9eu/jcaetWOywzapRd3/Tpp3D99U5HlQmxsfDNN3a90v/9n9PRKKXw8S4+sOMfXbvatZNe\n8xe8F4qKgqFD7fTx116z65u8Yh/Eo0ftLg8ffgiVK0PfvvavmsBApyNTyutpF99VNGwIMTHw119O\nR+KbEhJs2fUaNeD8edi2DZ55xkuS04EDUKuWTVI//mh3fOjUSZOTUh7C5xOUCHToYAfq3c3Xx6CW\nLoVbbrEljb77DqZOtWN+XqN8eTuTZsoUu2JYKeVRfD5BAfTuDR9/rLtK5JYdO6BtWzvTeuBAWL3a\nJiqPFBNjB8PSKxB27bXujUcplWl+kaBuvNHuKvH99+59X19rQZ04YYdomjSxt7/+gs6dPXAKf0KC\nrdXRqxeUKWMHxs6fdzoqpVQWedpXS5556inbFaWy7vx5GDnSDtfkz29bUAMHQsGCTkeWhkWLbNdd\n//424C1b4Icf7II4pZRX8ZsE1bEj/P03rFvnvvf09hZUTIzdnqhyZTv5Yc0aW379uuucjiwDN95o\nJzxs3GjLqZcp43RESqls8psEVaAADB5sp0KrjF26BBMnQpUqsGoV/PorTJ9u73uEvXvh/fftiuDU\nbrjBJimllNfz+XVQyV26BNWq2S/bO+5wY2BeIjbWThkfMcJ+xw8f7kFlMHbtsgtpv/kGDh60a5VG\nj4bChZ2OTCmVDt2LL5mrJSiwG8i+/bZdwOsVa3XcIDraznJ8912bwIcP97ByRk88Ab/8YtcLPPKI\n3aZef3hKeTxNUMlkJkEZAy1bQqtWdhw9Lw0dOtSjx6GiouzEkXHjbAmMwYM9dLr40aNQooTug6eU\nl9GdJLJIBCZPhjfftBO8/NHhwzYZVa4M27fbiW9z5zqUnPbvtz+Q1q1h0KC0jylVSpOTUn7I71pQ\nicLDbVffunX+M4yxdq2dhffTT3Z/wv79oWJFBwKJiLD1N375Bc6ehXvvtWV277sPihZ1ICClVF7Q\nLr5kspKgjIF//cvWivr2W9/9Az02FmbPtonp+HF49lno2dPhPHDkiB0MbNnSzsbwuJW+SqncoAkq\nmawkKLBf3q1aQVgYfPBB7n9POjkGtXu3nfjw6adQvbotAvvQQ25IxFFR8NtvsGKFreu+cKHvZn+l\nVIZ0DCoH8ueHOXPsAt5evbx/r76LF+0U+hYt7Cy82FhYvNhu6tquXR7niUGD7G4NZcrYvtN8+exC\nWR/6A0gp5V5+3YJKdOECPPqonW799dfetSN3QoJtrMyYYZcINWxoN8dt2zYPqkZERtr9jYKCrnxu\n+nTbFG3Y0K6KVkr5Pe3iSya7CQogPh5efdV+z37yiS2s6qmMseu4ZsywpdWvvx4ee8zeKlTIpTc5\netRuF/Tnn7B5M6xfb3eLnTcPmjfPpTdRSvkyr+3iE5EQEZkrIudFZJ+IPJbBsW+LyCkROSkib+dF\nPPny2TLl778P3bvbiQQnTuTsnLk5/nTxot1irm9fm4S6dLGVIn75xeaPQYOymZzi49N+fPx4O7Pi\n1Cl44AGYPx/OnNHkpJRyGyeX408CLgKhQANggYhsMsakqH0rIn2ANkBiRblfRWSPMebDvAjqwQeh\nWTNbtrxmTTs29d//2qU47mSMHRtbutROC1+82NbUe+ghm6hq1sxaCfulX39N84AA2Lkz5W3AALso\nKrU338y1z+INli5dSnNNvmnSa5MxvT55x5EWlIgEAR2AV4wxMcaYVcA8oFsahz8BjDbGHDXGHAVG\nA0+me/Lrr7fjIB062Klr2SilGxwMY8falkl0tK3a0K6dna597lzmz5OVFlRMjN0tfOJE21VXurRd\nFrRmjf0oe/bAypXw0ks2nqTkFBcHx47ZPr85c+zurmlY+tVXti5SVJQt5vT223aL8vQWx/qZpUuX\nOh2Cx9JrkzG9PnnHqRZUNSDOGLMn2WObgaZpHFvb9Vzy42qne+bt2+3uBIm36Oi0j1u+HIYNs7tf\nlyxpt9IJCbFNk6Y2jLJlbZffm2/ayRMffghPPgmNGtldFxo1grp1oVy5zNVGMsbmh/377d6nO3fa\nVtLGjbB7l+GmatHcXjuKjrWjGP1IFKWvjbK1LW6++cqTzZoFTz9tu91CQmywFSrYbJbWTrh16+pW\n7kopr+JUgioMRKV6LAoIzsSxUa7H0laihL01apRxBLVr29bD8eP/u+3da7c8b5oyTwYHQ68is+i1\nuhcJBQtyaWtBojcX4NykgvwQ8CDPx7xJ8eL2bQsXtrd6kUtos/VVChQoRHwCxF+ORy5fYlX+uwiv\n+gZVq0LVqnbf0379oN62GeT/T284URT+KGpX0hYtasd/0kpQ990HW7faFqOuM1JK+SBHZvGJyE3A\nSmNM4WSP9QeaGWPapjr2DHCPMWaD634DYIkx5oq9EETEd6YkKqWUD8jJLD6nWlA7gWtEpHKybr56\nwLY0jt3mem6D6/5N6RyXowuhlFLKszgyScIYEw3MAYaLSJCI3IGdqfd5God/BvQXkdIiUhroD3zi\nvmiVUko5wcmtjvoCQcAJ4EvgKWPMXyJyp4icTTzIGPMBMB/YAvwJzDfGTHUiYKWUUu7jUztJKKWU\n8h1+vVmsUkopz+VVCcrTtkfyNJm9PiIyUES2iMhZEdkjIgPdHau7ZeXfjuv4/CKyQ0QOuCtGJ2Xx\nd6uBiCwTkXMiclRE+rkzVnfLwu9VoIhMEZFjru+e70TEzXvQuJeI9BWR9SJyUUSmXeXY/7r+vZwW\nkY9EJP/Vzu9VCYqU2yM9DkwWkZqpD0q1PVJd4EER+bc7A3VIpq6PSzegGPAA8IyIPOqeEB2TlWsD\n8CJwzB2BeYjM/m5dB/wATAZCgCrAz26M0wmZ/bfzPHArcCNQGrtmc4K7gnTIYWAE8HFGB4nIfdjf\nqbuAMKAyMOyqZzfGeMUNO6HiElA52WOfAaPSOHYV0DvZ/Z7Ab05/Bk+5Pmm8dhwwzunP4CnXBqiI\nXcpwH3DA6fg96foAI4Fwp2P20GszCXgr2f1WwF9OfwY3XacRwLQMnv8SeCPZ/RbA0aud15taUOlt\nj5TWtkdZ2x7JN2Tl+qTWhHTWlvmIrF6b8cBg7F/N/iAr1+c24LSIrBKR465urHJuidIZWbk2HwN3\nikgp136jXYGFbojRG6T1nVxCREIyepE3Jai82x7JN2Tl+iQRkWGA4NtryzJ9bUSkPZDPGDPPHYF5\niKz82ymL3cC5H1AOiABm5GVwDsvKtdkJHMB2e50BamBbFirt72ThKt9P3pSgzgNFUj1WBEhrf/HU\nxxZxPebLsnJ9ABCRZ7B96q2MMV5e8D5Dmbo2rr9638Z++YL9BfIHWfm3EwPMNcb8YYy5jB1HuF1E\nMvyi8WJZuTZTgALYsblrgbnAj3kanfdI6zvZkMH3E3hXgkraHinZY1fbHilRutsj+ZCsXB9EpCd2\n0LKFsWVMfFlmr01VoAKwQkSOArOB0iJyRETKuydUR2Tl386f2C+W5Ay+m8yzcm3qAp8aY6Jcf/BN\nAG4RkeJuiNPTpfWdfNwYczrDVzk9uJbFgbjp2MG2IOAO4DRQM43j+rguSGnXbSvwL6fj96Dr0xU4\nClR3OmZPujbYP9hKJLu1Bw5hZ2+J05/B6evjOu4u4B/sl3F+YCywzOn4PeTaTANmYVsH+YEhwEGn\n48/ja5MPKAiMwk4eKYDtIk993H3AEaAmtoW5CBh51fM7/QGzeDFCsM3m89i+706ux+8EzqY69i3X\nL9Ip4E2nY/ek6wPsxc5MOottYp8FJjkdvydcm1SvaYYfzOLL6vXB/gF4yPX79R1Qxun4PeHaAMWB\nL4DjQCSwHLjZ6fjz+Nq8DiQA8clur2HHJ88BZZMd+zx26cYZ4CMg/9XOr1sdKaWU8kjeNAallFLK\nj2iCUkop5ZE0QSmllPJImqCUUkp5JE1QSimlPJImKKWUUh5JE5RSSimPpAlKKTcQkWYikuCN296I\nyBIRGe90HMr/aIJSPkVEbhKROBFZkY3Xvi4iW/IiLhdvXRXfHlt+BABXVdn+Dsaj/IQmKOVr/gVM\nBG4UkerZeL23JpFsyUzZbWPMGWPMBXfEo1RymqCUzxCRgkAXYCrwDdA7jWNKiciXInJKRC6IyB+u\n7rfu2H3Faru64uJF5AnXaxJEpEOq86RoRYjIf0Vks4icF5FDIjJVRIpmMf4OrnNEi8g/rq61UNdz\nr4vIFhHpJSL7XcfMdZVgT3z9zSLyk4icFJEoEVkhIreleo8EEXlaRGaLyHlgpIhcIyLjReSwiFx0\nnX9UstckdfGJyBLsju/vJrtOQa73S32NWorI5cTPoFRWaYJSvuQRIMIYsxW7aecTIpIv8UlXvafl\nQHmgLXAjMNz19FfAaOBv4AagFDAzC+8dDzwH1AIeAxphK/NmiojcgC389wm20F0T4PNUh4Vhd6J/\nCLgbWx7k42TPB2N3lL7D9f4bgQVpjHu9BizAfv6JwLPY6/EoUAXohL0OaemA3Sh2GFASKGWMiXbF\n3jPVsT2AecaYkxl+eKXScY3TASiVi3phv6AxxiwTkQtAG+xO1GC/3EsAt5j/1aHZl/hiV4siLjtf\nqMaY5MnogIi8BHwLdM/kKUpjfx9nG2MOuh7bnuqYgkA3Y8xhV7x9sLWrKhtj9hhjliQ/WESeAzoC\n92NLRiT6yhgzLdlxFYCdxphVrocOAWvS+ZynRSQeOG+MOZHsqanAahEpZYw5KiLFgHbAw5n8/Epd\nQVtQyieISBVsyyF5+fHp2DGpRDcBf5qrFUnL3vu3EJGfReSgiJwF5gCBIlIyk6fYjK2Rs01EvhGR\np0Tk+lTHHE5MTi5rsaUOarpiCBWRD0TkbxE5gy2jEoptMSb3e6r7nwL1RWSniLwvIq1EJEsFCI0x\nv2PrriUm5K7YkhNaUVZlmyYo5St6Y/89HxSRWBGJBV4CWopIGdcx2a36mlbF2KTJBa5qu99ji2R2\nBBrwv+6uwEy9gTEJxph7gZbYZNUL2CUidbIQ52dAQ2xXY2NsBdPDacSQYsKDMWYjdlxpMPZzhgM/\nZ+F9E32E7dbD9d9PjNbzUTmgCUp5Pdc40xPAIOyXcvLbn/zvS/MPoG4Ga5EuYyuEpnYSOyaV+H43\nJL8P3IxNWP2NMWuNMbuBMmSD6/UjjDGNsBVIOyV7ukyyZAtwKzahJHYF3gFMMMb8aIz5C5uIkseZ\n0fteMMbMNsb0BVoDd7tapWlJ7zp94YqxL1Af2zJTKts0QSlf8CBwHfCRMWZ78ht2okMv13HTgRPA\ntyJyp4iEichDItLM9XwEUEFE6ovIdSKS2PJYDPQVkYYiUh87kSEm2fvvwv4u/dd1zsewrZjU0m3B\nicitIvKyayZeORFpC5TFtsoSXQTCRaSeiDQGJgPfG2P2up7fCTwuIjVFpBG2u/PSVa5d4gzEziJS\nw5WUugJR2LGotEQATUSkdPJZhMaYs9jZk6OxZeD3XO29lcqIJijlC3oCi9MZW5oFlBeRe1yzzZph\nu73mYcdMhvK/tU+zgYXYsaATQGfX4wOAvcAS4GvshICkCQLGmC3YhPRfbELp6XpNahl1d0VhW0Dz\nsYnmXWC4MSb5mNo+7GzD+cCvwG5SzpzrARQGNmCT8cfYZHK1GM4BL2DHtDYAdYEHjDEX03lNYknv\nPSS7Di4fY7sUP0apHNKS70p5ARF5HXjYGFPX6VgyIiKdsC270skSnFLZotPMlVI5JiKFsONdg4EP\nNTmp3KBdfEqp3PAisAM4BbzhcCzKR2gXn1JKKY+kLSillFIeSROUUkopj6QJSimllEfSBKWUUsoj\naYJSSinlkf4fSRC3hLflMqQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85a0412278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0, 1, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"Actual sparsity\")\n",
    "plt.ylabel(\"Cost\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def kl_divergence(p, q):\n",
    "    \"\"\"Kullback Leibler divergence\"\"\"\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 1000  # sparse codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "#activation = tf.nn.softplus # soft variant of ReLU\n",
    "activation = tf.nn.sigmoid\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_outputs), name=\"biases2\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "outputs = tf.matmul(hidden1, weights2) + biases2\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # batch mean\n",
    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
    "loss = mse + sparsity_weight * sparsity_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0793574 \tSparsity loss: 0.429409 \tTotal loss: 0.165239\n",
      "1 Train MSE: 0.0541968 \tSparsity loss: 0.129066 \tTotal loss: 0.08001\n",
      "2 Train MSE: 0.0488085 \tSparsity loss: 0.104644 \tTotal loss: 0.0697373\n",
      "3 Train MSE: 0.042403 \tSparsity loss: 0.0571133 \tTotal loss: 0.0538257\n",
      "4 Train MSE: 0.0387614 \tSparsity loss: 0.255613 \tTotal loss: 0.089884\n",
      "5 Train MSE: 0.034724 \tSparsity loss: 0.022195 \tTotal loss: 0.039163\n",
      "6 Train MSE: 0.031282 \tSparsity loss: 0.0513407 \tTotal loss: 0.0415502\n",
      "7 Train MSE: 0.0284899 \tSparsity loss: 0.0547581 \tTotal loss: 0.0394415\n",
      "8 Train MSE: 0.0269682 \tSparsity loss: 0.0823977 \tTotal loss: 0.0434477\n",
      "9 Train MSE: 0.0254676 \tSparsity loss: 0.0968377 \tTotal loss: 0.0448352\n",
      "10 Train MSE: 0.0229019 \tSparsity loss: 0.140114 \tTotal loss: 0.0509247\n",
      "11 Train MSE: 0.0229129 \tSparsity loss: 0.18078 \tTotal loss: 0.0590689\n",
      "12 Train MSE: 0.0205623 \tSparsity loss: 0.090069 \tTotal loss: 0.0385761\n",
      "13 Train MSE: 0.0194877 \tSparsity loss: 0.0646944 \tTotal loss: 0.0324266\n",
      "14 Train MSE: 0.0205489 \tSparsity loss: 0.0711002 \tTotal loss: 0.0347689\n",
      "15 Train MSE: 0.0171912 \tSparsity loss: 0.112708 \tTotal loss: 0.0397328\n",
      "16 Train MSE: 0.0173727 \tSparsity loss: 0.107462 \tTotal loss: 0.0388651\n",
      "17 Train MSE: 0.0161895 \tSparsity loss: 0.0829932 \tTotal loss: 0.0327882\n",
      "18 Train MSE: 0.0153571 \tSparsity loss: 0.0605893 \tTotal loss: 0.027475\n",
      "19 Train MSE: 0.0190343 \tSparsity loss: 0.380507 \tTotal loss: 0.0951357\n",
      "20 Train MSE: 0.0151051 \tSparsity loss: 0.0600717 \tTotal loss: 0.0271195\n",
      "21 Train MSE: 0.014637 \tSparsity loss: 0.0431151 \tTotal loss: 0.02326\n",
      "22 Train MSE: 0.0148286 \tSparsity loss: 0.233548 \tTotal loss: 0.0615383\n",
      "23 Train MSE: 0.0144666 \tSparsity loss: 0.141698 \tTotal loss: 0.0428061\n",
      "24 Train MSE: 0.0168366 \tSparsity loss: 0.19707 \tTotal loss: 0.0562505\n",
      "25 Train MSE: 0.0143715 \tSparsity loss: 0.11717 \tTotal loss: 0.0378055\n",
      "26 Train MSE: 0.0145037 \tSparsity loss: 0.0846578 \tTotal loss: 0.0314353\n",
      "27 Train MSE: 0.0147734 \tSparsity loss: 0.0989351 \tTotal loss: 0.0345605\n",
      "28 Train MSE: 0.0159841 \tSparsity loss: 0.0424816 \tTotal loss: 0.0244804\n",
      "29 Train MSE: 0.0147141 \tSparsity loss: 0.36503 \tTotal loss: 0.0877201\n",
      "30 Train MSE: 0.0160793 \tSparsity loss: 0.0542803 \tTotal loss: 0.0269353\n",
      "31 Train MSE: 0.0134212 \tSparsity loss: 0.0656383 \tTotal loss: 0.0265489\n",
      "32 Train MSE: 0.013091 \tSparsity loss: 0.266929 \tTotal loss: 0.0664767\n",
      "33 Train MSE: 0.0137133 \tSparsity loss: 0.09697 \tTotal loss: 0.0331073\n",
      "34 Train MSE: 0.0116499 \tSparsity loss: 0.0867991 \tTotal loss: 0.0290097\n",
      "35 Train MSE: 0.0133154 \tSparsity loss: 0.131212 \tTotal loss: 0.0395577\n",
      "36 Train MSE: 0.0133752 \tSparsity loss: 0.0620405 \tTotal loss: 0.0257833\n",
      "37 Train MSE: 0.0138332 \tSparsity loss: 0.0871629 \tTotal loss: 0.0312658\n",
      "38 Train MSE: 0.0135218 \tSparsity loss: 0.189354 \tTotal loss: 0.0513926\n",
      "39 Train MSE: 0.0129376 \tSparsity loss: 0.158046 \tTotal loss: 0.0445468\n",
      "40 Train MSE: 0.0130302 \tSparsity loss: 0.179793 \tTotal loss: 0.0489889\n",
      "41 Train MSE: 0.0142986 \tSparsity loss: 0.136741 \tTotal loss: 0.0416468\n",
      "42 Train MSE: 0.013083 \tSparsity loss: 0.10546 \tTotal loss: 0.034175\n",
      "43 Train MSE: 0.0126906 \tSparsity loss: 0.132878 \tTotal loss: 0.0392662\n",
      "44 Train MSE: 0.0137902 \tSparsity loss: 0.403354 \tTotal loss: 0.094461\n",
      "45 Train MSE: 0.0125835 \tSparsity loss: 0.206639 \tTotal loss: 0.0539112\n",
      "46 Train MSE: 0.0123034 \tSparsity loss: 0.0914052 \tTotal loss: 0.0305844\n",
      "47 Train MSE: 0.0110827 \tSparsity loss: 0.130939 \tTotal loss: 0.0372705\n",
      "48 Train MSE: 0.0119118 \tSparsity loss: 0.114499 \tTotal loss: 0.0348117\n",
      "49 Train MSE: 0.0117469 \tSparsity loss: 0.303567 \tTotal loss: 0.0724603\n",
      "50 Train MSE: 0.0123661 \tSparsity loss: 0.143802 \tTotal loss: 0.0411265\n",
      "51 Train MSE: 0.0114821 \tSparsity loss: 0.131843 \tTotal loss: 0.0378507\n",
      "52 Train MSE: 0.012937 \tSparsity loss: 0.242028 \tTotal loss: 0.0613425\n",
      "53 Train MSE: 0.0110802 \tSparsity loss: 0.141899 \tTotal loss: 0.03946\n",
      "54 Train MSE: 0.0126423 \tSparsity loss: 0.373559 \tTotal loss: 0.0873541\n",
      "55 Train MSE: 0.0125335 \tSparsity loss: 0.37624 \tTotal loss: 0.0877814\n",
      "56 Train MSE: 0.0112377 \tSparsity loss: 0.238165 \tTotal loss: 0.0588706\n",
      "57 Train MSE: 0.0114678 \tSparsity loss: 0.0975782 \tTotal loss: 0.0309834\n",
      "58 Train MSE: 0.010855 \tSparsity loss: 0.0980799 \tTotal loss: 0.030471\n",
      "59 Train MSE: 0.0117381 \tSparsity loss: 0.260745 \tTotal loss: 0.063887\n",
      "60 Train MSE: 0.0111806 \tSparsity loss: 0.207365 \tTotal loss: 0.0526536\n",
      "61 Train MSE: 0.0116346 \tSparsity loss: 0.130314 \tTotal loss: 0.0376975\n",
      "62 Train MSE: 0.0117247 \tSparsity loss: 0.11941 \tTotal loss: 0.0356068\n",
      "63 Train MSE: 0.0110201 \tSparsity loss: 0.263818 \tTotal loss: 0.0637838\n",
      "64 Train MSE: 0.0112741 \tSparsity loss: 0.136148 \tTotal loss: 0.0385038\n",
      "65 Train MSE: 0.0179305 \tSparsity loss: 1.01886 \tTotal loss: 0.221703\n",
      "66 Train MSE: 0.0224729 \tSparsity loss: 0.391452 \tTotal loss: 0.100763\n",
      "67 Train MSE: 0.0143544 \tSparsity loss: 0.178856 \tTotal loss: 0.0501255\n",
      "68 Train MSE: 0.0256441 \tSparsity loss: 0.53517 \tTotal loss: 0.132678\n",
      "69 Train MSE: 0.0114807 \tSparsity loss: 0.841381 \tTotal loss: 0.179757\n",
      "70 Train MSE: 0.0208025 \tSparsity loss: 0.28139 \tTotal loss: 0.0770805\n",
      "71 Train MSE: 0.0134129 \tSparsity loss: 0.562559 \tTotal loss: 0.125925\n",
      "72 Train MSE: 0.0151896 \tSparsity loss: 0.364237 \tTotal loss: 0.0880371\n",
      "73 Train MSE: 0.0175552 \tSparsity loss: 0.407572 \tTotal loss: 0.0990697\n",
      "74 Train MSE: 0.0231324 \tSparsity loss: 0.447183 \tTotal loss: 0.112569\n",
      "75 Train MSE: 0.0177917 \tSparsity loss: 0.48703 \tTotal loss: 0.115198\n",
      "76 Train MSE: 0.01583 \tSparsity loss: 0.200223 \tTotal loss: 0.0558745\n",
      "77 Train MSE: 0.0165376 \tSparsity loss: 0.31585 \tTotal loss: 0.0797076\n",
      "78 Train MSE: 0.0293659 \tSparsity loss: 0.144718 \tTotal loss: 0.0583096\n",
      "79 Train MSE: 0.0181668 \tSparsity loss: 0.28298 \tTotal loss: 0.0747628\n",
      "80 Train MSE: 0.0265659 \tSparsity loss: 0.493911 \tTotal loss: 0.125348\n",
      "81 Train MSE: 0.0226866 \tSparsity loss: 0.287003 \tTotal loss: 0.0800872\n",
      "82 Train MSE: 0.0190752 \tSparsity loss: 0.310685 \tTotal loss: 0.0812121\n",
      "83 Train MSE: 0.0162453 \tSparsity loss: 0.158039 \tTotal loss: 0.0478531\n",
      "84 Train MSE: 0.0149467 \tSparsity loss: 0.978434 \tTotal loss: 0.210633\n",
      "85 Train MSE: 0.0292528 \tSparsity loss: 0.565522 \tTotal loss: 0.142357\n",
      "86 Train MSE: 0.012644 \tSparsity loss: 0.0845427 \tTotal loss: 0.0295526\n",
      "87 Train MSE: 0.0150749 \tSparsity loss: 1.45733 \tTotal loss: 0.306541\n",
      "88 Train MSE: 0.0185209 \tSparsity loss: 1.13442 \tTotal loss: 0.245405\n",
      "89 Train MSE: 0.0318469 \tSparsity loss: 1.24017 \tTotal loss: 0.279881\n",
      "90 Train MSE: 0.0217135 \tSparsity loss: 0.137173 \tTotal loss: 0.0491481\n",
      "91 Train MSE: 0.013548 \tSparsity loss: 0.267628 \tTotal loss: 0.0670735\n",
      "92 Train MSE: 0.0173665 \tSparsity loss: 0.56907 \tTotal loss: 0.13118\n",
      "93 Train MSE: 0.0142269 \tSparsity loss: 0.375439 \tTotal loss: 0.0893147\n",
      "94 Train MSE: 0.0127459 \tSparsity loss: 0.142757 \tTotal loss: 0.0412972\n",
      "95 Train MSE: 0.0146607 \tSparsity loss: 0.171066 \tTotal loss: 0.0488739\n",
      "96 Train MSE: 0.0193444 \tSparsity loss: 0.22398 \tTotal loss: 0.0641405\n",
      "97 Train MSE: 0.0340869 \tSparsity loss: 0.518088 \tTotal loss: 0.137704\n",
      "98 Train MSE: 0.019557 \tSparsity loss: 0.360302 \tTotal loss: 0.0916175\n",
      "99 Train MSE: 0.0174739 \tSparsity loss: 1.32454 \tTotal loss: 0.282381\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_val, sparsity_loss_val, loss_val = sess.run([mse, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHOJJREFUeJzt3VuMnXXVx/FVKJ3pdDoznR6Z4dCDWMqx9QBUERpTMMZY\nFEKMciFBE4mJJpKogcRESQyX2huDRgOIJpgo1ZhUwdI0KQ1U00htLKljW9PDtPYwds5H4L16L3zX\nb/E+/+49u6yZ7+dyZe1nP3vvDoud/cv6z3nnnXcMAIAMLrvUNwAAQFUMLQBAGgwtAEAaDC0AQBoM\nLQBAGgwtAEAaDC0AQBoMLQBAGgwtAEAacxv4XKzeQCPMudQ3MBv09vbW9PccbeK57DL//9H12Nqj\nrvv2229XfnzUq+5NPdfY2Jh8fHNzs6vNmVP7P+GS96wez6eo9+Gtt96qfA9dXV3yxvimBQBIg6EF\nAEiDoQUASKORv2kBgJnFv6Oo345KfnOJfstRv6WU/K4WUb2qNn/+/MrXrMd9lVDPV/KeR/c1NTVV\nubcE37QAAGkwtAAAaTC0AABpMLQAAGkwtAAAaZAeBFAX0daIiYkJV1ObIMxqT7JFvaoepfTU6yhJ\nJZZs37j88ssrP5d6DePj47J33rx5rlayjSJK+al7a3TakW9aAIA0GFoAgDQYWgCANBhaAIA0CGIA\nqIvoh/eSFUa1/nhfj5VCKlgQhRiUquEMMx3EUDUzHWi54oorZO/k5KSrlYRUot7pOjqmBN+0AABp\nMLQAAGkwtAAAaTC0AABpMLQAAGmQHgRQFyXrh0ZGRmSvShpG11Upu2hdUpSyU6oe7Bg9n3qukpRf\ntOJK1YeGhmSvSjCq5zLTK5+i11uSNCxZ+VSCb1oAgDQYWgCANBhaAIA0GFoAgDQIYgCoi5If5Ftb\nW2WvWrcUrTVS9SjwoOpR2ECtS1I1M7O2trZK9xWtgVLvTXRGVlNTk6tFK7JUQGTuXP2f+5KzxtTn\nU3JWWPSel6zv4psWACANhhYAIA2GFgAgDYYWACANhhYAIA3SgwCKqXRZlB5UibN6UIm8KHmn6iUH\nMy5cuFD2qhVI6n2IknvqutG6pZLDJdU9jI6Oyl6V/ovWbKn3puTAyChpWIJvWgCANBhaAIA0GFoA\ngDQYWgCANGZVEOP11193ta1bt8re7u5uV4tWpnzxi190tc7OTtkb1YFMSlb/qBBC1KvCBoODg7L3\n73//e6WamQ43ROGKFStWuNp1110ne9V/E9Rqp+iMLBVsiN4btUpqbGxM9qr3LApBLF++3NWi/06V\nBFqUkjPXInzTAgCkwdACAKTB0AIApMHQAgCkwdACAKQxJ0pzTIOGPVFk7dq1rtbT0zMtz9Xe3i7r\nd9xxx7Q833RZuXKlqz3++OOy95prrpnmu6mkegwJF+3kyZPu77kkARYdijg8POxqhw4dkr27du1y\ntf3798tetcJoyZIlsletZjp37pzsVcnIK6+80tWi16tSjdGhlyrVGK1bUq+3o6ND9t5///2utn79\netmrDtOMVmeptGNJerCrq0v+g+KbFgAgDYYWACANhhYAIA2GFgAgjVm1xum3v/2tq73xxhuy98Yb\nb3S1aEXM3r17Xe13v/ud7H3ppZdcbdWqVbL36NGjsl5VdIaP+qH4+PHjla+rwhlmZt/+9rcrXwO5\nlYQu1I/v0Uoh1dvS0iJ7VfAnCkCpcEN0DwcPHnS1BQsWyF617kiFFfr6+uTjVfAkWi81NDTkatEZ\nWXv27HG1KDCxYcMGV7vhhhtkr/rc1X2Z6fdXhVzM4oCGwjctAEAaDC0AQBoMLQBAGgwtAEAaDC0A\nQBqzao1TI0WHs/3rX/9ytSg9eOTIkZruIUrqqPRgdA9nz551tW3btsne++67r+Dupg1rnBqgt7fX\n/T2XJAKjXrXuSK06ikRJQ/X3GP2NqqRfdA+tra2uphJ20RonVY/em8WLF7talGh+8sknXS1KDz71\n1FOutmnTJtmr3geVgDTTn3t0kK46SLK7u5s1TgCA3BhaAIA0GFoAgDQYWgCANGbVGqdGam5ulvXr\nr7++8jXWrVtXr9v5L2rtVHRe0O233+5q9957b93vCbmoH9nVj+lmZlNTU64WnRmlViBF4Yq2trbK\n96ACRU1NTbJXBR7Ua4iuoV5b9Hj134novtRatmj9mgpzbNy4UfauWbNG1hUVHIk+y+izUErWgvFN\nCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAG6cEZLFqv8tnPftbVotUxP/zhD10tWsWC2UOlB6MEmEoE\nRv/eVG+UQitKnIlrnD9/Xvaq1UxRGljdb9WamU4ERoe3qvVS27dvl70qgfixj31M9qq1btH9qtVX\n0XujPp96rA3kmxYAIA2GFgAgDYYWACANhhYAIA2CGDPYs88+K+unT592NbW6xszs2muvrectYYYo\nWdGjRKt/1I/30Qqk0dFRV4sCHip0EQWVVBBi4cKFsledWaceHwUQVP3YsWOy94UXXnC1F198Ufbe\ndtttlWpmOlg1ODgoe5Xos1SvreR9iPBNCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAG6cEZ4vDhw672\n2GOPVX78a6+9JusrVqy46HvC7BIlwErWLan038TEhOxVK4Wi9UNqrVG0LmnBggWV7stMv2Z1UGL0\n3qjrnjhxQvb+5je/cbXe3l7Ze/fdd7vakiVLZK96H6P3XKVGo883en8VDoEEAMxIDC0AQBoMLQBA\nGgwtAEAaBDFmiN///veuNjk5KXsffPBBV1u9enXd7wmzS/Rjesk6HyVa46RCAdG/+ZaWFleL1g8N\nDAy4WrTySfWq87hULaJCVWZmp06dcrVNmzbJ3rvuusvVonOv1GsrWbMVhVSieq34pgUASIOhBQBI\ng6EFAEiDoQUASIMgRjLRD83btm1zNbUFwMzsqaeecrXoh1dAUUGKKIih6tG/YxWuUBsmzPTGhWgj\nhrqH6Ewwdb5UX1+f7FXnTqnHR6GEgwcPulq0nUZttPjc5z4ne1euXCnrValzwszKQjUlnyUbMQAA\nMxJDCwCQBkMLAJAGQwsAkAZDCwCQBunBZH72s5/J+u7du13tC1/4guxlZRNqpRJjURqvJGmoUnYl\n143OgVLp2ChdqxKIUZpO1dvb211taGhIPn7nzp2u9sorr8jezZs3u9o999wje9Vr6+/vl70l7436\nfKL0oKqrs7vMdOIywjctAEAaDC0AQBoMLQBAGgwtAEAaBDHew9544w1X+9rXviZ7Ozo6XO3JJ5+s\n+z0BZjocUfKDfMkZTCUrfqL1UKoeXVeFKxYvXix7o8DC/7V9+3ZZV+vXouDJZz7zGVeLAgwq8BCF\nVNR7Hp29pdYwlYRqovstOXuLb1oAgDQYWgCANBhaAIA0GFoAgDQYWgCANEgPvgeMjo7K+uc//3lX\niw5Re+ihh1yNdU2YLiWJwJL0n7pulGRT64daWlpk79TUlKuV3G9ra6vsVenBN99809Veeukl+fgz\nZ8642sMPPyx7161b52pRIlCtooo+B/U+liRBo96SpGGUmJS9lTsBALjEGFoAgDQYWgCANBhaAIA0\nCGI0mPrx91Of+pTsPXTokKupH2PNzL73ve/VdmNAAfXjfck6n0jJOV0qdKHuK7qHKAA1Pj5e+brD\nw8Ou9txzz7na66+/Lh9/5513utoDDzwgexctWuRq6l7N9CqqaMXV3LnVx4DqVSEXM/25Re8552kB\nAGYkhhYAIA2GFgAgDYYWACANhhYAIA3Sgw3W19fnart27ar8+Oeff17WOzs7L/aWgGIqjRelBEtW\n9ERryqoqSbJF66FUki26rkr4qqTgrbfeKh//yU9+0tVWrFghe0tSfmq9U8lqppIUZtSrrluSEozw\nTQsAkAZDCwCQBkMLAJAGQwsAkAZBjGnS398v63fccUfla/ziF79wtQ0bNlz0PQHTqehMpKBXnQMV\nBSZUKCC67tjYmKtFoQB1RpY698rMbPv27a6mVhWtX79ePv7222+vfF8qpBKFX9R6pyjIUbKSq+Qc\nNRVeiT6fKCSi8E0LAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAbpwWnyzDPPyPqRI0cqX0MdEBeleoCZ\nQCXcRkZGZK9KGqqUoFnZgYRtbW2utm/fPtm7Y8cOV1Opue7ubvn41tZWV4sOdlSitVfqvxNRQk+t\nfIpSfrUe6Bkp+e8a37QAAGkwtAAAaTC0AABpMLQAAGkQxKiDnp4eV/vud7/b+BsB3oPUD/Ila4JK\nQgFRUEAFNKLeU6dOudqf/vQn2XvgwAFXU6uZbrjhBvl4FTyZnJyUvUp0lpWqR9dV72/03pQEJtRr\nq8d1+aYFAEiDoQUASIOhBQBIg6EFAEiDoQUASIP0YB3s3r3b1QYGBio/ft26dbIeHQYHzCbRqqKS\nNUHqIMlo5ZNa2bR3717Zq5J3N910k6uplVNmOtEX9ar3oWSNU6TksMYorVhV9JmRHgQAzEgMLQBA\nGgwtAEAaDC0AQBoEMRrsIx/5iKtFK2IIYuC9qmQ1U61nwJU8PgoxzJs3z9WilUIdHR2utmbNGtl7\n8803u9qWLVtcbenSpfLx6rVF96VeW7SaqWQdlrqHqLfqc0XXJYgBAJhVGFoAgDQYWgCANBhaAIA0\nGFoAgDTmlKxCqVHDngizWm1RNVTS29s7LX/PJf89Ur3q4EGzsoMV1Xqn6L5UKlGtOipZf1SyQmlo\naEj2NjU1VX6+kiSoSjaWJA1LdHV1yZvgmxYAIA2GFgAgDYYWACANhhYAII1GBjEAAKgJ37QAAGkw\ntAAAaTC0AABpMLQAAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABpMLQA\nAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABp\nMLQAAGkwtAAAaTC0AABpMLQAAGkwtAAAaTC0AABpzG3gc73TwOfC7DXnUt/AbHD8+HH393zZZfr/\ngd95x//pl/RGSnrV87399tuVHz9njv5npa6het966y35+CuuuMLVotel6tF9KVGvureot9bni95z\n9fl0dXXJJ+ObFgAgDYYWACANhhYAII1G/qYFYAaLfq+4/PLLXS36jSf6rauqkt9tSq4RXbfqa4te\nV0mvqtfj9y913ahXXbfkt8F6/JbJNy0AQBoMLQBAGgwtAEAaDC0AQBoMLQBAGqQHARQrSZyNj4+7\nmtoEYVZ9w8S71WtVkmBUqTeVCFTvgZnZ/PnzK9XM9OsdGhr6/27xXe8rUpIerMdWDtKDAIAZiaEF\nAEiDoQUASIOhBQBIgyBGHfzyl790teHhYdm7b98+V/vJT35S+bm+853vyPrHP/5xV9u0aVPl6wK1\nitb5zJs3z9Wm64f+yclJWVeBhQsXLsje/v5+Vztz5kzl605NTbmaeg/MzNauXetqq1atkr0dHR2u\n1tzcLHvHxsZcLQqYTFegpWTlE0EMAMCMxNACAKTB0AIApMHQAgCkwdACAKRBerDAV7/6VVn/8Y9/\nXNN1S9bGfP/735f1bdu2udqrr74qe9vb2ys/H1BVlABT/76jlN/cuf4/SdF1VeotWlU0MDDgagcO\nHJC9hw8fdrWjR4/K3iNHjriaSvl1d3fLx58/f97VohVXixcvdrXW1lbZqw6njKj3LHofVb0ea7ZK\nevmmBQBIg6EFAEiDoQUASIOhBQBIgyBGQIUuag1cmJlt2LDB1R544AHZ29PT42rPPfec7D148KCr\n/frXv5a9X/rSl97tFoGLEv2Yrlb3RGEDdY0oVKCuOzg4KHt37Njhai+++KLsVSvYVAjCzOyaa65x\ntRtvvNHVosCECohE93X27FlXu+uuu2SvOr9LrXYy0+/vggULZK9aUVUUoigInYXXqPkKAAA0CEML\nAJAGQwsAkAZDCwCQBkMLAJDGrE8PHjt2TNZ/+tOfVr7Ghz/8YVf74x//KHtbWlpcLTogTq1M+ec/\n/yl79+zZ42rnzp2TvUCtSlYolaTLVJItOjhQJQV37twpe1944QVX+/e//y17P/3pT7vabbfdJntv\nvvlmV1u6dKmrnTx5Uj7+5ZdfdrVDhw7JXvXfiVtuuUX2Llu2zNWiZKVKBM6fP1/2qnq0kkuJektS\nhXzTAgCkwdACAKTB0AIApMHQAgCkMeuDGFFYQZ3howIXZnpFTLS2pcSzzz7ran/5y18qP/6+++6r\n+R4ARYUjoh/To/OwFBXEUEEBMx2kiP6eV61a5WqbN2+WvVu2bHG1aI2TWnekXsOJEyfk40+dOuVq\np0+flr1qZZRaA2VmdvXVV7uaCmeYmQ0NDbmaOtfMTH+W0ecbfW614psWACANhhYAIA2GFgAgDYYW\nACANhhYAII1Znx78wAc+IOsqhRStW4pWntRKrZKamJiYlucC3gtUKjFa46Tq6gBGM7OrrrrK1dav\nXy97VSIwWlGlqNVwvb29slf9PS9atEj2qoMz1cGQZmbnz593tei/Xyr9Fx0Yqa4RpQTVyqboQM+i\ngyQrdwIAcIkxtAAAaTC0AABpMLQAAGnM+iBGpL29vWHP9fzzz8v6/v37K1/j3nvvdbU1a9Zc9D0B\n70b9cB6t81E/vkfBhpKzmTo7O11t4cKFla8b9arXFv33YHh42NX+9re/uVp0RpZa2dTc3Cx7VUCj\nv79f9h49etTVOjo6ZK96bdHnU7K+S33uJYGLCN+0AABpMLQAAGkwtAAAaTC0AABpMLQAAGmQHmyw\nv/71r672la98RfaOj4+72pVXXil7t27d6mpq7QtQDyopWI9kmFoJFCXZVPovOrxQpQfb2tpkr1pV\npA5KNNOHOL755puudvLkSfl4dQ9Lly6VvevWrXO1JUuWyF71WUQr4ErWLan0YPT5qH8j0XVLDgrl\nmxYAIA2GFgAgDYYWACANhhYAIA2CGA322muvuZoKXEQeffRRWX//+99/0fcElKo1dBE9XtWjc6BU\nECMKBajVSOrcLDP993j8+HHZe+DAAVdTq5miAFVXV5errV27VvZ+6EMfcrXoLL8LFy64WvTeDAwM\nuFpTU5PsbWlpcbXovDO13ikKXESroGRv5U4AAC4xhhYAIA2GFgAgDYYWACANhhYAIA3Sg9PkkUce\nkfVf/epXla/xjW98w9W+9a1vXfQ9AfWiUn5RiqzWAyOjdWSqrtZAmemUXZRKPHbsmKvt3LlT9v7j\nH/+o9FzLly+Xj+/u7na1lStXyl613il6DUr0Po6OjrpalOZTn3H0WapeDoEEAMwqDC0AQBoMLQBA\nGgwtAEAaBDHqQJ2184c//EH2jo2NuVr0I+0TTzzhaiU/vALTpeT8I9Vb8vgo4KH+lkrCIOrxZmY9\nPT2u1tvbK3vVWVRXX321q1133XXy8bfeequrRSuf1Flhg4ODsre9vd3VWltbZW9fX5+rRYGWkZER\nWVdKzlyLPjeFb1oAgDQYWgCANBhaAIA0GFoAgDQYWgCANEgP1sGDDz7oamfOnKn8+K9//euy3tnZ\nedH3BDRalAwrSZGpgwqj9UMquRddVx3sqNY1mZnt27fP1dSqIzOzRYsWuZpat6RSgmb68FaVEjTT\nr1fVzHQaL1rNFD2fot7f6PNRCcSS9V0RvmkBANJgaAEA0mBoAQDSYGgBANIgiFFA/UBrZrZr167K\n17j//vtd7bHHHrvYWwLeM0qCGCWix6sf+qN7UEGK7du3y97du3e7WhR4WL9+faWaWu1kZtbc3Fz5\nudTrjUIUKnTR398ve0vOO1O90T2oUE2t/xbM+KYFAEiEoQUASIOhBQBIg6EFAEiDIEZA/XD7+OOP\ny96JiYnK1/3gBz/oapyRhWxKzj9S2w6ix5f8UK+uG219UBtqovO01D1Ef+NdXV2uprZcRGdZKQMD\nA5V7o/92qPdGbQUx0wGP6PNR2y9KQhtRL+dpAQBmJIYWACANhhYAIA2GFgAgDYYWACAN0oOBp59+\n2tVeeeWVyo9/5JFHZJ2VTZgJopSeopJhUYpMrQSKEnJqBdLg4KDsffXVV11tz549slel9zZu3Ch7\nN2/e7GqLFy92tWg108jIiKsNDw/L3oULF7pa9N40NTW5WvSZqeeL1mGpawwNDcletcYpuq663wjf\ntAAAaTC0AABpMLQAAGkwtAAAaRDECDzxxBM1Pf4HP/iBrLOyCTNBybqlkl71Q330N6OCCX19fbJ3\n7969rrZjxw7Zu2DBAlf7xCc+IXvb2tpcTQUQorOsqj6/WVlIRYlCKqoenZGlQhtnz56VvfPnz3e1\nZcuWyV7WOAEAZiSGFgAgDYYWACANhhYAIA2GFgAgDdKD0yRabVKy/qaEWoOiDoIz0+mm6IA4RR2Q\naWa2devWytdQovtVSU51GB0urWhFj6pHvSq1FqUP1Wok9W/bTCfyVq5cKXu7u7tdbcmSJbL3xIkT\nrnbq1ClXiw6RVK9Xpe6ietR77tw5V9u1a5fsVZ9F9HrVdaO/xVtuucXV1IorM72SK8I3LQBAGgwt\nAEAaDC0AQBoMLQBAGgQxpon6MXc6Pfroo67W1dUle0+fPu1qP/rRj+p+T/Wi3ssvf/nLl+BO8L9U\noCgKTKgf+qPQjQomRCt+xsbG3u0W/8uiRYsq1czMVq1a5Wo9PT2y9/Dhw66mAiLRvap76OjokL3q\nfbhw4YLsPXTokKv9+c9/lr0qkLJ69WrZqz6f66+/XvaqcFgUlGGNEwBgRmJoAQDSYGgBANJgaAEA\n0mBoAQDSID0YeOihh1ztmWeeuQR3Us3TTz89LddVaaEo+aU8/PDDsr5x48bK1/joRz9auReNoZKC\nUXpQJQ2jNU4qXRYlztR11cGQZmZ33nmnq/X29sreI0eOuJo67NHMbGRkxNXUoYjRmiJ1v+qaZmYD\nAwOuFqUH1dqqlpYW2bt06VJXU8k/M7Nrr73W1W666SbZq1KQ0dqp6N+DwjctAEAaDC0AQBoMLQBA\nGgwtAEAac6IfT6dBw55ouvz85z+X9eisnKr2798v67WuVvrmN78p6+973/sqX2PLli2utmzZsou+\npwao/osuLtrJkyfd33NJuCLqVet8ovOa5s2bV7lXhRui1Ux79uxxNXWOlJkOg6g1adHjp6amXE0F\nLsx0CKKzs1P2Ll++3NVWrFghe1VAI7quWjsVnZGlwhwl5+BdddVV8h8J37QAAGkwtAAAaTC0AABp\nMLQAAGkwtAAAaZAexExDerABTpw4UTk9qJQcBhj1qiSaWjtmptcoRf/tGx0ddbWhoSHZOz4+7mr/\n+c9/XC065FClD6PXq1YgRauZ1Bqn6PWq90zdV6RkrVvJdbu7u0kPAgByY2gBANJgaAEA0mBoAQDS\n4DwtAA0XhTZUPfrxXgUW1FokM73GKepV143uQa0qWr16tatF53z19/e7mgqCRPcQnXuleqOQSoko\nUKKUnLlWgm9aAIA0GFoAgDQYWgCANBhaAIA0GFoAgDRIDwIoVrKyqYRaCRQl90p6VeptbGxM9qr1\nUCWHF05OTrqaSi9G9xWtZlKi+ypJ+U3H483K/o0UrYK6mJsBAOBSYGgBANJgaAEA0mBoAQDSIIgB\noFjJOh71o34UmFDXLQlXRAGCktVMKhQQhQrUKijVG62MKllbpURnb9UqCkaUPF/JWWEl+KYFAEiD\noQUASIOhBQBIg6EFAEiDoQUASIP0IIBiKvUWJQpViixK45Uk50pWDanni1YgqdehVjNF1y1JBJYc\n1qjua2JiQvaWHPio3seSeyhZ11RyoGd4jcqdAABcYgwtAEAaDC0AQBoMLQBAGnNK1rEAAHAp8U0L\nAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAZDCwCQ\nBkMLAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAGQwsAkAZDCwCQBkMLAJAGQwsAkMb/\nAK9IFziV3M82AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85984fa630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer(mode=\"FAN_AVG\",\n",
    "                                                             uniform=True)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "weights1 = tf.Variable(initializer([n_inputs, n_hidden1]))\n",
    "weights2 = tf.Variable(initializer([n_hidden1, n_hidden2]))\n",
    "weights3_mean = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights3_log_sigma = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights4 = tf.Variable(initializer([n_hidden3, n_hidden4]))\n",
    "weights5 = tf.Variable(initializer([n_hidden4, n_hidden5]))\n",
    "weights6 = tf.Variable(initializer([n_hidden5, n_inputs]))\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros([n_hidden1], dtype=tf.float32))\n",
    "biases2 = tf.Variable(tf.zeros([n_hidden2], dtype=tf.float32))\n",
    "biases3_mean = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases3_log_sigma = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases4 = tf.Variable(tf.zeros([n_hidden4], dtype=tf.float32))\n",
    "biases5 = tf.Variable(tf.zeros([n_hidden5], dtype=tf.float32))\n",
    "biases6 = tf.Variable(tf.zeros([n_inputs], dtype=tf.float32))\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "\n",
    "hidden3_mean = tf.matmul(hidden2, weights3_mean) + biases3_mean\n",
    "hidden3_log_sigma = tf.matmul(hidden2, weights3_log_sigma) + biases3_log_sigma\n",
    "noise = tf.random_normal(tf.shape(hidden3_log_sigma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.sqrt(tf.exp(hidden3_log_sigma)) * noise\n",
    "\n",
    "hidden4 = activation(tf.matmul(hidden3, weights4) + biases4)\n",
    "hidden5 = activation(tf.matmul(hidden4, weights5) + biases5)\n",
    "logits = tf.matmul(hidden5, weights6) + biases6\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_log_sigma) + tf.square(hidden3_mean) - 1 - hidden3_log_sigma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "with tf.contrib.framework.arg_scope([fully_connected],\n",
    "                                    activation_fn=tf.nn.elu,\n",
    "                                    weights_initializer=initializer):\n",
    "    X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "    hidden1 = fully_connected(X, n_hidden1)\n",
    "    hidden2 = fully_connected(hidden1, n_hidden2)\n",
    "    hidden3_mean = fully_connected(hidden2, n_hidden3, activation_fn=None)\n",
    "    hidden3_gamma = fully_connected(hidden2, n_hidden3, activation_fn=None)\n",
    "    noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n",
    "    hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n",
    "    hidden4 = fully_connected(hidden3, n_hidden4)\n",
    "    hidden5 = fully_connected(hidden4, n_hidden5)\n",
    "    logits = fully_connected(hidden5, n_outputs, activation_fn=None)\n",
    "    outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train cost: 18681.3 \tReconstruction loss: 14916.0 \tLatent loss: 3765.3\n",
      "1 Train cost: 16837.2 \tReconstruction loss: 13169.1 \tLatent loss: 3668.03\n",
      "2 Train cost: 16473.7 \tReconstruction loss: 12709.5 \tLatent loss: 3764.26\n",
      "3 Train cost: 16427.3 \tReconstruction loss: 12693.5 \tLatent loss: 3733.71\n",
      "4 Train cost: 16322.2 \tReconstruction loss: 12443.6 \tLatent loss: 3878.57\n",
      "5 Train cost: 16089.5 \tReconstruction loss: 12343.0 \tLatent loss: 3746.47\n",
      "6 Train cost: 15852.3 \tReconstruction loss: 12113.3 \tLatent loss: 3738.98\n",
      "7 Train cost: 16351.6 \tReconstruction loss: 12485.8 \tLatent loss: 3865.77\n",
      "8 Train cost: 15883.2 \tReconstruction loss: 12135.3 \tLatent loss: 3747.87\n",
      "9 Train cost: 15548.8 \tReconstruction loss: 11862.2 \tLatent loss: 3686.54\n",
      "10 Train cost: 15927.1 \tReconstruction loss: 12166.0 \tLatent loss: 3761.05\n",
      "11 Train cost: 15431.3 \tReconstruction loss: 11705.0 \tLatent loss: 3726.36\n",
      "12 Train cost: 15416.7 \tReconstruction loss: 11582.3 \tLatent loss: 3834.4\n",
      "13 Train cost: 15952.8 \tReconstruction loss: 12100.7 \tLatent loss: 3852.07\n",
      "14 Train cost: 15030.0 \tReconstruction loss: 11370.5 \tLatent loss: 3659.41\n",
      "15 Train cost: 16014.0 \tReconstruction loss: 12168.8 \tLatent loss: 3845.25\n",
      "16 Train cost: 15119.2 \tReconstruction loss: 11359.5 \tLatent loss: 3759.71\n",
      "17 Train cost: 15485.2 \tReconstruction loss: 11724.3 \tLatent loss: 3760.88\n",
      "18 Train cost: 15660.7 \tReconstruction loss: 11923.0 \tLatent loss: 3737.76\n",
      "19 Train cost: 15140.1 \tReconstruction loss: 11433.5 \tLatent loss: 3706.6\n",
      "20% Train cost: 14649.5 \tReconstruction loss: 11003.5 \tLatent loss: 3646.0\n",
      "21 Train cost: 15123.6 \tReconstruction loss: 11323.8 \tLatent loss: 3799.79\n",
      "22 Train cost: 15559.9 \tReconstruction loss: 11697.5 \tLatent loss: 3862.37\n",
      "23 Train cost: 15286.7 \tReconstruction loss: 11621.3 \tLatent loss: 3665.34\n",
      "24 Train cost: 15056.4 \tReconstruction loss: 11286.9 \tLatent loss: 3769.5\n",
      "25 Train cost: 15055.9 \tReconstruction loss: 11362.3 \tLatent loss: 3693.59\n",
      "26 Train cost: 15262.4 \tReconstruction loss: 11440.8 \tLatent loss: 3821.58\n",
      "27 Train cost: 14764.6 \tReconstruction loss: 11083.2 \tLatent loss: 3681.31\n",
      "28 Train cost: 14401.4 \tReconstruction loss: 10794.6 \tLatent loss: 3606.83\n",
      "29 Train cost: 15019.6 \tReconstruction loss: 11285.5 \tLatent loss: 3734.15\n",
      "30 Train cost: 15039.5 \tReconstruction loss: 11377.6 \tLatent loss: 3661.85\n",
      "31 Train cost: 15255.3 \tReconstruction loss: 11477.2 \tLatent loss: 3778.07\n",
      "32 Train cost: 14924.6 \tReconstruction loss: 11089.7 \tLatent loss: 3834.92\n",
      "33 Train cost: 14970.6 \tReconstruction loss: 11192.9 \tLatent loss: 3777.66\n",
      "34 Train cost: 14655.6 \tReconstruction loss: 11002.7 \tLatent loss: 3652.89\n",
      "35 Train cost: 14648.6 \tReconstruction loss: 10957.4 \tLatent loss: 3691.18\n",
      "36 Train cost: 14887.6 \tReconstruction loss: 11174.4 \tLatent loss: 3713.21\n",
      "37 Train cost: 15093.4 \tReconstruction loss: 11285.3 \tLatent loss: 3808.08\n",
      "38 Train cost: 14361.6 \tReconstruction loss: 10650.0 \tLatent loss: 3711.66\n",
      "39 Train cost: 14880.0 \tReconstruction loss: 11147.7 \tLatent loss: 3732.28\n",
      "40 Train cost: 14502.7 \tReconstruction loss: 10890.7 \tLatent loss: 3612.03\n",
      "41 Train cost: 14589.5 \tReconstruction loss: 10894.8 \tLatent loss: 3694.75\n",
      "42 Train cost: 14840.2 \tReconstruction loss: 11139.7 \tLatent loss: 3700.59\n",
      "43 Train cost: 14817.3 \tReconstruction loss: 11115.9 \tLatent loss: 3701.33\n",
      "44 Train cost: 14382.4 \tReconstruction loss: 10651.9 \tLatent loss: 3730.52\n",
      "45 Train cost: 14367.3 \tReconstruction loss: 10742.7 \tLatent loss: 3624.6\n",
      "46 Train cost: 14984.9 \tReconstruction loss: 11259.7 \tLatent loss: 3725.18\n",
      "47 Train cost: 14434.4 \tReconstruction loss: 10758.7 \tLatent loss: 3675.71\n",
      "48 Train cost: 14973.5 \tReconstruction loss: 11268.3 \tLatent loss: 3705.18\n",
      "49 Train cost: 15031.1 \tReconstruction loss: 11319.0 \tLatent loss: 3712.11\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        cost_val, reconstruction_loss_val, latent_loss_val = sess.run([cost, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Encode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = hidden3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Decode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's plot the reconstructions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAHGCAYAAABaaN0mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGPZJREFUeJzt3XmsXVXZP/B9eztQoJS2UGaUqcgkFCjUggERJcSEGROk\npgYkEhMxJREFYoxBo4kYQRNFQECgf1QwOGDEKKCAIGGQGlTaQukAMrXI3N72tn3/+f3yvmutXc69\nT88595xzP5//np21z1417vNln+eutfs2b95cAcBwjRnpCQDQnQQIACECBIAQAQJAiAABIESAABAi\nQAAIESAAhAgQAEIECAAhY9t4LXumjC59Iz0BWsr9PLrU3s+eQAAIESAAhAgQAEIECAAhAgSAEAEC\nQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACEC\nBIAQAQJAiAABIESAABAiQAAIGTvSEwD4/zZv3lwc6+vrG4GZMBSeQAAIESAAhAgQAEJ6sgfyt7/9\nLamvvfbapN5jjz2KcyZOnJjU8+bNK8ZMnTr1fWtgeNavX5/Uf/nLX4oxt99+e1LPmjUrqY8++uji\nnN133z2pd9lll2LM2LHp19+YMeV/T0f6L3V9nEY2bdpUHNu4cWNS5/Pr7+8vzml3v8gTCAAhAgSA\nEAECQEhf5Pe6oLZd6MADD0zqpUuXNuVzJ0+enNSzZ89uyuc2ywc/+MGkvvzyy5N67733buNsKn+8\n39uacj+vW7cuqS+88MJizB/+8Iekfv311xt+bt7f2GeffYox+f2w4447FmPy3sTbb7+d1O+9915x\nzuDgYFJvt912xZgddtghqd94441izHPPPZfUn/70p5P629/+dnHOhAkTimNNUns/ewIBIESAABAi\nQAAIESAAhPTkQsJf/epXSf3UU08l9SGHHFKc889//jOpH3300WLMr3/966TOm3tVVTbrnn/++fef\nbI28AVhVVbXbbrsl9apVqxp+Tt5U/+pXvzrsuUArjR8/PqmvuuqqYsyKFSuSOl8oXPeHQHkje/ny\n5cWY/B6KbOSYL/arU9fYzu/xgYGBYszatWuTetGiRQ3PaWETvZYnEABCBAgAIQIEgJCeXEjYKvmi\np7rfVfMeyLJly4Z9nfx34aoqeyB1C6Nee+21pL7rrruS+vTTTx/2XLaChYS9rSX3c9330erVq5P6\nJz/5SVLXbUS45557JnXdhot5D2HlypXFmHzD1HwBYN6nqKqqGjduXFJv2LChGPPss88m9be+9a1i\nzAsvvJDUF110UVLnm8RWVX3/tEksJASgeQQIACECBIAQAQJAiCZ6l8gXNs6ZM6cYc8wxxyT1fffd\nl9T5WxdbTBO9t7XviyP7jsoX79Ut9suPDeVNfUMZ06y3DeZ/XDN37txizKuvvprUN9xwQ1KffPLJ\nw57LVtBEB6B5BAgAIQIEgJCe3Eyx27377rvFsTPPPDOp635Xveaaa5K6zT0PaIm8N9HCxXINDaVP\nkqvrmyxYsCCp87cPVlVVnXrqqUl94oknDvvareYJBIAQAQJAiAABIESAABCiid6BbrnlluLYyy+/\nnNTTpk0rxnzgAx9o1ZSAIcr/wOWZZ54pxvzoRz9K6ro/DLjiiisajhlpnkAACBEgAIQIEABCOu9H\ntVEoX0R06aWXNjznkUceKY7tuuuuTZsTEJO/ufTKK69sOOakk04qxsyYMaO5E2sBTyAAhAgQAEIE\nCAAheiAd4Le//W1Sb9iwoRhz7rnnJvW+++7b0jkBjdVtlHjPPfck9QMPPFCM2X777ZO6bu1Xf3//\n1k2uDTyBABAiQAAIESAAhAgQAEI00dusrkF+1113JfWECROKMd/5zneSuhsabNDr1q9fXxzLFw7W\nvWH0kksuSeqpU6c2d2Jt4gkEgBABAkCIAAEgRA+kzX72s58Vxx588MGk/sxnPlOMsXAQRl6+cPCG\nG24oxixZsiSpd95552LM/Pnzk7qvr68Js2s/TyAAhAgQAEIECAAhAgSAkL663SRbpG0X6iRPPfVU\nUs+aNasYM2nSpKR+/PHHizFd2ETvzq4gQzUq7+f//Oc/Sf2hD32oGLN27dqkvummm4oxc+fOTeou\naKLXTtATCAAhAgSAEAECQIiFhE2W//553nnnJfXGjRuLc84///yk7sJ+B/SkgYGBpP7CF76Q1HUb\nJZ566qlJXbcwuAt6HkPiCQSAEAECQIgAASBEgAAQYiHhVti0aVNx7OSTT07qP//5z0l90EEHFefk\nu/F269vJMr3RJWRLeu5+rvsu/MEPfpDUl112WVLvtNNOxTmLFy9O6smTJzdhdiPOQkIAmkeAABAi\nQAAI0QPZCqtXry6OTZ8+/X3Pqdso8cgjj2zanDqIHkhv67n7+a233iqOHXDAAUn9xhtvJPWNN95Y\nnNOFGyUOhR4IAM0jQAAIESAAhNhMcRjefPPNpJ49e3bDc26//faknjlzZlPnBMQMDg4m9SWXXFKM\nyfuce+65Z1KfeeaZxTk90vMYEk8gAIQIEABCBAgAIQIEgBBN9GG4+eabk3rZsmUNzzn++OOTejQ1\n2KCTrVy5Mql/85vfFGPyhdZf+tKXknq77bZr/sS6iCcQAEIECAAhAgSAEJspbsHSpUuLY7NmzUrq\nus3XcsuXL0/qvffee6vm1UU0e3pbV93PAwMDxbFzzjknqe++++5iTP4yqOeffz6pp0yZ0oTZdQWb\nKQLQPAIEgBABAkCIAAEgxELCLXjwwQeLY0Npmh900EFJPXHixKbNCYhZtGhRcez+++9veN4pp5yS\n1DvuuGPT5tQLPIEAECJAAAgRIACE6IFshTlz5hTH/vjHPya1HgiMvPxtolVVVZs2bUrqunv1rLPO\natmceoEnEABCBAgAIQIEgBABAkCI3XhpFbvx9raOvp/z77U1a9YUYxYuXJjURx99dDHmmGOOSepR\n/EZRu/EC0DwCBIAQAQJASDt7IAD0EE8gAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACE\nCBAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAA\nCBnbxmttbuO1GHl9Iz0BWsr9PLrU3s+eQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASCknetAALra\n5s2b37euqqrq6+t737qXeAIBIESAABAiQAAI0QPZggULFhTH3n333aR+4oknijHXX3/9+37u17/+\n9eLYSSedlNQnnnjiEGYIbMnGjRuT+rXXXivGPPnkk0n9pz/9qRjzyiuvJPW//vWvpJ40aVJxzooV\nK5J63rx5xZj58+cn9eTJk4sxY8Z0/n/fd/4MAehIAgSAEAECQIgAASCkr24hTIt09AtovvjFLyb1\nT3/607Zd++CDD07qhx56qBhT12TrcL27eoqq6qD7edOmTcWx1atXJ/XVV19djPnFL36R1PkfyVRV\nVa1duzap80WBebO+qqpqw4YNSb3PPvsUYxYuXJjUhx9+eDGmw5roXigFQPMIEABCBAgAIaNyIWHe\n76iqWM9j5syZxbGzzz47qZcuXZrUP//5z4tz8sVJd955ZzHmwgsvHPb8oBflPY8lS5YUY/IFu3WL\nBMeOTb/+tt9++2LMXnvtldS77LJLUj/yyCPvP9mqqqZOnVocy/si3brhoicQAEIECAAhAgSAEAEC\nQMioaKKvXLkyqW+88caG58yaNas4ds899yT1tttuW4wZP358UucLjZ599tninL/+9a9JnS+CAv5X\nvlDv6aefLsYsWrQoqXffffdizEUXXZTUp556ajEmb6yvWbMmqb/73e8W57zwwgtJ/bWvfa3h52qi\nAzCqCBAAQgQIACGjogeS9xTqNpDMex51C4/qFho1cssttyT1Y4891vCc008/fdjXgdFqxowZxbGh\n9DnzDQzrepp5byKv880Wq6qqpk2bltRHHnlkMaa/v7/h/LqBJxAAQgQIACECBICQUdEDyX+DrFtn\nka/fmDhxYlOunf8Wu379+qZ8LoxW+b160EEHFWMGBweTuq7vmX9O3Quc8vv1e9/7XlI/8MADxTn7\n779/Uu+www7FmG5d95HzBAJAiAABIESAABAiQAAIGRVN9NzkyZNb9tm33XZbUuebutX55Cc/mdT7\n7bdfU+cEvSRvQOdvFtzSsUbyjU+rqqruuuuupL7++uuTuq7xfs011yR1s/4gpxN5AgEgRIAAECJA\nAAjpq1tg0yJtu1C7/P3vfy+OHXfccUk9MDCQ1Lvttltxzn333ZfUdZvDdaHeWCnFlnT9/bxp06ak\nrltgfMIJJyT1smXLkvqkk04qzsn7Jttss010ip2k9n72BAJAiAABIESAABAiQAAIGZULCZvlkUce\nKY7lTfPcxRdfXBzrkaY5dJW8iZ4vAq6qqnrxxReTetKkSUl9xRVXFOdMmDChCbPrDp5AAAgRIACE\nCBAAQvRAhuGCCy5I6oULFzY8Z/78+Ul92WWXNXVOQGN5v6OqqmrVqlVJfdNNNxVj8jcbHnjggUl9\n2GGHNWF23csTCAAhAgSAEAECQIgAASBEE30L3nnnneLY73//+6Ret25dMWaXXXZJ6nyh0fjx45sw\nO+D95E3zNWvWFGPmzZuX1EuWLCnG5G8/POKII5J63LhxxTn5Duf5Z7RT3W7rzZyPJxAAQgQIACEC\nBIAQPZAtOPfcc4tjr776asPzLrnkkqSeOnVq0+YElOp+51++fHlSn3/++cWYxx9/vOHnnHLKKUl9\n9dVXJ3V/f3/D+dQtYhwzpnP+233jxo3Fsbp/V53O+VcA0FUECAAhAgSAkL663/1apG0XinjiiSeS\n+vjjjy/GrF+/PqnPOuusYsyCBQuSehSv+xi5P36nHTrmfq5bj3XGGWck9UMPPVSM2bBhQ1J/+ctf\nLsZcddVVSZ2v+xjKmooeWQdSe5InEABCBAgAIQIEgBABAkDIqFxIuHbt2uLY5ZdfntR5w7zOUUcd\nVRwbxU1zaIu8MXz//fcXY/I/iskb5lVVVXPmzEnqvGFeVY3v505rkA/lj6JspgjAiBMgAIQIEABC\nRmUP5LrrriuO3XvvvQ3Pu+CCC5L60ksvbdqcgKHJf+evWySYvxCurpdx5ZVXJnXdy6HyfkF+7Xb3\nHBpdu27jxlwzFz96AgEgRIAAECJAAAgRIACEjMrdeCdOnFgcG8rCwTfffDOpt99++6bNqQfZjbe3\njdj9nC8K/NSnPlWMefjhh5O67l599NFHk3qvvfYqxjRqog9Fs5roeYN8cHCwGLN69eqknjRpUjFm\n2223Teq6tw/WzNluvAA0jwABIESAABAyKhcSRuWLk8aMaU7+TpgwIanrfpPcuHFjUg8MDDT83HzT\nyGuvvTYwu3I+V1xxRTGmbhEWtEJ+L7z66qsNx4wdW37V5YuHzzvvvGJM/v/9/HPreiJDuXb+XVK3\nweuKFSuKY//XjjvuWBzL/7c44ogjijF5f6Pu2nmfZEs8gQAQIkAACBEgAIQIEABCNNGHYY899mjJ\n51588cVJvfvuuxdjXn755aT+8Y9/3JK5DEXd/w6f//znR2AmjEZ5U3rnnXcuxixevDip86Z1VVXV\n97///aTO32JYVeUfuDz33HNJ/fbbbxfn1DWlc0uWLEnqul108yb5oYcemtRTpkwpzsnfsnjIIYcU\nY/LG/9YsdPQEAkCIAAEgRIAAEDIqN1Os+73+5ptvHoGZNFf+23DdgsTc5z73uaT+yEc+0vCc4447\nrji277775odsptjbRux+zr+z8k0Rq6qqzjnnnKR+/fXXG35u3eaE+bXqxuTynkLdQsLcTjvtVBzL\nexz7779/Un/sYx8rzjnxxBOTuq5fOW3atKSu64HYTBGAlhIgAIQIEABCRmUPpM6tt96a1EN5wVSd\nRYsWJXVkvcZXvvKV4lj++2ed0047LamnT58+7Gs3kR5Ib+uY+7luDUW+7uN3v/tdMea6665L6mef\nfbYYk6/pyOu6TQdnzJiR1HnPoarKtV4HHnhgMeaoo45K6mOPPTap6zZzzY/V9Tfy3ugQ14HogQDQ\nPAIEgBABAkCIAAEgRBOdVtFE7209dz+36rtwKJ87xMV8I0kTHYDmESAAhAgQAEK8UAqgal3PocN6\nGU3lCQSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACE\nCBAAQgQIACECBICQdr6RsHdfywWjj/sZTyAAxAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAACBEg\nAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAi\nQAAIGdvGa21u47UYeX0jPQFayv08utTez55AAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAi\nQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQ0s4XSrEFq1evTurp06cXY+64446kPvvss1s6\nJ6CxwcHB4tjtt9+e1P/+97+LMaeddlpSz5kzpxjT19f572TzBAJAiAABIESAABCiB9IBFi9enNRj\nxpS5vueee7ZrOjAqbN68ueGYvA+Rn7Nhw4binFtvvTWp8/u7qqrq2GOPHcoUO54nEABCBAgAIQIE\ngBABAkCIJnoHePTRR5N60qRJxZheabrBSBhKwzxiYGCgOPbMM88k9VtvvVWM2W233ZK6GxYN1vEE\nAkCIAAEgRIAAEKIH0mYvvfRScewb3/hGUs+fP79d0wH+n0gfIu93VFW5Oer48eOLMR/+8IeHfa1O\n5AkEgBABAkCIAAEgRA+kzVasWFEce/fdd5N67ty57ZoO9KR83cemTZsanlO3iWneF8k/55vf/GZx\nTr7B4syZM4sxEydObDifbuAJBIAQAQJAiAABIESAABDS16pNxmq07UKd7OMf/3hxbNWqVUn99NNP\nF2PqFiN1uO7cHY6h6uj7Of9eGxwcbHjO2LGN/6Zo3bp1SX3YYYcVY5YvX57U+WapVVVVRx11VMNr\ndZja+9kTCAAhAgSAEAECQIiFhC32xhtvJPX9999fjMk3VuvCfgd0lHwBYH9/fzFmKIsLc//973+T\n+pVXXinGTJs2LakPPfTQYV+nW3gCASBEgAAQIkAACBEgAIRoorfYk08+2XDMXnvt1YaZwOhV97bB\nut13G3n44YeTOl9YWFVVNXv27KQeN27csK/TLTyBABAiQAAIESAAhOiBtNhjjz3WcEzdW82AkZe/\nXfD6669P6rrFiKeffnpSR3ot3aJ3/2UAtJQAASBEgAAQIkAACPFGwiZbtmxZUh977LFJffDBBxfn\n3HvvvUk9lDejdQFvJOxtXX8/528prGuI/+Mf/0jqE044IanrGuQrV65M6ilTpkSn2Em8kRCA5hEg\nAIQIEABCeuLH9k6S9zNWr16d1IcffnhxTo/0PKCr5D2Pt99+uxjzwx/+8H3POeCAA4pzJk2a1ITZ\ndQdPIACECBAAQgQIACF+fG+yxx9/PKnzF9nMnTu3ndMBtiC/N1etWlWMWbx48fueM3/+/OKc/v7+\nJsyuO3gCASBEgAAQIkAACBEgAIRoom+Fd955pzh29913J3W+cPCYY45p6ZyAmCeffLI4lm+mmG+M\nePLJJxfn5I32XuYJBIAQAQJAiAABIEQPZCvceeedxbGXXnopqc8777x2TQcYhnxjxAULFjQ8J3+h\n1PTp05s6p27jCQSAEAECQIgAASBEgAAQoom+FZ577rmGY6ZNm9aGmQDDtW7duqSuu5+32WabpP7E\nJz6R1GPGjO7/Bh/d/3oAwgQIACECBIAQPZCtcNtttzUcc+aZZ7ZhJsBwvfjii0k9MDBQjNlhhx2S\nes6cOQ0/d/PmzUndy5sregIBIESAABAiQAAIESAAhGiiD8PSpUuTOm/CAZ0p33m3qqrqjjvuSOr+\n/v5izH777ZfUO+20U1LnDfO6Y5roAJARIACECBAAQvRAhuGXv/xlUm/cuLEY89GPfjSpZ8yY0dI5\nAY2tXbu2OJb3Jg477LBizOzZs5N64sSJDa81ODiY1OPGjWt47W7lCQSAEAECQIgAASBED2QLNmzY\nUBxbuHBhw/PmzZuX1KP9hTMwEvK1GO+9914xZs2aNUk9YcKEYswZZ5yR1Hk/o66XkV+7bpPG8ePH\nN/ycXCf2TXy7ARAiQAAIESAAhAgQAEI00begrvm96667JvXMmTOLMZ/97GdbNicgpm4xX75wcMqU\nKcWY/Fj+vVD3PZE3u+v+ICc/b+zY8qu4E5vmOU8gAIQIEABCBAgAIX11L0RpkbZdiI7Q+T/gsjW6\n/n4eyndfM/oQ7bpOi9VO0BMIACECBIAQAQJAiAABIMRCQmBUalfjugsa5GGeQAAIESAAhAgQAEIE\nCAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIe3cTLF3dxSD0cf9jCcQAGIECAAhAgSA\nEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQI\nACECBIAQAQJAiAABIESAABAiQAAI+R9NkWlOEN9opQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85a048ce48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Generate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure generated_digits_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEGCAYAAAAwpAFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd4VFX+/1/TJ5n0RhJCC5CEEgIIIhCaBSvrWlgV0bWv\nYlfUn373WXUrft1118Kqu4oF9auCqIgoIIhKr1ICoaaRkIT0ZCaZdu/vj/uc4wxGV2Vmkqz3/Tw+\njwlJ7plzz/nU9+fzMaiqig4dOnTo0NHdYOzqBejQoUOHDh2dQVdQOnTo0KGjW0JXUDp06NCho1tC\nV1A6dOjQoaNbQldQOnTo0KGjW0JXUDp06NCho1tCV1A6dOjQoaNbQldQOnTo0KGjW0JXUDp06NCh\no1tCV1A6dOjQoaNbQldQOnTo0KGjW8Lc1QsIgN4UUIcOHTr++2D4qb+oe1A6dOjQoaNbQldQOnTo\n0KGjW0JXUF0MRVG6egk6fgB8Ph8nj6ZRVRWPx6O/Qx06woTulIP6WUFVVQ4dOsTUqVOJiooC4NFH\nH+Waa67BYPjJIVsdIYJQRs3NzQBUV1dTVlZGXl4eDQ0NAOzevRuAyy+/nOjoaAD93Z0EVVW7bE9U\nVUVRFIxGzQ7vKe/G7/fT3NxMU1MTAFlZWVit1i5Zi7gHXbV3uoLqIlRXV/PYY4/R0dHBuHHjABgx\nYgQ+nw+LxdLFq9MhLmZcXBwAJpMJl8tFY2MjK1askP9msVjw+/1dts7uCLF3Xq+Xbdu2kZubS2xs\nLAAWiyUiwk5RFBoaGli5ciUOhwOAadOm4XA4MJlMYX/+j4XYs4aGBp544gmWLVvG7bffDsBNN90U\n8bW43W4A/vznP3Pw4EH69OnDHXfcAUCfPn2k0g83dAUVQaiqyrFjxwC444472LZtGzExMeTn5wMw\nYMAAjEZjl1qdXY2Ojg4AnE4nSUlJXbYP4gIKwREdHc2wYcOorq6WBkRFRQWtra388pe/lAI40lAU\nhfb2dgD2799Pfn4+NputS9aiqirt7e18/PHHADz//PMUFxczYsQIKWQvuugibDZb2N6reF8ej4cP\nPviA+fPn4/F4AMjLy+PBBx9kzJgx3UpJKYrCgQMHALj77rspKioiNTWV3NxcgIgpA9D2r7W1lX/9\n618AvPzyy3R0dJCdnc1tt90GRNab0nNQOnTo0KGjW6JHelCKouDz+QBob2/HZrNhNpulpRFJi+OH\nQlVVampquPfeewFoamri4osv5u677yY9PR3QrPRQrD0wmS8S+V6vV/5tv9+Pz+dDURTpDVitVoxG\nY5BlaTQaMRgMEbOYFEXh7rvvBmDt2rUsW7aMwYMHR+TZ/wliDywWC4MGDZL/P2vWLOLj47vM0/P7\n/Tz//PMAzJ8/n1mzZvH4449jNkf2arvdbrZv386XX35Ja2srADNmzCApKYn6+nqZtwvn3RQeHMDS\npUt59dVXqaqqknvx9ddfs2DBAgYPHkxiYiLQ9XkpVVVxOp28+eabgBY5uPDCC7n00ksZP348QMTe\npaIoVFZWsmDBAt5//325HrPZzAUXXCDlVCT3rEcpKJH0LC0t5dFHHwVg586dREdHM3LkSM4++2wA\nzjjjDBISEjAYDFLgmkwmLBaLFLqRRmVlJffcc49Mul977bXMnj0bq9UatJ5TDe8J5STyIm63mz17\n9vD+++/LMEJlZSV1dXW4XC75rPj4eKxWKz6fj6FDhwJazP7MM89k4MCBUpGFU8D4/X6+/PJLQAuf\nvf322/z2t7/tUiEinh34Xg4ePCj/PSkpKeLKQEAYHytXrgSgrq6OL774Ar/fH7E1ifDZ2rVr+fLL\nLxk9ejRnnXUWoBlcV199NcuXL5dKS1GUsL1PRVEkcWXdunVkZ2eTnp5OdnY2AOXl5VRVVTFv3jwe\ne+wxucauhKIoFBcXU11dDcDUqVO5+eabycrKkrIr3OdfsFBramp46KGHKCsro3///gAUFhZywQUX\nMHr0aOx2e1jX0Rl6lIIyGAyoqsrRo0f56quvAKivrwegqqqKzZs3A5Cbm4vRaMRqtRITEwNoVkhB\nQQFTp06lX79+8nuREH5Op5Nnn32W5uZmHn74YQAmT578nULkVJVUoIdZUVHB4sWL2bJlixQSfr8f\nq9WK1+uV7CBhyTU3N9PS0gJASUkJmzZt4vrrr+f0008HwOFwhE1Jtbe3S0vb6/VSUlLSrfJxPp8P\nr9crLcn8/HxSUlLCvr7v2wNFUWRC2+Px4Pf7IxZBUBSFZcuWAbB69WqmTp3KRRddFMQ4czgc7Nu3\nT64pXLkfkTsRezF27Fjy8/NJTk6WJIkjR47wl7/8hUWLFjFy5EgArrrqqi5lGXo8HoqLi0lNTQXg\n1ltvJTMzM2I5MlVVqaysBODee++ltraWyy+/nDPPPBOAxMREYmJiiI6O7pJ96lEKCrQDPmTIEGkV\niToUs9lMQkICAK2trTKBLeB0Olm0aBFTp07lgQceAKCgoCBslqaqqlLQP/vssxQXF3PrrbcyZcoU\n+Tk6w6keAoPBECSgDhw4gM/nIy0tjdNOOw2AYcOGkZOTQ0xMjPSMRBjw008/ZefOnYCmoA4dOsTi\nxYulV2W1Wr/l9YUKPp8viBJcUlKCz+frMootfGNdut1umpqaUFWVESNGAJCdnR1WVpp49vf9fVVV\nKS4ulj+fmZkZEQWlqipffPEFCxYsAKBXr17MmDEjiKDh9/t54YUXeOutt7jhhhuA8IWrFEXBZDJJ\nZTRu3Dji4uJISUmRd028t+rqau677z4AfvWrX0XcAw6Mcuzbt4/6+noKCgoASE1NjWiKoqKigosu\nugiAtrY27rrrLq677jp559xut/R6u0JBdb9kjQ4dOnTo0EEP9KAMBgOZmZksWrQIgD179lBeXs5n\nn30mvYGSkhKam5txOp0y1OX3+/F4PJSVlUlLIJyWSmVlJU899RSg0X/nzp3LlClTIua6i8+Wl5fH\nsWPH6NevH5MmTQIgJyeHqKiooM+vKAqKotCvXz/WrFkDwJo1a2hpaWHSpElyb8MZFvX7/dK6VFWV\nuro6vF6vfHYkLThVVVFVVeYMy8vL2b59O4WFhfTp0wfgB3uSP7XY8Yecz82bN8uCTqvVysiRIyNi\ngXs8HjZv3ixznbfccsu36O3Nzc3885//xGQyMXfuXCA871BVVYxGIzabTZJqoqKi5FkNJLjY7Xa8\nXq+Mbni93oh7UMIzPn78OJ9//jl9+vTh/PPPB374mQoFvF4vDz74IBUVFYBWAnDTTTcRFRUl5abJ\nZMJqtXZZbWaPU1CgXdyUlBQApkyZgsfjIT8/n/Xr1wOaEPV4PJSUlATlXaKjozn99NPlIQ7XQWhu\nbuaKK66Q8fDbbrvte3NOJ+NU8y6BYT6r1YrdbmfgwIEkJycHPePk3xE1NaKzRXx8PJMnT2bMmDEy\nmRxO4dfa2kp8fDygFSzGx8fLixJJqKqK1+ultLSUJ598EoAvv/yS/Px8CgsLZbL4x1zacOTSVFXl\nxRdfxOv1Atq7/sUvfhERAed2u/H5fEydOhWAkSNHyucKpfXKK68AcN1114W8TixwPwMVkLhjnYWk\nVFUlPT0do9Eoz7HH45HnPVIQ70t0Jhk1apRcQyTDe9XV1Xz55ZfSaL722muJioqSuTHQlGmo2MU/\nBT1SQQVCWE4jRoxg4MCBgGaZzJ8/n4qKChlLNRgMJCUlccYZZwR9L9Rwu93Mnj07iFJ+9dVXf68w\nC/QaThZkP3WN4tD17t2biRMncuDAAUll9Xg8ZGRkUFBQIAVHa2srTU1N7Nq1C5fLBWgV42PHjiUt\nLS3sFpSqqtjtdlm0XFpais/ni6h1G1jkWVxczJ133ilZYTabjby8PNLT06WC6ozh1xnCFb/3+Xys\nW7dOfh0TEyMp8OGEqqpYLBb69esnnycKzAOL0YXHOXfu3JB+/u/a75PZsAJCYVZWVrJjxw5sNpuU\nAU6nUxpFkUDgugYMGICqqiQnJ3dJHmzv3r34/X5J0MjJyZGkG1Ewb7fbu5Sk1OMVFCDp5ELY2u12\nJk6cyNatWykvL5ffGzFiBGlpaWETFqBVXm/atIlx48Zx/fXXy2efDGGlVFZW4nQ6Ae0iORwO+vbt\ne8rEgJPrdux2O6tWrQK0sJDNZmPNmjUyLCPam7S0tDB27FgABg8ejM1mw263y7BEuFh1fr+f+Ph4\nsrKyAE3But1u3G63THyHC0KwCst2z5493HDDDUE1NJdffjm33XYb0dHRci9EA9lABWQymTCZTBG5\n1MePH6exsVF+nZ6eHhFvQHjoJpOJoqIiAPr27UtSUhJOp5NnnnkG+KZtT6jf33ftrShDAWhpaaG8\nvJxDhw6xYcMGALZs2UJ5eTlWq1VGE9avX8+ll14aUdac8EaSkpKCahEjjYqKCoxGo1TQjY2NREdH\n09DQII3U9PT0sN+/74NOktChQ4cOHd0S/xUeFHzbvS8qKgqyLlNTUxk4cCB9+/YNi7Ukkq4vvfQS\nRqORxx9/vFNrVlh4R44c4d///jcVFRWyKG7QoEFkZWWRnp4eMmq1wWDAbDbTv39/Sdo4cOAA69ev\np7KyUhYItrS0sH//fmJjY6X1lJycTFxcHE6nU1p5JpMp5PFo4YVYLBYZ+xZhmUhYl4qi0NbWxuLF\niwGtq3xtbS1ms1mGP0RfNJfLJYkT77zzDrW1taSkpEgKc0FBASkpKWGvsVMUhTfeeAOv1yufc+GF\nF0Y0HNPQ0MCmTZsALe86bdo0amtrZTRh+vTpDB48OCJrUlUVv9/P119/DWi9Lg8cOBA0DkXQ0DMz\nM+U5e+GFF8jNzSU/Pz/s6/T5fLjd7qDQcKSa554MVVXZsmULVqtVekilpaV89dVXMqIDcOWVV34r\nXx1J/NcoKAjuonzs2DEaGhqkkhg2bBgzZsygd+/eYRGwIldRUVFBbm4uQ4cO/dbBc7lcPPTQQwB8\n9tlneDweZsyYweTJkwFNKHu9Xjo6OmSBcSghlF5+fj5DhgyhtraWrVu3ArB48WJSUlLo37+/zAMp\nioLH48FgMNDW1gaEtvI+8OAbjUa8Xi9btmwBtMvc1NQU9ti8CLU+++yzzJ8/H4ATJ06gKEoQi+/v\nf/87L730Ei6XS17g5uZmzGYzcXFxsi5v7ty5TJo0CYfDEVb2od/vZ/ny5cA3tUUXXnhhyJ/zXTj5\nM/Xt25eEhARMJhMzZswANJJNpEJniqJQXl7Os88+C0BxcbE8u0IGpKamUlhYyJAhQ1i6dCkAhw4d\n4oYbbuCtt94iJycnbOvz+XyUl5fT3t5O7969Ae0umUymoLB5pArTVVVl8ODBHDlyRL6vhIQENm7c\nyPHjx6WBOH78eNLT07usue5/pYJyu900NDRgNpulUB41ahSDBg0KG41TCKO4uDjGjBmD2WwOIj/s\n2bOHe++9l6NHjwIa62nu3LkMHz5cvvyKigo2btxIZWWl7BUWLgFtNBoxm83s27dPfu+SSy5h6tSp\nDBgwIOgzdXR0yPyMENzhYKQJ1qP4OiYmJiIXQxA0xGcUXqLBYJC93ZxOp3yfYk02m424uDgURZEX\n+tChQwwbNgxFUWTheCg/g1hDbW2tbF3Vq1cvgIgQJMQa/H4/w4YNk+czNzcXv98v/wNNAHu93m95\n3Sez70Kxnvb2durr68nLywO0/FxHRweZmZkMGTIE0BR4VlYW0dHRsi3arbfeyt69e/nd737HW2+9\nBYSWSSe8t6KiIrZu3crAgQNJSkoCvmm/FtiSTVXVTu98OO5bbGws+fn5DB8+HIChQ4fSt29fHn30\nUZm3i4mJYcyYMSF99o/Bf42CCkyQrl69WlbXC8s2KysrqKFsqCE8i/j4eCwWCz6fTx6q1atXc9VV\nV6EoCrNmzQLgySeflJadCIkcPHiQyspKxowZE3ZaZ1tbG3feeScnTpwA4Pzzz+eGG24IopSK9Qva\nPgQPgQvVpRF/5+R6i7i4uLArKIPBgNVqZerUqRw5cgTQPF2v18uJEycoKSkBNAaY1+slOjpaKoKz\nzz6b7Oxsmpub5bpPP/10DAYDXq83rKGRsrIyWlpaMBgMMvwYFxcXsXCR2+0mISFBKgSn08nKlStZ\nt26dbHJqt9vx+XwkJCRIApMYtRFIJAlFmMtsNjN06FC5nltuuQWn0xkk/AVbLnDPBg4cyL59+9iz\nZ49kroUySlBbWwvAa6+9Rp8+fUhNTZVe+dGjR0lPTyc+Pl6SlTrzusV9CyX5RjSMbmtrk+xn0fQ4\nKSlJpkc++OADbrnlFkaPHq13ktChQ4cOHToEerwHJaxUn88nw1V/+tOfZA2UKNRNTU0lKioqLFaA\nwWCQYbH8/Hz27NnDhg0bpDU0Z84cfD4fl1xyCU8//TTwTUcGVVUlKeGLL75g0qRJDB48OKweVFtb\nG5dffjmHDx/m0ksvBbSk8nfVPJjNZpkTE2G4UJA4Tq73MhqN0tI2Go0MHDgwIlab0WgkLy9P9mhs\naWnhwIEDrF69mrKyMkCzbGNiYrjpppuYPXs2oFnkBoOB0tJS+bcyMzOlRRyO8Kw47y+//DJ+vx+T\nycTMmTOByI1lECFej8cjm/s2Njaydu1a7Ha7DDn2799fhtnFXfD7/Rw/fpympiYyMzMBfnRz1JND\nzCIsFuiJORwO4uPjg6YddxZBEec58B6GyoMShdQA7733HtOnT2f06NEyGuFyuUhISCAzM1O+O7E+\nn88nIysNDQ1YrVZSU1NDdh98Ph+rVq3i2LFjQdGR2NhYZs2axSeffAJozbgXL14c1r6l34ceq6DE\nRRWFZeXl5bz99tuAJmACG48CpKSkhHWSpxCsZ555Js899xz33HOPDDkKhtP8+fODYs1+v58TJ07I\n0SGtra2MHDkybG3tRQjj9NNPx+12c80118ju6t/3TMGwA+1ShSuRazKZJGtOKKxIKCgR5hMCYfXq\n1SxZsoQDBw5IAWaz2UhPTyclJUXW1lVVVVFWVsbhw4elsJ0+fTqpqalha3ArzpQgkzgcDiZMmABE\nrguBwWDAZrORmJgo2yy1tbWRn5/PiBEjOO+88wDtToj3KO5re3s727dvp6KiQobjYmJiiIuL+0Hv\n+7vCpie3kwqsTQsUrCJELdirX3/9NUajkTFjxoS824Xb7ZZElo6ODqxWK1FRUfJsmEwmSUAIlGce\nj4empiYZXlYUhbS0NJKTk0P2jo1GIy6Xi5qaGj777DMArrnmGmw2G3369JGEmw8++EB2mOmKqdE9\nUkEF5pu8Xi9tbW2UlZVJaykjI4Pm5ma8Xq/8uXB3CRaKZ+bMmWzdupXXXnstSLilpaXR3NwsL0ZZ\nWRkffvghy5Ytk8L/0ksvDQvLELRD/te//hXQ4uKjRo3i3nvv/d7x4KKA1efzSSsr8N9CrTxMJlNQ\naYB4d5GAqqoyX7Br1y4OHDgQNC9LjG7ZuXMnq1evBjRDyOVy0atXL7k/DocjIsQOl8slvUzR9itS\nEAZLXFycFLa1tbVMnjyZ4cOHB3nBJ58Rr9fL1q1bOXbsmMzl2e32UzZGAj0lAZfLRV1dndwf4RnV\n1NTIFlbV1dVERUVx7733hrykQRiEoJ3l/fv3s2rVKgoLCwFtlIXBYKCurk6uX1EUysrKOHjwoMyj\nZ2ZmcuWVV8qIC5w6acJqtTJixAh27tzJc889B2hnNz8/n4yMDKZNmwbA4cOHAe2sCzp6JNse9UgF\nBcGWlMlkCkqQzpgxg9///vfs2LEjyIWPhDUeHR3NX//6V44ePSoFmdvtZvHixXzyySeSFaYoCrGx\nsfTt21eGls4555ywWN6irf+HH34IaAd+7ty5su/WyQgcByDanog6L0GiCBcbUpA2AMlkDDdErdiw\nYcMAbWhcR0cHVVVVUqhFR0djs9lwu91Siba1tWEymcjIyOCSSy4BNEZdODtJCKEnWIUOh6NLKv0N\nBgMOh0OyUo8fP05hYaEUuvCNgRHYpWPFihU4nU4mTZrEOeecA/y4djrf1eLIYDDQ0tIiDYVjx46x\ndOlSfD6f7E4yYsQI6urq+OCDD+SQR9CapA4bNizkgtdqtUoD0GQysWfPHsrKymRpR2pqKuPHj6et\nrU2+w/z8fJqammRtFmhEr4SEhJBGFIxGI/fffz+fffaZfIcPPPAA2dnZTJs2TZaVmEwmqqqq+OKL\nLyQdPZwz4b61zog8RYcOHTp06PiR6JEelAg7AbJoMiEhISjWbLfbpQcARLQrtt1u57333pPD3N55\n5x3Ky8vx+XwyVzFs2DAefPBBhgwZ8q3mo6GEz+dj79693H777XJy5rRp08jOzpYTWCG4/kJYu/X1\n9Xi9Xo4fPy5JEmKCbDgo1Kqqcvz4cfm1mFwbCRiNRuLi4gCYNWsWv/rVryQRAJA900T/RNDOXq9e\nvUhLS5MlA+Huwyc8KL/fL3NnXTGORDxL9HFLTU3F6XRSU1MjQ3yiz2RSUlLQyPdJkyYxdepUeaZC\nsW6LxUJsbKwMoe/Zs4eamhpaW1tZu3YtoIX+nU4nRUVFUh5MnjyZefPmhTy8J3LMItzr9XrxeDxY\nLBa5xlGjRpGXl0dCQoI863a7XXaGF15KOHKxBoOB9PR07rzzTv7nf/4H0M7zwYMHaWpqksXEbreb\nxMRE2fhArDFcDZBPRo9UUD6fTzKnKisrSU1NpaGhQV6M7du3s3PnTtmVoSsQFRXFnDlzAI3FJwpc\nA2uMwvmChQKpr6/n+eefp6KiQoar7HY727dvp0+fPvKipqWlSfaV2LOamhpqamqIiYmR5IWoqChZ\nlxGONQcq60i+u8B6mcAcUmfhs0jnfAIhZvcAsulpV4xCEMpRFCN7vV527txJa2urDAdXVlbS3NzM\nmDFjZLhqypQpWCwWEhMTQ7ZucZesVqs8w2eeeSbDhw9n3759MpcjlMbgwYM544wzAJgwYUJQWDKU\nsNvtksW3efNmamtrmTlzJmlpacA3s5+6qlu42WzmxhtvlNN8P//8c+x2e5BhmJGRgclkIioqSt5N\nUeMZqDjD9Rl6pILyer2SPfT1119TVlZGZWWl9Ab27NlDU1MTJpOJkSNHAoSti/n3IfB5keyWDN/Q\nZ999913279+P3W6XFeypqamS2isETO/evbFYLEFjs0E7oIGstHAfSOGJ2O12xo0bF5Zn9GSIcoak\npCQ6OjrIycnpEvovaAJWvKO8vDzphYuzLijmQnmI74Xr7BiNRnl++vfvT79+/Rg7dqzMhQWOLg/s\nChKOuymIJKJ9UjjbKJ0KTCaTfIfjxo2TxCgBQUgL/L7wNgMN7nChRyoou90u25eUlZWxbNkyiouL\nZWjK6XTKBqmCwv19bLX/FohDpCiK9IzsdjsDBgygX79+/PKXvwQ0mnlMTAxms1kKt8BBb+LQJScn\n097eHnSBw6nkTSYTjz32GABLliyRpBcd30CExVavXs0777zDb37zmy6zwA0Gg4xadAUF+fsQKDi7\nqo9cT0RnHl1X7p9OktChQ4cOHd0Shq5spX4SftRChNteU1PDvHnz2LZtm/QajEYjf/jDHxg2bJiM\nSXfVyOJIoq2tjb1795KXlyc9xuPHj9Pa2kpWVpa0vjvrfdYZDT+ws/rJjVJ16NCh4wfiJ7v4PVZB\n6dChQ4eOHoGfrKD++90KHTp06NDRI6ErKB06dOjQ0S2hKygdOnTo0NEtoSsoHTp06NDRLaErKB06\ndOjQ0S2hKygdOnTo0NEt0SM7Seg4daiqSl1dHXv37gVg/PjxYR3o+N8AUXsnGshardafRX1dT4Si\nKLS1tREdHS3fW+BU38BZY2Lcin72ux/026VDhw4dOrolerwHJXrPuVwu2UV527ZtNDY2UlBQIDtJ\npKSkdNpB4eeM1tbWoImoGRkZIR870JOhqqpsuvvnP/+ZpUuX0tbWJr2me+65h9mzZ8vR5j8niL2Z\nN28eAAcPHmTXrl1UV1dLD9Pj8aAoCjk5OUEj6sOJwA4zjY2NZGdnBw1OBO3c19TUANpQTLvdTlxc\nXJc13YVvJgL7fD5UVcVisQRNPvi5oscrKDEt9v777+fYsWMANDc3yzHroqns/fffT25ublgm1vZE\n+Hw+nE6nvLx6uCoYiqLwwQcfcM899wCawBPdsMW4hPLycvbt20dOTo6ci/RzaQWlqirt7e1SsK5Z\ns4a6urqg0JlAaWkpL7/8MgB33nln2ASuqqpSOdbV1cnzLO680WhEVVUSExPltGaXy0VbWxuNjY2y\ns39g4+RwQ4Qin3nmGQA+/PBD6urqGDJkCG+//TaAnFPW1QjsOhQppdljFZS4GCUlJbzwwgvANy3t\ne/fuTWtrKyUlJXIWzBdffEF2dnaXeFFiwKLP55MWudfrxWQyYTQaZQ9BMXMlsKN4uNbj8XiIjo6W\n+2g0GlEUpUsErDj4QgEIdNWsHEVRKCkp4e2335ZjXcxmM7169WLWrFlce+21gPYOGxsb5cwsiPxY\nlZMRqf0SIzTE84xGIxaLJWhWmMfjkZ7WK6+8Amiz0cLlqQTO9Bo4cCAWi+U/5pYcDgdmsxm32y3v\nAnzzHkO9n6qqBg0Era2tpbKyktraWkDbR5PJRHp6esQMRnGWhHwKHMq5c+dOduzYQVVVFY2NjQBU\nV1czZMgQ/ud//ocRI0YA+jyoIPj9fqqqqgB4+eWXMRqNzJs3j0GDBgGaMOno6GDr1q0sX74cgNzc\n3C5RTKC9+H379vHSSy+xe/duQAuJdHR04PP5pIUXHx/PrFmzuP766+nbty9ASENuYj1erxdVVUlJ\nSZHPrq+vlyM4IrFPgfNl2traqKmpobi4mF27dgGa4TFo0CBiY2PJzc0FoKCggOjoaOx2e1iEnLD+\nq6urmTdvHqtXr5ZCYty4ccyfP5+BAwfK/fH5fJSUlGA2m6VwC9cwR/G3hYdw4MABPvroI9avXy8j\nB36/n+uvv545c+bI4ZThHo8i5q0VFhZSX19PcnKy9E5WrVpFWVkZiqJw5MgRANl8OBwInE8kvKXO\nmiAHwmg0Yjab8Xq9MuwXHx8vx4eEcv9EKqKoqAiAw4cPk5GRwZgxY+RMpq1bt8pBp5EYEaQoCvX1\n9QC8//5disXBAAAgAElEQVT7bNy4kQ0bNnDixAn57zabLWg6ufid5cuXy5E44VqrHtPRoUOHDh3d\nEj3Og1IUhZqaGtavXw9okzMnTpxIbm5ukFtutVoZMGBAlw69ExbH66+/zh//+EeOHz8eNN3zZDQ2\nNvK3v/2N1atXs2jRIgAyMzNDFuoSocT29nbsdjtRUVFBE2w7Ojrk1+GECHmWl5cD2tTf4uJiKioq\naG9vBzTSxpo1a+TID4E+ffowb948zj77bCB0ITUxDhxg5cqVrFu3Do/HQ3JyMgBz585l4MCBQc+z\nWCz079+f9vZ26dkYjcawWJM+n4/jx4+zYMECAJ5//nlaWlpwu93yLBkMBh555BFWrlzJ0qVLAcL2\nPg0GA3a7nQsvvBDQRqe73W7sdrskKx0+fJjy8vKITF4VEGdc3L3o6OgfNAW6rq6ONWvWADBy5EhG\njBgRUi/d5/NRVVVFUVGRjKJcfPHFMu0gogmxsbE0NjZSVVUV9j1TFIXq6mr+8pe/ALBu3ToaGhpo\nbW2Ve5WcnMzIkSNpb2/n+PHjgPZe3W43tbW1MlwZGO4NJXqcghIXVeQGRo0aRf/+/b8VmhICR5Ak\nMjMzv/eFB+ZBTlXoqapKR0eHTLC/+uqrUoB1hsAL5Pf7qamp4ejRo4A2qj4UYTeXyyXjyhkZGfJA\niT1RFIVjx46Fnc0kchIHDhzgT3/6EwCbN2/GaDRit9sly8tgMODxeILGc7e2tnLo0CH+/ve/M3Xq\nVCC0OR+hHBcuXEhZWRlGo5EZM2YAMGXKlG89SxhCJpNJCsZQ7p343H6/ny1btvDAAw+wbds24Bu2\n18k/7/P5WLt2rWTNTZ48OaxhPhEiTktLkyE1oaBFXjMwzBbO8T6KolBXVwdo4cXBgweTn58vQ4qd\n7YNYc11dnVSsMTEx+Hy+kJwtcaaeffZZVq1axZAhQ3jooYcASE9P/9Yz3nnnHVwuF3l5eWFXUO3t\n7fzud79j1apVgHbO7Ha7DKkDjB49mtGjR9Pe3s7hw4cBbcac0WgMMo7+Uyj1p6JHKiiHw0FGRgag\nCduoqKggL0NRFLxeL2azmeHDhwNaMvS7CBIns1NOZbNFEvSf//yntHaF8AqEyWTCarUSGxsrrdyU\nlBRSU1MZNmyYTD4ajcZTfvEej4etW7eSnZ0NBFuVAk6nk7a2tpAKELGPQkiBloPYu3cvr776Kps2\nbQK0ixEXF0dCQgKZmZkADBgwgJiYGLKysmhoaABgxYoV1NbW0traGrI1CiiKwsGDBwGoqqrCZrNx\n6aWXShq13W7v9PdEYj6QLBAqiHOzfft2fvWrX1FdXR3kLZnNZqKjo6WQa21txev14vf7WbhwIQCT\nJk0Kq4I62TtRFAWn0wnAvn375M/1798fCG9exefzSQX+1VdfsWfPHgwGA2PHjgW++90YjUZcLpc0\nLlpaWr51736KTPB4PLzxxhsAfPrpp0RHR/PQQw9J2RW4HmHAfvTRR5hMJmbOnBnW9+b3+1myZAkr\nV66Uzx42bBjDhg2joKAAl8sFaPk4RVFITk5m//79gMYqTEtL4+qrr5YGZbiUaY9TUCaTicTERCls\nQRN6gTRpn89HQ0MDCQkJcgOFcvpPB+1UFZTX6+W9997jscce61QxCSrrlVdeyZVXXsngwYODXq5Q\npMIyPdXwnqqqlJeXU1RUJBOxgc8LtNIFVTqUEHtwsgVdUFAgwzB2ux2LxYLFYpGXd+jQofTr14+U\nlBRKSkoArb6ttraWuLi4kLPlvF6vDGHY7XauvfZafvvb30qywX9CqC+oqqq0tbUBMH/+/CCaO2j1\nO5deeil33HGHDLPMmTOHbdu2oaoqe/bsAQgi4UQCfr+f9957D9CMHuGlC+ZjuL1zYWTs2rULu93O\nlClTZIi4s0iEuO8pKSnSKOzbt++3yi5+7B1UVZUTJ07wf//3f4DmdTzzzDP06tWr07OyYcMGQCPo\nOBwOSZIKNcT9q6io4C9/+QsNDQ306dMHgLy8PEaMGEFcXJwk3tTU1NDe3k5iYiI7duwAtDNVUFDA\nkCFDwu7l6SQJHTp06NDRLdHjPCiz2UxUVJQsliwtLWXp0qW4XC5pHaSkpJCZmcnw4cODchonozNP\n6aeEuETsH7RE45133inDHIGIjY3lvvvuA+Cuu+7CbrfLWiixRrGmULn3fr+f4uJiFEXplLIuiAE2\nm43Y2FgURZHfCwzf/NT1nBwqiYqKoqCggKFDh3LllVcCmqVtt9uD8n9+v5+oqChMJpMMCx07dgyT\nyURBQUFILTeRhxChjhkzZjBnzhySk5N/0OcWuUPxeUMFkb84duyYPBdZWVmA5i3NmTMHu90uz94V\nV1zBjh078Pv9MkTTGRknXPB6vaxZs4Z3331Xrt9gMGCxWCTZJFxQFIXjx4+zfft2QAvTpqen43K5\n5Ln/vsiIz+eTMiUxMfGU75+iKFRUVMj3NWLECMaOHdup568oiixk9nq9xMbGykhLKBHYGeUf//gH\n5eXlGI1GWT96zjnnMGrUKNra2uT7ysjIoK2tjfXr18vfbW1tZcuWLUGh7XChxykoo9FITEyMFFo2\nm40XX3yRo0ePyovqcDiYPHkyV111Feeeey5AUJgjUAkF1uPATxPGfr9fFtq9+OKLMkcS+HdsNhsj\nR46U8XCTySSV0w9hGf1UiNzKd/1tIcBaWlqwWq1ER0dLwSgS8UlJST8ppNbZXorPLGorQHtfQngE\n1hOBdmE/++wz4JtQ7jnnnBNyBXX8+HEZ4ps1axYpKSk/+Bk+n0++81CGH8Xz/X4/BoOBqKgoLr/8\ncgBmz56NzWbDZDLJPauoqMBsNgcZGeFWUIG1bH/7299Ys2YNBw4cCPo3h8Mhc1DhgtfrZcOGDTIM\n5fV6Oe2005g8ebJUUN9lpDqdTtasWcPQoUMBZD3lqUBRFGpraxkwYACAZHr6/f4gtrGiKLS0tEhW\nssViYcKECURFRcn9C5VcCMyzfvDBB7jdbrKzs3nyyScBgliqw4YNk7/jdrtxOp384Q9/ALROPUlJ\nSd+Zlw0lepyCEkJPHLoBAwYwYcIEjhw5IhVUc3MzK1asoLKyUjJS8vLyvqXxO7Oofsph8Pv9NDc3\nA0iLLVAwWCwW6cmJwmGDwUBhYaHsHBH4/FASFUQhYk1NjcxVCKUY6Plt3rwZh8NBbm6uZELFxsaS\nkZEhCy9DhZPzfKKforiwoFlpHo+HkpISSZkGOP3005kwYULI1iI8pyNHjsgi4XHjxv1gwa6qKo2N\njaxduxaAs846KyQWuMFgkGdGURTMZjNWq1W+Q6fTSVJSEoCs8N+xY4fs3iB6UIazs4Xf75ekhPvu\nuw+PxxPU+2737t34/X4GDhwoc4vhgKIoVFZW8uKLL8qi04SEBAoLC4mLi+vUAAwsWt+8eTOlpaVB\nSjQUHpTVapVsY4/Hw1tvvcVFF10k75MwBBctWiTzjXa7nVtuuSUsuR2fzyeJWzU1NRiNRq6//noG\nDhwIdH5WhDHZ0NAQ5JWbTKaI9O3scQpKQLzAuLg4nnzySbKzs5k/fz6gJRq9Xi+HDh3i6aefBuDR\nRx/9VuIxVJ6L2WyWB95qtRIfHy9bhoB2GVwuF7t375bW5fbt27n11lu5+OKLgyrrQ93ex2QykZOT\nwxdffEFFRQWAPJB+v1+GRBYsWMDYsWNpbW2ld+/egNYyKiUlJeRCTiikQCUglKUII3g8Hr766ive\ne+89WXlvNBqlUg/lWqqqqnjnnXfYuHEjoCnm9vZ2hg8fLjsKiAR7oMftdrs5fPgwl19+uQzpPvHE\nE8yaNSskaxOEggEDBrBz504URZGUYKvVyiWXXEJmZiYfffQRoHWXEIo/Pz8fCG0nkkC43W6WLFki\nKdM+n4+zzjqLadOmSUVfXl6OyWQiOjpa3tdQs0TFWh5++GG+/vpr+ZzU1FQsFgsVFRVS8Yi9CPR4\nN2zYwLx58+jo6GD06NFAaDwWk8lEVFSUlDlbtmxh+fLl7Nu3T5KRDh8+TGpqKuvXr5eGYnx8PNnZ\n2WFp8+X1elmyZAmg7UGfPn245557ftD9fuedd6Q8M5lM/PrXv45IxxmdJKFDhw4dOroleqwHJWAw\nGIiNjeXOO++USffVq1fz5JNPcuLECUnfLC4uJisrKyx0YEVRZMfh008/nSFDhlBTU0NZWRmgWUqt\nra20t7dLN/nrr7/mD3/4A06nk2uuuQbgB1OafwwMBgNDhw6lpaWF119/HYCHH34Yi8XCsWPHeOSR\nRwAtR3bGGWcwceJE6aGIXF+o+5H5fD78fn8Q5VhVVWlti3Xn5eVhtVrlz3m9Xun9nSqE9e3xePj4\n449ZvXq1DMcsXryYJUuWoKqqfK+jRo2itbWVqqoqWfBssVioq6vD6XTKNa5du5YrrrgiJHRq8Tfy\n8vKIjY3F6/XKHmmi00WvXr3YunUroOURRdcG0UEllO8uMN/03HPP8eqrr8rQlMFgoL6+nq1bt0qP\n1263k5qaKnMqgAxRhsIrF+s5evQomzZtwuv1yvNz4sQJnnzySWJjYyWNWoS6zWaz9AaKiopoaGgg\nKipK1tuFwsszGAxBIcMBAwawd+9eysvLJYVbTBMQvTFB68gRLoLEjh07ZITCZrPx0ksv/aBohMvl\nYvXq1fLr1NRU5syZExEPqscrKAGbzSYP4uzZsznttNOYM2eODKmtXLmSadOmhbwOQ3SsEOGqpKQk\ncnNzGTZsmExUr1q1io8//pj9+/dLMkVHRwclJSU8+eSTFBQUAJpyC0fOoHfv3jzyyCN8/fXXgNYY\n1uVy8eijj0qFecUVVzB9+vSgliWhVE6Bhboej0d2bYZv4tyB+TeLxcLQoUO57bbbZAsaIGT5MPEc\n0WEjsL2LCAG53W7ZlLi4uBij0Si7I4AmZIXAFX9v//79uN3ukJwzsT8XX3wxR44ckcoJNKZqdHQ0\naWlppKSkyDWI0FBgnWAoILqjADz++OO8/fbbtLS0yDXGxcVhtVr58ssvZT7W7/eTlJREdHS0DGG1\ntrbKOj9BXDpVNtjRo0dxOp2YTCb5HlpaWmhsbMTj8UjihNlsxmg0EhsbK9fd1NQkz6PYs1AYsUaj\nkV69epGamgpoCmvixIlUV1dLQsTOnTvl2RP707dv37DUiqmqytdffy3P7vDhw5k4ceJ//D1FUfj7\n3/+O2+2W72vmzJlhMaY7Q49TUCez7sTXgWw4i8VCXl4eY8eOlX2vPvnkEx5++GFSUlJC7hE0NTXJ\nNimpqamcfvrpJCcnSwX1y1/+UnbrFgJN5GCqq6tlhbZg+IUaJpOJCRMmBFXKb9q0idjYWNk5+aqr\nrgI0q054DaGMgwcqqB07dgQlzgMLKAOfZzQaaWxslPkCq9VKRkZGSNYk1lNbW0tDQwOZmZmSGZqb\nm4vf72fv3r1SkCUnJ5OdnU16erq0tGtqaqioqGDPnj1SwCQmJsp3fKoQnzMnJ4enn34ar9crDQqb\nzYbNZgtS9G+++Sag7Vu/fv1CsoZAiM4QCxcupKGhAbPZLM9USkoKHR0dVFdXyzUajUaZDxZEjr17\n91JUVITZbObiiy8GoF+/fj+ZJQra+U5OTsbj8cj8aWxsLCdOnKC1tVXuRZ8+fSgoKKC9vZ1169YB\nWr4atGjB9OnTgdAVEwfm/wwGA6mpqUHdUrKzs9m4cSP19fXSK6+vrw9L/snn83H48GFpzEycOPEH\ntX7bvn07r7zyCgaDQe7t7bffHrHeij1SQbW1tUkGjOi8cHK9jcViobCwULZ8cTqd7N+/n8LCwpC+\nfKPRSFRUlDxgDocDl8sVRHwQB7C+vv5b7DCj0cgZZ5wBhJdxZbPZZOL86NGjREdHc+ONNwb13RLs\nvXB0lBBoamri4MGDUtnAdytCVVV5++23ZTgmMTExZHRlccEcDgczZsyQU0xBU9Z+vx+/3y+FW2Ji\nIjabDUVRpMKsr6/n1VdfleQEgCNHjuD1ekMydiPQ4BINRUX4R/ybqHODb8aoOBwOWX8Tyno6cZda\nWloks1Ccn8zMTHbt2kV7e7tU1haLhebmZpYsWSLXeODAARoaGqRnAXDzzTcHESl+LCZPnsx1111H\nc3OzJKj4fD5qa2tJTk6WjMbo6GhaWlr48MMP2bt3r/xcdrud119/PeTkm0CmqogOmEymoNBtTk4O\nOTk5cmChUJihhtvtprKyUoZaLRYLfr9fRgUERMhRRHp+//vf09DQgMVi4fzzzwc0Ly8S4T3QSRI6\ndOjQoaObosd5UABlZWV8+umngOa2n3vuuUFjkQVxIbDNv8PhCMvoZNEsVMTdS0tL2bt3L4MHD5Zh\nvw8//JC1a9cGdbsAzWPKzc2VVnq4rRLhGSUkJOD1etm7d68sJGxra6OjoyPkNU8C4rNlZGQwcuRI\n+vbt+70FlKDlKz799FO5Z8nJySFrNhq4Hrvdzu7du+X4j6NHj5KTk0NUVJQMTYnprE6nU1LKTSYT\nlZWVWCwW6dFbrVaKioooLCwMWRiksw4jwkJvaWnhtddeA76ZSJyQkBDyPIYIEcE3RAeLxSK9pW3b\ntlFXV4fP55PPFnmrI0eOcOjQIflZQItoiB6L9fX12O32H71f4m85HA7uvvvuoF6FiqIwaNAg2tra\npMdbV1fH/PnzWb58ufxedHQ0d999N+PHjw/Z+xIlE+3t7TJvI4ZNNjc3yzVarVYcDgcxMTFBReuB\nk31DtZ7GxkbKy8tllOTEiROyX6l4nslkoqmpiWPHjsm6vsOHD+P1eomPj2f27NlA+EoXOkOPVFB1\ndXV8+OGHwDf5lJtvvlkmOVVVZe3atTz88MMyHh4TE/ODCyh/TLNYoaDS09MBeOWVVygpKcHj8Ug3\n2ePxfKvuB7TL8fDDD0dkBpNYK0BWVhZpaWksXbpUrttsNtOnTx/Z3DPw50P5bLvdTkFBgTQioPOO\n7aqqsnr1atlwFOA3v/lNyGPfZrOZxMREfvOb37Bs2TJAy7WIinshyBoaGnA4HDgcDil0Tpw4wdGj\nR4Oa3I4aNUq2jgkVAvOugWNhvF4vv//976USFSGkM888M+RCxGg0yvyFzWajo6ODtrY2meMNNAgD\nlajoRiLWbTKZiI2NJTMzk0mTJgEasehU3qvBYCA6OlruCWjv5siRIyxdulQSpYqKijhx4gR+v1+G\n4C+88ELuuuuukE75VRSF9vZ2ysvL5bsRY4IGDBggO1aIgv66ujr5c1VVVdTU1DBgwICQ3T+DwUBi\nYiKqqsrzXF5ezscff8zUqVNl2Li+vp7Dhw+zfv16Saiqrq7GYrEwY8aMoAkLkUKPU1AGg4ExY8bI\nl/zhhx/y5ptv8v7770sPqaWlhbq6uiDmyZgxY2SvrVDDarVKQV9VVcWxY8eCZqXAt6mrRqORyy+/\nnPPPPz+iLxw0oTxkyBC2bdsmPZLExERJKw9nx2nRsieQWtsZPB4Pv/3tb4Mo/FdccUVYvEyz2UxS\nUpLMX9TU1LB48WLa2trk+dm8ebPsWCCEf0xMDGlpadx3330yf3juueeSlpYWlnKGwD1TFIV169bx\n0UcfBY3giImJ4fLLLw95PtNsNsuShAMHDrB9+3bZgkmsz2AwyF6ZoN0Lp9OJz+eT+yHaD5155pmy\nI4jIIZ8qjEajfDdxcXFkZGTQp08fSktLAe1MWSwWBg8eLGe1XXDBBSHp/HEyLBYL6enpcj2NjY2c\nddZZQf0dBcFq7dq1MvfU0NDAggULeOyxx0J6D00mE/3795fKetu2bZSUlLBixQrZ1sjpdFJcXMye\nPXtk9EfM1LvxxhulrIhU/gl6qIKKjY2VScXhw4fz0ksvUVpaKmm4fr8fVVWJiYlh5syZAPzud7/7\n3qmP32fN/yeIlw9aq5vS0tLvbKEkLuLQoUN56qmnIuY9nbyWXr16cc0118gLFB8fH/Kap/+EwNZL\ngU1iAf73f/9X7qNgGIaz4ajRaJTKKCsrizvvvFP2IQONSVdUVCSp06CxLkeOHInZbJbrF4n2UO+j\n2+2mublZMgidTieLFy+Ws4tAE4pnnHEGhYWFIX02EETDXrRoEU888QTLly+X4U6fz0dycjJnnXWW\nrFUTRmRjY6PsqJCfn090dHRIhnB+1zpBMx4GDhzIbbfdxmWXXQZASUkJQ4YMISoqSr6ncDQ8FWfJ\narVK70Q0az6ZpWq1WklLSwsazbNx40ba29u/d9Dij4XVauXuu++WralqamqoqqqitraWzz//HED2\ncfT5fPKMT58+nf/3//5fWGpIfwh0koQOHTp06OiWMIRzBPOPxE9aiCiUdTqdMr4qul6npKQEjZ8O\np3cgPLDS0lLuv/9+Vq9eLfNfYo9FaA3gtddeY8SIERH1WALh9/vxeDzS8v+uacPhgqIosvDT6/XK\nrtzPPfccAH/+85/xeDwMGDBA9gsUffG6AiKEdXLn+0hA7NXBgwdl0emmTZvYtm0bFRUV0hONi4vj\n3nvv5ZZbbgnqyBEOnJwTO7nhsY4fBhHmu/HGGwEtfDpu3DieeeaZkJ53Qai5//77Aa3bTk1NTVBH\nF6vVit1uJycnR0aeZsyYQe/evU/V0/zpv9jTFVR3g5gD88gjj8jmoy6XC4fDwYQJE7jrrrsALRbf\nFS5zd0JgR+njx4/z+uuvy9BtY2MjUVFRrFixgvHjxwM/X8En8k91dXV89dVXgEbkEEw4kWAvLCzk\n2muvJSsrq0vyBTq6NwJnnh0+fJi33nqLnTt3ytD24MGDGTlyJNOmTZPh9O9Li/wI6AqquyGQpSZa\nIQUmg3XB8W0FtXDhQjm4zeVycd555/H8889HrK1Kd4U4S06nU9KEhedbX18v81JZWVmSNi+s4nBH\nDnTo+AHQFZSOng3hJQhha7VaiY2NjWjNhQ4dOsKCn6ygft4xJh06dOjQ0W2he1A6dOjQoSOc0D0o\nHTp06NDx3wVdQenQoUOHjm4JXUHp0KFDh45uCV1B6dChQ4eOboke14tPR8/HycQcg8GAoigcO3ZM\ndpK47rrr6Nu3rxwSCFpNT7h6uOnQoaP74b9KQQlBJsYO6OieEKMZBIxGI36/n9jYWC655BJAa5jp\ncDiCOkKDXuCsQ0d3wI8ZSXQq6LE088BeYIqicPz4cflvXq+X1NRUHA7Hz76dUHdEZ33txLysmpoa\nQBudLro8i07x0dHRP2vDI1JCIZQQ7zWwg0pP+ww/ZwTeUfH/HR0dtLe3k5SU9EPf5U9+4T3WgxJj\nGY4cOcKTTz7Jxo0bZcv67Oxs5s6dy6RJk/7j1FYdXYfAdyKaUYq2RqmpqZw4cYK6ujqplAYNGvSz\nUFAnK6LOhMR3/Z7wTkWro0iee9G4WbRjWrx4MW+++SZlZWUyulFYWMi///3vsE1u1nHqCBww6fV6\n8Xg8FBUVsWnTJkAbGHv33XdHxGDS3QsdOnTo0NEt0SM9KL/fL0c1rF+/Hp/Px3nnnSenQJrNZjZs\n2EBBQYEcvBVpS9Ln88lJnrt376a1tZXt27eza9cuQBtvkZSUxJgxY2Rr+6ysrIiPvfg+dDZKIRT4\nvr8nPOOmpiaKi4vZu3ev7MCcnZ0d0TBXoCUpvgbC3vDX7/djMpmCpucGWrSgDQi0Wq2oqkp9fT0A\nCxYsoLS0lNzcXG655RZAG0QZiTC33++nrKyM5557TnbxLykpoaOjI2i69FdffcWyZcuYPXt2WPYv\ncM/ECB7xvdjYWEwmE1arVXrigWf859rI+eSm1oENhlVVZc+ePbz77ruUlJQA0Lt3b6KioiJyrnqs\nghKTPAcMGPCtce5HjhzB7/d3Wf5JURS2bNnCrbfeCkBFRQUul6vT8MyqVas4cuQIoE2RjYuLO6UL\n0ll+R3xfCDev1xt0AOGb7tgdHR0yD7R161b8fj/nn38+KSkpQT8XDohx8KCNoO/VqxcVFRVyzldL\nSwspKSkRESBer5fS0lL27dsHQGVlJYmJiQwYMIA+ffoA2oTfkwUdfBOuFOv8KROaTw5/nvz/Qpi4\nXC6efvppADlhNzExkYKCAgCmTJmCzWYL210Qwu3IkSPcdttt7NixQxoUJpMJm82Gx+OR4fempiYW\nL17MlVdeGdJGwKqq0t7eztatWwH44IMPKCoqwuv1yrPr8XgYPnw4w4cPp1evXoD2XouLixkxYgQX\nX3wx8M2IiXDP0vL5fHg8niBSl8ViCcuU3+9bS3t7OwCtra3ExsZitVqDzouqquzYsYO2tjYA5syZ\nE7EJAz1OQQmrVoziHjlyJFFRUZhMJnkJevXqxf79+2lpaSE+Pj7ia+zo6OCpp56SikcohKioqKCx\n5W1tbcTExHD66acD2rjwUz2YYky5UEYnTpzA5XLh9/uDLNucnBxsNpvcn46ODlpaWti3bx+7d+8G\noKioSFpQf/zjHwHCPqI+cIzEoUOHaG9vl3vS3NxMQkJC2L1MVVVpbW3liSeekHvW3NxMSkoK55xz\nDpMnTwa093XixAl27tzJ3r17AcjMzOTaa69l2LBhMs/yY5X69yk28T2j0YiiKJhMJimA6+vr8fl8\n5OXlyfcfOPblpyrM74NQRitWrKC4uDjIMMzKyiI/P5+NGzdSVVUFaOczMCcVCiiKwsGDB5k1axZl\nZWWAduf8fn+QsE1MTMTtdlNXV8fBgwcBLbrR0dFBXFyc/CwzZ86U87RCCb/fT2VlpRwp8/nnn0sZ\nIUbQDxw4kNzcXB588EHS09MBwlZaIbzvd999F9AU8wUXXBBk7AMsWrSI3bt3c8YZZwAwYcKEiBn/\nPVJBtbe3s3//fkBLnGdlZcmJkaCF/VatWoXdbpfWbiTX19DQEGRJ2u12Jk2axI033kheXh6gXaCD\nBw9it9uZPn06ADab7ZS9J4/HQ2VlpbT8xTqMRqOc8Gs2m6XVdPToUblGl8tFS0sLRUVFgCbwbDab\n9GDCCZFgF+t588036ejowGg0SkGvqioul4uYmBgp9MOlqCwWCxkZGXLPQNsjq9UqJ/xu2bKFrVu3\n0oqxlP4AACAASURBVNzcLH8mMTERh8NB//79ZXg5XDAYDJhMJuk1+Hw+jEYjEyZMYNKkSQBSQIdL\nwAlFY7PZGDRoECUlJVKQzZkzB9A88cBaNofDERLvSXgipaWlzJ49m4MHD0rFbLPZiImJITMzk7i4\nOAAyMjIk+UaEq1paWqQ3Wl1dHfR3Q4m2tjbuv/9+Pv74Y2lI22w27HY7cXFxcmhgS0sLX3zxBR6P\nhxtuuAGAESNGhMULdrvdvPrqq6xZswaAm266iYSEBIxGo9yDiooKFi5cSHt7O+eccw4AMTExIV3H\n90EnSejQoUOHjm6JHudBNTU18fbbb/P5558DcPXVV9OrVy/cbjfvvfceoFnfhw8f5rTTTuPMM8+M\n+BrLyspobGyUVmtCQgJXXHEF5513ngwdqKpKXl4eJpMpZFR4YT0aDAbpOaakpBATE0NsbKyMG0dF\nRWGxWKRHAlr8+cCBA2zbtk1aT1arleTkZG644YawhDwCoaoqTU1N/OMf/wDg448/ZsqUKVx11VUM\nHz4c0CzgznI+4SBxREdHc8stt0iPt7a2lkOHDnHo0CEOHz4MaN5ne3s7qqpKq3L8+PHccccdZGZm\nRiQMcuzYMelBmUwmBg8ezG233Sa9hnCvQZyLq6++mhkzZmAwGIKISZWVlZw4cUL+vNFo5JJLLgnp\nO6usrKSurg5FUaQnctppp3H22WdTWFhIRkYGoJ37+vp6NmzYwObNm4FvzpHFYiE3N1f+fyigqqqs\nz7zkkks4evQoOTk53HfffQBMnDgRh8MBIIkumzdvZu3atURHR7N8+XK57kGDBhEVFRWyfRN3f/Pm\nzfLsjhw5UkZxRArl8ccfx+v1Mnr0aG6//XYgsiSSHqWg/H4/H3/8MevXr5d1HnV1dTQ2NlJXV8dX\nX30FwIEDBzCbzRQWFnYJUcJoNOLz+WRYo6GhgU8++YTx48fTu3dvQIub+3y+kLrLBoNBhqZEHNls\nNmMymYLi2IEECXEZzWYzzc3N7Nq1SzLpEhISmD17NgUFBWE9lH6/n5aWFm6++WZWrFgBaOG0m2++\nmXHjxsn4vHiXnTHrTu5McaowGo2kp6fzi1/8AtBIAIcOHWLr1q2yzsfj8eBwOOjbty/33HMPAL/4\nxS8kWyxwjeHYP6/Xy29/+1tqa2sB7b1Onz6drKysiJz7QOZbdHT0txLnfr+fP/3pTzidTvl+HA4H\nM2fODOl+1NfXy2eIvzt06FAuu+wysrKypNIS5720tFQqBNDedXZ2NuPHjwe+TVL5qairq2Pq1KkA\nVFVVce211/LEE09IpRT4DHHGzzvvPM444wx2797N+vXrAXjvvfe49tpryc7OPuU1CYh0QL9+/Rgy\nZAiANKoURZG51927d5OWlsa///3vsBupnaFHKSij0ciYMWN46623OOuss+T39+3bR1RUlLyo0dHR\nDBw4kIEDB0acMmowGBg8eDAWi0V6J263my+++II777xTsodaW1uxWCw88sgj0kM4VcvNaDRiNBo7\nJTJ0tg+Kokiv64033uBf//oXFRUVch1Dhw5lxowZIWXuBSoSYaXV1NQwd+5cli1bJv89MzOTsWPH\nEh0d3WnRqlD+iqKELYks8oSg7c+nn34q8xSg5ZtuvPFGbrrpJmmlCwbYyZ811PD7/axbt47Vq1fL\nXGdaWhq33357SNlx/wkn73sgNX/btm28+eabQYpjypQp0kgLFQYNGkRaWhptbW3yPL///vskJCTw\n61//Wj6vo6ODF154gYULF8q7qaoqsbGxzJo1i9jY2JCtye/388ADD1BRUSHX+Le//e07SUbC4Bb5\n1cBccF1dHTk5OfTt2zek77a+vp7y8nLp5Z1//vkkJCRw+PBhnnnmGUAzridOnEhmZmbInvtj0KMU\nFGgXYubMmfTr1w/QQkElJSUcPHhQ1kYlJCSQlZXVZV0HkpKSmD59OkuXLgU0xZGUlCTrB0Bz5f1+\nP0899RRPPfUUwClTqDsjDnTmbQjFVF1dzaJFiwBYuHAhZWVl+Hw+GR668MIL6d27d0iFv0hIt7W1\n8cknnwBa/Y4IUwkhMWvWLBwOR6cC0Ofz0djYKD9rYmIiJpMppF6Dy+Xi8ccf54033gCQws9isZCT\nkwPAZZddxk033SSffzJOrpsKBYR3e/ToUf74xz/icrnksxcuXEhWVlaX1Yn5fD68Xq8UrFdffbW8\nk8JDuO2220J2L8XnzMnJ4bLLLuMf//iHVDx1dXXMnz+fFStWcOWVVwJaKHDhwoU0NjbKdYtIy/Tp\n06WnFYr9czqdrFy5Uv6thx9+WO7Bf/pMLS0tLFiwQNZMDhgwAJvNFlKDR1EUXnjhBZYtWybPZ0tL\nC6NGjWLHjh0UFxcDmsGVlpYmjaBIQydJ6NChQ4eOboke50H179+frKwsqdHj4uKoq6ujpKREJmM7\nOjpQFIWWlhbpDUQy1GcymViwYAFLliwBNGtq8uTJxMfHU1lZKdf92WefsWHDBulJXH311WHz+oTn\nARotd8mSJaxfv17S0QXdNjU1lWuuuQaAyZMn097ejsvlktalxWKR4Ygf+3zQQpv79+/n1Vdflb29\njh07Bmh7MmLECACuuOKKb1noosarpaWFHTt2AJp3MnXq1KB8yKlCURSWL1/O888/L+n4wgu+6qqr\nOPfccwGNECFCkIEdCURIK9RnLrCo8oUXXmDnzp0oisJVV10FwLRp0yJ2zgUtW9TMbdiwgfr6eg4e\nPChre2pqauR7EV5nYWFhyNdotVqZO3cuBw4cYNWqVQC0t7fT1tbGrl27ZEmKz+fD5/MFFcamp6dz\n9913SzJMqNclvKa0tDTZIaQziJB1Y2Mjc+bMYf/+/XKfevXqxciRI0MiG8Q5dblcrPn/7J15YJTl\ntf8/s0/2PSQkhEDYokBkVxZBwQoiyKL2unCV1rVqW+t+vda23qpV2ypqr62W4m0raKGKKCJSlEW2\nRIjITogQsu9kmX3e+f3x/p7jDOACmYlI5/sP7ZiZeeZ5z/Oc7XvOWbsWt9st37Nx40YaGhro7OyU\nfFNaWhoDBw6Uswdf5LW7Q9a+UwrKYDBgt9sJBAKSaBwxYgR+v5/CwkL+93//F4AtW7awY8cOfvnL\nX/LUU08BesuX7gz5xcXFce2114asPRAISFFldnY2+/fvp7KykldffRXQQ0bqd4UTqsZI1Ylt2bKF\nnTt3sn//fnnN6/WSmJjIjBkz+P73vw/ogtja2squXbsktNSvXz8GDx58yrFwZVBs3LiRhQsXSpU/\n6CHZ3r17k5qaytSpUwFdyTc1NWE2myW3WF9fT3l5OSUlJRI6Ovfcczn//PPDum9er5fVq1fLOkA/\nqPfccw+zZ8+WAkqLxSJJZbU/fr+fQCCA3W6PqLxt2rSJ9vZ2bDYbjzzyCBB5xl5wu6UdO3bwxBNP\nSDJdhRp9Pp+EsYPrr4YPHw7wjcJcp4PExEQWLVokYfWXXnqJbdu24XK5QmrZQH+mynCdMGECAwYM\nkGcZLsTGxjJp0iSpMXrllVfw+/0MHDiQjIwM4It8pdvtFsP1ySefpLi4WAgvAL/61a/o06dPWNan\nnmFFRYU8K9U84MILL6RPnz5UVFRI4fl5553HwIEDqaurExlPTU09oVg+Unng75SCUgjeCGWpZmdn\nc/311wP6xbFmzRoOHTrE/fffD8DFF1/MjBkzwpoI/ToEU6KVglKeSGpqKi6Xi/b2dqEtu93uiCgo\nTdNwuVySt0lPT5eWKgo2m41BgwYxZMgQWYOmabS3t/P6669Lse7kyZPJz88nOTn5lARSvX/79u2U\nlpZiNBqF6KKYREajUZhgGzZsoLq6mqqqKimEbWlpobm5GbPZLGvs1asXra2teL1ekpOTgdMnm6jD\n29HRQWtrKwkJCdJp48orr2TWrFnk5OTI56tCVY/HI17Dxx9/zIgRIxgyZIh8riqo7SoMBgPNzc0A\nlJWVEQgESEtLE4UQyT6FgUBAjIJt27bxhz/8gcrKSrm0fD4fbrc7pGxCWd1qjyKNmJgYaVcEep5O\neecK6r5QZRj9+/cnPj4+7PtmMpm477775NkcO3aMpUuX4vP5hBQ1adIkLBYL27Ztk3zTxx9/jM1m\nY9asWTz00EOAbhyFS3mq59DR0cHdd99NXFycdEaJi4ujpaWF3/zmN1IqcPHFF5OVlSVtmUAv9dE0\nDYvFIufQZrPJ3RZOfCcV1PEwGAyYzWaxbIcPH056ejrNzc3SmeDll19mzZo1/O53v5NLp7sbyAb/\nq2kazc3NeL1ecZ1VCC6c3+fxeHC5XDQ0NAi1Vl3+zc3NIohFRUVMmDCBSZMmyT76/X7a29uJjY0V\n4Wxvb6e6ulqUwamsBfQaMaPRSEFBgYTKhgwZgtVqpbGxkXXr1gGwcuVKDhw4gMPhkOdkt9vp0aMH\nqampYmjU19fz4osvMnXqVOnPePnll5/WgVYeXXl5OXFxcQwaNIiRI0cC8L3vfY/MzMwQReP3+/F4\nPBw9epRt27YBX9Ctg2tJgpuTdhWK4q7aZ02YMIH9+/cDhCircMPv98v+HD58mPT0dHr27CmhseLi\nYhoaGggEAnK+HA6HNGtV+9PZ2SneSySgws9NTU00Nzef0FJJtRxTodKOjg46OjpOSZ6/CQwGA4WF\nhTzzzDOAfrabmprYvXu3GKRr164lLi6Oqqoq6WwRExPDuHHjuOuuu+Rshks5BRsZvXr1YvDgwSGs\n0/b2dp5//nm2bt3KOeecA+jRjbS0NCwWixge7e3ttLW1YTKZQozwSLQgi5IkoogiiiiiOCNxVnhQ\noGtwlatwuVzMnTuX9PR0cfFLSkpobW1l586dnHvuuYBOaTabzRHv9Oz1eiUer7yJDRs2iHen3ORw\nWb/B40jKy8vZvn075eXlUj1fXFyMy+XCZDIxY8YMAB588EEyMzNDqN2aptGvXz+uuuoqoYGnpqbS\no0ePU7KUDAaDeDyZmZmkpKTgcDhkHEljYyM7duxg37594g20trZK+EglbO12O7GxseTk5MieqQF5\nPp+P3Nxcee1Un6nf75d6EPVcTCaTyM/KlSvRNI3CwsIQksjhw4f55JNPpAtBXl4eWVlZEStqXL9+\nPaB7IrGxsRQVFYn1H84GrMcjEAhIN2un00lhYSEXXXSR5HfWr1/PBx98gMFgIC8vD9D3saGhAb/f\nL/Vjf/3rX7ntttsilp9Ta3zqqafEo1ayGhcXR25uLmazWULOGzZs4Kabbjph6m84oPoOgr5/iYmJ\n9OrVS9Z19OhRNmzYQGtrqxC88vPzmT9/Pjk5OWG/lzwej+yPGtXi8/mkVuvBBx8U+Zo/fz4Aw4YN\nk2bcCnFxcaSkpISQPsLR6PpkOKsUlMoDFBYWkpmZidVqpV+/foDeJdjlcvH888+zYsUKAK644gpG\njx4dkdhp8HoqKioYOXIkNptNwmwLFizA5XJht9uZPn06EB4FpcZqqFqUxYsXs2XLFurr62WMht/v\np2fPnlx99dWSo0tISBAlGixoNpuNUaNGSYFxjx49TqtDvLrUZ8+ezerVq9m7d68wwFwul5ALgqFq\nm9TzSU5OlvENwZ3CjUYja9askYOvigy/KTRNo6amRro6HzhwAIPBIHkV0OvWamtrmT17NgUFBYB+\nyK1WKxdccAH5/38svZqTE4nD6vf7JWSkaRopKSlcfPHFohwVYSOcDMLgvJYyeoINAKUUExISyMvL\no1evXtIdxWQy8fHHH4cwSINHvYQbmqbx97//HdBrntTaleGyYMECMjIy+Mtf/iIXsepuru6JSEH9\nZrPZLPuTnZ1NamoqFRUVQlSYPHkyRUVFYQ+XKVZzSUkJoDNnY2NjcTgcfPTRRwDs27cPm81GUVER\n48aNA74ITx+vLG02Gz6fTxRUpAyOs0ZBeb1eli9fDugkgEcfffQEMoXZbKayslJ6XDU1Ncmoi3BC\n0zSOHj0q7LyJEycCugWjvnvTpk34fD769OkjXZ/DYTH5fD7KyspYvHgxoLf0NxqNuFwu8WIuv/xy\nbrrpJoqKik5anBjcpUFV2qt2KKfbVVm9p7CwkDFjxvD5559LHiBYOam/y8rKYvTo0fTv31/aNsXF\nxTFy5EgSExMlj+H3++X9iiHpdru/MRVe0zScTie7d+8Wz8nj8eB2u0lPT5fktRpjYbPZpDOB3W4X\nDzwSoyyOh8PhCClQjo2NJS0tLaS/oyJuqH08XZk6GW1eGQAbN25E0zTxdgF27dpFU1MThw8flv3Z\ns2cPgUAAo9EoF9iECRMiFrHweDy88MILAKKok5KSpNh6zJgxdHR0kJ6eLnnWhoYGPvzwQ4YOHdpt\nBCp1vmpqati1axcOh0PKK2bOnHnSAvXj8U0IMcF5b4/HQ1NTk9Dw6+rqsNls2Gw2yXVdfvnlOJ1O\nRo8eLWtsbGwkKysrJPrj9/vx+XwhRKtIyf1Zo6CMRqMkcdetW8emTZuYMGFCCONqx44drFu3TqjV\nQ4cOPa2anq+C3+8XCq4S+IKCAvx+P0uXLuXhhx8G9MsmNjaWn/3sZ2K9dfXgKjp5WVmZhM/MZjMD\nBgxg6NChzJkzB9AvieP7xQV/hhJO1U/QbDaHKLLTEUb1nvj4eH7zm98wadIkNm7cCOiWW3t7O/36\n9ZMRA2PHjj2Bjacuu6/7/lNZn9/v5+jRo2zatEkuf1WSEDy+oWfPnowZM4bx48eH0Ki7C0p+g5ub\n5ubmiueroCjvwQ1/T8e6PRlTVnmtaWlprF27ltLSUtkfpbz8fr+E2quqqsRjUl6MYrCFGyrEr0gk\nyiC98cYb5fJXryclJUkkQzVJdjqd3aKgAoGAsFI//PBD1q9fT25urvR8zMjI+EZy9U1kPLjllt/v\np6WlRbx/TdNkQrUabtm3b19RROosKKMtJSVFvGCj0dhtffmiJIkooogiiijOSJw1HpTNZuOHP/wh\nALfddhu//e1vWb58uVh927dvFwv08ssvB+Dmm28OuxV87NgxHnvsMXbv3i31BYcOHeIf//gHS5Ys\nkaSyzWbjvvvu44YbbgirF2c0GunTpw8jRowAdMt/yJAhjBgxQijBxyc9gxHckUGtK5xV40ajkcTE\nRL7//e9z9dVXA6HhpNP9ntPtU6bCn59//rlYiGazWWiz5513HqCHaefOnXtC89rugppsrJp2JiQk\ncOmll2Kz2UKGAQaPhIfQEF1XRpQYjUYpML3vvvtwOp3SiUB9ntfrpaamRvaxs7MTTdOIiYmRGsVI\n5HsVgkO7qmF0YWGhREyUl7Vz504hSXR2doqnF2moUFtwLVtKSgoTJkwIW8Pok0F5kz169JBBljNm\nzKCzs5PMzEzxHE0mEx6PR0pTQJ++nZOTI9OaFTIyMkJCfJGqwTtrFJTBYJBJnvfeey8rVqxg8eLF\nEnrw+Xz06NGDhx9+WJpHhrMoVl0Sn3zyCVu2bMHtdkuro+XLl8vYdUU2+K//+i9uvfXWsCong8GA\nx+OhR48eEs5ra2ujoKCA2NjYEIWj/j5YORyPk5Emwolw5m1O5TOOV2YFBQVMnz5dLq2WlhYaGhoY\nOnQoc+fOBfT8RXDNSHfDYDAwffp0SktLAT3/NWfOnJBkuvo3OAwarJyObyJ8qkxMpVxyc3N57LHH\nsNls/Otf/wL0ejQVSlJKC3RD7OKLL+aWW24JWWO4EQgE6NGjB7NnzwZ0pmp8fDxbt26VommPxyNs\nUVXXZzAY8Pv9Ea3NUuvzer0cPHiQNWvWAHr+a+TIkUyfPl2UxDcl2Jzq8zOZTOTm5oZ0JVd3lroP\nFCFJ0zRZjyImtbe3S2gyMTFRmgpEOvdqiORIgFNE2Bbi9/tpbW1l+fLlQo8uKirimmuuITExMWIM\nK4ClS5dy//33U1NTc8KMovz8fGGYTZw4MSJtXxwOhyhl0IUvWPjVv990DyLZneDbxPFduGtra8Wy\nVfOzMjMzhbEYKWbeN4XKCxw5ckT+f//+/YmNjQ2Zk6VyiMraDZ739VXGyOmsR3lRoBthlZWVNDQ0\niKI3m82MHj2an//85zJ9IFJ5O6UAFFN19erV7N+/H6PRKGUlXq8Xp9PJX//6V6FWm0wmbr75Zu69\n9145j5F4zpqmUVVVxRtvvCEMWzWupV+/fiFKIpJK/GSvfdX3KYOjo6ND9kcxVU8hsnLaP+isVFDf\nBtQ+1tbWcv/997Nq1Sp5LTY2lgsuuIBHH31UKMGRoGX6fD4qKirIysqS71atZ4I9oVNRTooRdrYp\nqePl3u12h4SrrFarTBc9E6DCQ0rxKDal0Wg8YaSKYl9CqHV8NkMpZ/UMm5ubqa2txW63i3eUkJBA\nS0sLq1atYtOmTYDOZps3bx5z5swJGbgYTpo+6PL10ksvsWbNGvFiLrroIqZNm3YC0SWc33+G4LR/\nzNkttVFEEUUUUXxnEfWgziJomsaSJUu44oorQqzB45/xl1nTx7v7mqbhcDhCOnOfZZadIJher7zN\ns/W3/jvgy0JXKhSoavA8Hg8JCQkR85aVTNXX1/PTn/6UhoYGbr/9dkAfPZKenh4SoVBkjeMLdb/j\nofZoiC+KLxBOYVahk7M9RBRFFJGAul87OzvZvXs3SUlJwiwOVk7q71wu12kXw5/BiCqoKKKIIooo\nzkhEc1BRRBFFFFGcXYgqqCiiiCKKKM5IRBVUFFFEEUUUZySiCiqKKKKIIoozEmdNq6Mooogiim+C\n4wc7nmWMubMKUQUVRRRRRAThbK0UjnU4HA6OHj3K3LlzZUBgbm4u9913H+edd963vs7uQPCMqEAg\nELFBg+HCWamgVFuYqqoq9u3bB+hFcfHx8d+KtaRpGh6PB03TQsbAx8TEYLFYwj5qOooovk2odkN7\n9+4lKSmJ/Pz8M0K23W43K1eupLm5WQp1Bw0aRP/+/c+I9XUHVJ/Ek800OxP3IOrbRhFFFFFEcUbi\nrPKglPtaV1fHddddx44dO8QqGD9+PM8++yy9e/eOuBel3GfVVXzbtm28+eablJWVceDAAUDv9Gw2\nm/H7/TK76cEHH6SwsDAiM2Gi+O4juB2T3++XseZqjMqZ0NS3sbGR+++/H4DS0lKuv/567rzzzojO\ngfqmsNvt1NfXSzdu9dq3vWeRRiAQwOfz4XK55G7RNA2n04nX65XXfD4fNpvthNZm3+b+nFUKqrW1\nFYCf/vSn7Ny5E5PJJG1FOjo6ePfdd5k3b550N47kxvt8PlnP4cOHqaurIz4+XkYuqwFvBw4c4L33\n3gPg008/5YknnmD69OkRjw37/X6OHTvG9u3bAcjPzyc9PT1kbtSZmjwOBAJ0dHQA8Oyzz5Kamsp1\n110nozG6OvTQ5/Ph9XqpqKhgx44d8lpOTg6HDh2SfWltbeXYsWP06NFDRhG43W7y8/OZNGmShFHC\nMYRRrUmNKj98+DCvvfYa+fn59O3bF4BJkyaRkpIS1hljpwK/38+uXbvYuXMnADU1NRw5cgSfz/et\nKii1jx6Ph+TkZMaOHSsjy6+99tqIjC/3+Xw4nU4ZmW40Gtm+fTuDBw+mV69eAPKsIj1ew+1209bW\nht1uFwOns7NTBkouW7YMgB07dpCVlcV5551Heno6AGPHjpXUyLehqM4aBeXxePjFL34BwLp16xg6\ndCi/+tWv6NevHwD79u1jyZIllJSUyKTbSHsqyoPq3bs3M2bMoKCgQC6ThIQEfD4fb731Fo8//jgA\nFRUV/PSnP2XixIly2YYLKi8HsGnTJp544gmKi4tljlVubi4TJ07k5ptvlvk5JxvQF8mmlce33VKe\nqFpjR0cHe/fuZdGiRTIor66ujuzsbM4991wmTpx4yt+l+qBpmkZDQwMAb7/9Nq+//jp79uyR8RZu\ntztk1AUgoy6CG3v6fD4MBgO9evWSg9+nT59T3ge/3y/KaO/evZSWlvL222+zf/9+QB+qqBqLKuWY\nmJjInXfeyX333fetKCmn08nvf/97ysrKAH1/vF6vPEf4dvIcwSQJ9czU2JvU1NSwr0kZUDt37mTF\nihUArF+/no6ODlJSUkQeioqKuOSSSxgwYECITIUbRqOR5uZmUlNTZUir1WrFbDbT0NBA//79Af18\n1dXVsXLlShlOuGTJEi677DKmTJkixn53Gq5njYJqaGiQ8NnYsWN58cUXyczMFOEcOHAg1dXVLFu2\njNGjRwORVVBqtDlAdnY2eXl5ZGRkyGvqIV933XXk5+cDMHPmTGpra/nzn//M3XffDYRvuFxVVRWX\nXnopoCtCv9+PzWYjOzsbgDlz5pCdnY3VapUkt9Vqlcvl+OGLXV1X8Gd6vV5aWlqoq6vjnXfeAaCk\npISamhoqKyvFWzKZTNjtdtra2mQkdSAQkIvnVL9fwefzUVVVxe9+9ztAH75XW1uLz+c7YdaSmloM\nX0wgNZlMsmdKkblcLlFQ99xzzyntl8/n49ChQzz44IOA/rzq6upobGwUCzh4XSrh73Q6eeKJJ5g9\nezaDBg06pf3oKgKBANu2baO0tFQMiry8PMaNGxcyn+rbUFBqPXv27KG6upoePXrIiPWsrKywXbjq\nN/r9fhwOBx0dHRw+fFhemz9/Pv3795foiGoge+jQIfLy8gCdtBE8qTYc63E6nSQnJ5OYmCivmc1m\njEYjdrtdpnz369ePRYsWUVNTI0bG5s2bef/995k9ezZPPfUUoE8i7y4ldWbGcKKIIoooovi3x1nh\nQSkr+j//8z8BmDJlCmlpaSHTRmNjY9m7dy8tLS1iAUcKKmykvqepqYmsrKyQ8dzKQjKbzUKSsFqt\n+P1+8RjCgUAgQEVFBcOGDZPPTUhIYPz48cyfP5/zzz8fgKSkJBmrodamaPFut1ssqoSEBHr27Nml\n5LKayXPw4EEAFi5cyLJly2hubhbPCPQ9On6stNPpxO/3y2vx8fHMmjWL0aNHn9J61N8q2Vm5ciUr\nV64EdG9chaaOX0twOC8mJoa8vDzi4uJkf5xOJ5qm4XK52Lt372ntTXt7O6+88gpbt24FdI/Kt3R7\nnQAAIABJREFU4XCE5CtSU1OFZKNG1Xd2dtLR0cGtt97Khx9+CHRfOMbn87Fq1Sq8Xq9ECQoLC2lr\na+PYsWMSWupuBAIBGQO/fv16cnJy6N+/PwMGDAD42vzT6Xh+RqORpKQkPB6PRC0GDBjAqFGjQnJx\nTqeT2tpaNm3aREVFBQBlZWVMmzaNxMTEsHmbsbGxxMbGYjAYTph5FhxFstlsJCUlUVtbS01Njayx\nvb2df/zjH+Tk5ABw++23k5KS0i2ydVYoKNDDaJdffjlwche0paUFv99PUVER8fHxEV+PyWQiOTkZ\n0A9vZmbmSUOKwUJotVrp7OyUWoVwoK6ujmnTpkn8G3QSye233x7C1jmZsAUCAVpbW1m9ejUvvPAC\nAGlpaUydOpXbbrvttPIcilG0ZcsWfvzjHwNQXl6Oz+fDbDZLcrZv375MmDCBoqIiOVQffvgha9eu\npaGhQcISN954I/fcc4/kYb4pVNjHYDAQExNDQUGB7IUK5QUzmGJiYujVqxfDhg0TOevVqxdOp5OX\nXnqJPXv2hHxuIBCQMNKp7k9HRwfp6enk5uYCuuz27duXoqIirrvuOtmfmJgYTCYTS5cuBfTn6nQ6\n+eyzzyQ/FQkCwMmgaRq7du3C6/UKCcBsNtPZ2Sl5uW8DXq+Xbdu2AXoub86cOYwZMybkOYUr/Hiy\n96empgIwZMgQMerU98XFxZGXl0d6ejq7du0C4K233uLgwYPce++9QrI5XZxsjLx6LdjIU+erpaWF\nzs5ODh48GBJCBz1/t2rVKkAn44wcORKr1RpxJXXWKKjgIV/Bm6ZyAzfffDN+v5+rrrqq2xhF6gJP\nT0+X/30y0oESBkUdnjx5cpcPi/Le7rrrLg4fPkxcXBx/+ctfAJg8ebJ4JiejLQcTA/bu3curr74q\nFl58fDwXXHDBaed8fD4f7777Lj/72c+oqqoC9OeVmprK+eefz3/8x38AcMkll8iBrq2tBWDr1q20\ntbVhNpslj/jAAw+csnICQpRRIBCgX79+FBUVAfr0U9BzlMry79u3L/PmzWPKlCmiRGtqaliwYAEf\nfvihJJWVF5qQkMDUqVOBU7v4DAYDmZmZjBgxQvY4NjaWyy67jNzc3BPkyGAw8IMf/ACA4uJiFi1a\nhNvt5vPPPwfotlxUfX09u3fvJj4+nqysLAAOHDhAcnIy8fHx3xo54uDBgzzwwAOALrsdHR0cPHiQ\n6upqAIYOHUpOTg4JCQknXePprFsZYQcPHpTnFUw4CvbezWYzMTEx8tqhQ4cIBALU1NQImSISijMQ\nCOB2u2lvbxfluGLFCpYuXUp7e/sJSttkMsmerVixApPJxJAhQ2Ryd6QU1VmjoIItoWCtf8sttwA6\nc23w4MEMGzYs4odFCai6BDVN+0o6qWL7qWS3OuBd+X4VctqwYQOBQIChQ4cyadIkAFHQKpmr1mgy\nmfD5fPJeTdPYs2cPgUBAQiK9evXCYrHg8XhOSTEoRXjs2DFWr14tYRfQFcEFF1zA7NmzueiiiwBE\n8N1uN3/4wx8AeP3113E4HOTn5/Pwww8Dp+8hBD8Ls9lMZmYmd911l3zmwYMHycvLE09EESZ27twp\nz+m//uu/hEwR3N/NarVy5ZVX0rt379Nal8ViYcyYMYwZMwbQvbevkh8lZ08++SRLly7F6XTy3HPP\nAfDiiy9G1MpVZ+2ll16ipaWFXr16ibJubm4WL/TbIEk0NjYyffp0uVgTExNZvXo1JSUlYhz169eP\n66+/nsGDB4sshWON9fX1HDlyRAzFjo4O7HZ7SNoBvri3FEni3HPPZdu2bTQ2Np4y+/PLEKwQlfHZ\n3NzMokWL2LlzJ4cOHQJ0D9PlcqFpmsiMuitMJpMY0kuWLOGdd97hrrvuYv78+fJ3kXi2UZJEFFFE\nEUUUZyTOCg/K7/eH5G3i4+Npampi3rx5bNmyBdBzJ3/+85+7Jf+kqrSVxWGxWE7ooBwMVSzrcDiI\niYkRb+V0YTAY5LvdbjcWi4XBgweHhDZVPFrFuRUpwOVysXHjRgCpnXjkkUcklm42m8nLyzvlsFpw\nLqdv376kpKSId9K/f3/GjRsnuQu1HofDwa9+9SteffVVAFwuF3a7ndGjR3POOeeEfO7pQln4MTEx\nnHfeeQDcdttt7N+/n/T0dNauXQvodNvnn3+ehoYGKcBW1mgwLBYL559/Pj/60Y9OK/QIergkPj5e\nLO1vSutPSEhg2LBhbNmyRbzg9vb2sCbcj4fywJcsWYLJZCI9PZ3GxkZAl5+1a9fy6KOPnlDjFmlP\nqrm5mYkTJ0poGnTvfePGjdjtdnk2NpuNkpISLBaLhEO7mrfz+XxUVlZSWVkpr/3jH/9g8uTJISSS\nlJQUCWOr8/XDH/6Q7OzsEEp4OPdKeUYHDhxg5cqV7N27V8hTSp6tVqvsQUZGBh6Ph/b2dtra2gD9\nbNpsNjZv3szcuXMBJOwdbpwVCkpVQyuhO3z4MH/729+orq6WjXvzzTcpLCyM+MHQNI3du3dLaA3g\n4osvJj8//6RFroFAgDfffBPQBTEvLy8siW1VW5WWlobD4cBut4sABrcxUTFy5f5rmibvLSoqorCw\nkNjYWNlbj8dzWjm84HzKLbfcwtSpU0lISAB0pWcymUJqsPbs2cNf/vIXli1bJq8ZDAaSk5OZPXt2\n2DptqGeiappAL1p2uVysWrWKf/7zn4AeLnK5XCdctGpdaj0ZGRlcc801XZK10w2LmUwmxo0bx+7d\nuyUMuX79eqZOnRqRjgWBQIDi4mIA2traSE1NlZo70EPWTqeTiooKMYQsFksIEzLc61HFzYMHD6a+\nvj7kzKnwmqoBBJ2xuX79eiwWixT1dzVcZTQaSU9P56qrrhIjzOFw8P7772MymeS7Z86cic1mQ9M0\nIW2YzWaGDBkiBKtwIlhOBwwYwIABA9i9e3eIoWW320lMTJRu77GxsVRXV+P3+0OIJV6vl3379oWc\nw0gU8X/nFZSy+r1eLx999BEAH3/8MQ6Hgz59+rBgwQJAF9hIxuKVIK5YsYIXXngBm83GqFGjAP2h\nKxKCgnqQHR0drFmzBtAF+8orrwzLOlUnip/85CesWbOGhISEr2R2qWJKs9nM0KFDAaSFTzCbTSmT\n04XBYCAxMVG6VQQjEAgIZfpPf/oT7733HkajUdiHAAUFBV32ME+G4GLS2tpalixZwuLFi8VD+DLF\nZDAYsNlsQqaYNGkSU6ZM6XIR+Mm+7+vg9/s5evQoBQUFUj4QGxtLa2trRFogORwOYRCqzgR79uyR\nnKrq0LFq1SpR/v3794/IOVT0/OHDhwOIcjKZTOKxDBgwgLi4OAYOHChGT11dHW1tbdTU1HxllONU\nYDKZOPfccznnnHPkOarPPv63+/1+XC6X5MlKSkro6OggNTWVjIyMsKwnGOocp6enc8MNN1BdXc0n\nn3wC6B74+eefT//+/aUEZPPmzTQ3N4f0gVS/sW/fvvJcFTko3PjOKyjQH35LS4v0TXM4HEyaNIn7\n7rtPqL6RbP3i8/l47LHHAD0prWkao0aNEossOTkZk8mEx+MJuej9fj+vvPKKWH1ms1loxF2FUiJX\nXnklHo+HtrY2jhw5AuiJYaWklNCpWhuDwSAHQ40CCVZQ4bKQTuZJBgIBCYts3LiRQCDAsGHDhNxR\nWVlJQkKChEPCBTUORVHF582bR3l5uViMwTAajXIQ4+PjhQ2plGj//v3JyMjocpPN4xPp6rtPBrXO\nf/3rXxw4cICBAwdKbV1KSgoulwuXyyXPPBzeVCAQ4PPPP5dw8LFjx/B4PNLfDXQZVOM21DOLVPcW\nv9/Pr371K2F8KsNh/PjxjBw5Uv6uf//+jB49Wu6DlStXsmbNmpCz2VWcyrNXYXa1noaGBoqLi6mt\nreWhhx4CiEgNmclkYsSIETz66KNs2rQJgBEjRpCTk8OePXvE2K+vr8dsNqNpmhi4oBNO8vLyQu6F\nSHhQUZJEFFFEEUUUZyTOCg/KYDDQ1tYmRImMjAx+9KMfcc4554TkS1SdT/D7ugq/389bb73F008/\nDXwxiLBPnz5cfPHFgB5mCaacg+6xNDQ08Ne//lU+KzU1lZ49e4bVCsnIyGDmzJls3749JNzg9/ul\niwLAZ599RkJCgljcoBc/H1+wGilomsb69eu56aabAD25n5qays0330zPnj0B3atKTk4OS44ueC+O\nHTvGiy++KOHgpqamE0Jsqm+ZzWYToo0KzyhyAsCYMWO6nMNQ+cCWlhYAqqurqaiooE+fPkI9tlgs\nOJ1OqSsD+MUvfkFCQgJpaWnSlDgmJoaKigrKysrIzMwEQj3o04XH42HNmjWyxuD6OSXrNpsNm82G\n2WwWDyoSXfpV3mvRokUhRbDXXHMNDzzwgPToLC0tJSsrK2StBoOBgoIChg8fLt5dd1LhjUYjNptN\n8r6XXXYZtbW1GAwG6eZQUFAQ9jUZDAasViuFhYVCDrFYLLS0tLBw4UIpblY5Z/X3gLxvzpw5smeR\nKiM4KxRUfX09W7dulct/2rRpDB48WOoO4IswjsfjEZc5HHmChoYGbr31Vql3MJvNDBo0iAceeEAu\nBHUo1cgE0BPIb7/9Ng0NDUJAuPTSS0+b+fVlMJvNcqkpxdPe3k5MTAyaplFSUgLoyfRAIMDEiRND\nCgQj2b1cwev18thjj/GHP/xBQh05OTlcddVV5OfnS16qvr6eMWPGhCVMpMJira2t/PrXv+Yvf/nL\nSVtMqfWkpqYSHx+Pz+eTOh/VIaFPnz6SFxs1alSXQ2iaplFXV8e9994LwEcffURbW5sYFfDFRRAX\nFyeXrdls5uKLL+auu+6SPJCaQbZ06VImTJgA6EXrXSXj+P1+4uLi5DMSEhLQNA273S4yrkLEI0aM\niGhHC03TWLp0qfxm0Aurb7zxRqqrq3n22WcBnehSWVlJUVGR3AFut5vMzMywydXpIJislJ+fz9y5\nc/nggw8k9JaVlRUR9rHBYMDhcAjrcv/+/Tz00EMcPnw4hDihaRo2m02MjIKCAu6++24GDhwYkpc6\nnbzp1+E7r6A0TaOqqorY2FguueQSAO6++26SkpJCLKWOjg7efPNN9uzZwz333AN84SGcLgKBAK+9\n9pp4TaArx9/+9rdkZmaGsOUUe0gVqNbX17NmzRqcTqcUpc6fPz8iFqbZbCY/P1/GSbS2tvL5559z\n+PBhGQewfft2bDYbPXv2lJi9YrdFinGlEtUPPvggCxcuJCYmhilTpgBwww03EBcXx9GjR2XEhNls\nln6GXV2Tkosf//jHvP/++3R0dMhnWiwWbDYbiYmJQn1vbm6mqalJWveAbv1mZWUxadIkbr/9duCL\nfGNX4Ha7+eijj1i/fj2gP6+TdVYH3dhQl9eYMWO47bbbMBgMkuRet24dxcXF7N27VxR9IBDgf/7n\nf7q0RqvVyqxZs8TK3759O83NzZjNZnleHo+HzMxMcnJyIt4Sp7S0NGSPjh49yi9/+Uuqqqrkd5tM\nJiorK2lqahKPNy8vj8LCQpKSkr6V+WcqqqNkJiYmhiFDhvDxxx9LznjXrl2MGjUqrHeDpmkcOXKE\nm266SeZ3tbe3S7lJcK48KyuL0aNHi6d17bXX0rt37xA2pvodwUW+4cB3VkEpQWxoaGDZsmVkZmby\nwx/+EEBqCDo6OqQO6pVXXqGkpISePXt22ZVX3+1yuVi+fDmJiYkMGTIEgOeee076xAVDJRlV+/3i\n4mKqqqqwWq3S2+10erd9U1gsFqGO1tfX88EHH3D48GHxJDo7OzEajUyYMEG8uEgoJ8UGqq2tlXES\n77zzDkajkeHDh3PDDTcA+niUbdu2UVJSIiysxMTEkLYwXYFSjuvWrROPSP3ulJQUCgoKyMjIECWh\nejkqZhhAz549Oe+88/jP//xPIUmE4xIJBPRpzCrs43a7Zb3KYlUXiSIiAAwbNoxPP/2UjRs3ipL4\n5JNP2Ldvn4QyQWezdZU0pPomKi+vuLiYVatW8dZbb0nYWI1RifTFbzAYGD58OMuXL5coQVtbGxs2\nbBC2GSBz1sxms7Acs7OzyczM7LaehcFQZR1q9IX6LXa7nUGDBvHaa68Beqh0+PDhYZMt0IlkN954\nI5s3bz6hns9kMonRPHDgQO69914GDhwo90d6evoJ+xUpYzZKkogiiiiiiOKMxHfeg9qwYQPbt28n\nOztbagkMBgNNTU28/vrr0gkA9D5Xjz76aJernpUV29raSnt7OyaTSTpPd3R0SOPJYE/L7/fz2Wef\niTtdWlqKy+ViwIAB0uwz3Pmn46Gs5piYGC6++GKcTqeMm4+Li5NhaV/V4fx0ETyc8PPPP+eWW26R\nDhqg03+nT58ulltxcTHNzc2kpaVJT7uUlBT69u0bllyBopQH919TXkpzczNOpxOHw3FCV2eLxUJB\nQQGgN+K9/PLLyc7ODgnndhU2m42LLrpIyiaOHj2K1+vFbrfLb3e73bhcLqxWq3iYGzZsYNOmTdhs\nNskrNDY2SgPaadOmAeGrRVIdOEDPvVVUVLB8+fKQ/o7Hjh3D7XZH1EMxGAxceeWVvPfee5K3UbTx\n7Oxs5syZA8CMGTMkHKr6Xar6rW8jvKdydWazOSR8q2maTLcFPQwZrjIZJeN33HEHmzZtOqGUwmQy\n0bt3b6688koArr76agYOHBjSjPtkpKlIpCbgLFBQ5eXlVFVV0d7eLsnQzs5O3G43jY2Nsqnz58/n\n4YcfltxUVxBMvDjnnHMoLi6WinpFjgiuG7BarbS1tXH48GE50Ha7nalTp1JYWCjhwUg1XFRQn223\n28nOzqa9vV0uPBWy6Woh7pdBPa+6ujoWLFjAnj175LDYbDbS09Nxu91SB1VZWYnVaqV///5SOJyW\nlkZCQkJYLhNFYMnMzKS2tha32x3Scd3pdJ4wikHV1TzyyCMADB8+XEZehBMmk4mCggIJnzmdTkpK\nSoiNjZXkfmJiIo2NjSEdBzIyMrBarcTExMhIc9V9Y9asWaJYv6xz9+kguIXVrFmz+MMf/hDStFlN\nllVdQyIh3waDgZ49e7Jw4UJ++9vfArpSHzlyJHPnzpWp0cFF5+Gu6zsVBE/e9Xq9mEymEJZvVVUV\nq1evlvznhRdeGJZ1apomM8+WLFlygnKyWq2cc845PPDAA4wfPx74Qqa+jX2C77CCUpfUmDFjWLx4\nMc3NzWI1JiUlUVhYyIUXXsi1114L6F0Rwm0lpaWl8aMf/YjXX39dPJHS0lLa2trQNC2kwE5RbtU4\niZkzZxIfH4/dbv/SURzhRvDn79q1iwMHDgidtKamhvr6+rDHkYMvfdCZQh999BEulyvE66ipqaG8\nvFwukwkTJpCVlUVqaqrQW4OLZLsKld8ZPHgwhw8fprGxMeRiDW6BBPqlfs011/DII4+EsDMjdeGa\nTCZhUz799NNihCkPID09nZiYGDwej1w0sbGxeL1evF5vCCNReZ3h9PJOtua4uDheeOEFZs6cCeh5\noBEjRnTb/LWcnBxRUKo7/7d1sX4ZggeZ7t27F4fDQf/+/eV8VFZW8tZbb+F0OmX8S0pKSlh+hxo/\nAoSw75SR2rdvX5588knGjh0rd9e34VkG4zuroNQDGzduHC+99BKNjY1SLzNo0CBxSSN1GEG3yMaM\nGUNRUZG4xB988AGVlZXYbDap5s/Pz2fQoEGkpaWFhJSOpwx312FKS0sjEAiwcOFCabUPugWVmZkZ\n1nUYDIaQ4WgHDhygtbWV+Ph4Ef7ExERGjBjBtGnTxJtMT08/oc2S+rxwQH33z3/+c2JjY/n73/8u\nl7rf78doNBIXF8cFF1wA6A1kJ0+eTExMTLcc2mDqcWZmJpmZmSd4dMeXAQQrWKW0jEZjRBVTMIxG\nI0OHDmXJkiWAPr/r0ksvxWazdYtsB8vKt32xfhWUMiovL2fDhg24XC5RWoFAgOrqajIzM6UmMFyh\nf7/fLyH0uLg4HA4HFotF5pY98cQT5OfnR6xX4ungzH2KUUQRRRRR/FvDEIniqtPEGbOQsx2q+8At\nt9wioyMyMjJYs2YNAwcODLv1pHq0gW5Vv/3225SXlws1Oycnh5tuuimkeDTYAo5UHRbo+UqPx8Nn\nn33GsmXLAL2rRlFREbfddpvkAbrLcwo3uqPQOopTg/Kg3nrrLX79619jMBiEtHHdddcxYcIEkpKS\nJF8djgngqrxD1UK+8847HDp0iBkzZkiD3QiOcD9tAYwqqH9TuN1ulixZIm1gpk6dyvjx4yNymfl8\nvpAx6l6vl7a2NjmUqoVQ9CKN4t8JikjldrtFGXVXPrqbEVVQUUQRRRRRnJE4bQX13YtZRBFFFFFE\n8W+BqIKKIooooojijERUQUURRRRRRHFGIqqgoogiiiiiOCMRVVBRRBFFFFGckYgqqCiiiCKKKM5I\nRBVUFFFEEUUUZySiCiqKKKKIIoozEt/ZZrFRhAeqUFvNoPH7/VLVbjKZzqjGkVFEEUX3QtM0fD4f\nzc3NMr9t+PDhpKWlhWUu29chqqD+TeH3+6murmbx4sWA3oX90KFDtLS0iNIqLCzk6aefZuTIkSFd\n2KP47kD1YFP/G8I7tiT4e7xeL1VVVWzcuBHQ22llZWUxefJk6cgdlZ/vFjweD2vXruXRRx+VWW1T\np07l17/+tUyPiCSiCqqbEdxaSo1FUI1UHQ4HlZWVdHR0MGzYMEAfRRHugXiaplFbW8szzzwjE37b\n29sxGo1y0QAcPHiQBx54gN/97ncyOiRckz1PBcFjJODESy5Sl56maTJRt62tjfLyco4ePUq/fv0A\nKCgoIC4uDrPZ/K1evMfLVPAzdLvd1NfXU1ZWRk5ODqCPMklMTAzLnCY11mPfvn38/ve/Z/PmzTQ1\nNQF6I16LxcIVV1zBiy++CCDjHroDai+C5cbn8+FwOEKmS0dqGuyXrUeN1mhtbZXnpP5taWmhra2N\nfv36kZSUBOh71t2zrdRz3bNnD3feeSdVVVWyjzU1NaSmpnbLOqI5qCiiiCKKKM5InBUelApjBA9p\n+ybDCrvLIlH5nSNHjkj38IqKCux2O4mJidJ+f+3atVgsFnr06CGjKM4555ywWXjKAmpvb+fll1/m\ngw8+oK2tDdC7jI8fP56kpCRZ48GDBzly5AhPP/00f/rTn4DwTff8ujW63W7q6upYt24d//rXvwAo\nKSnBZrMxaNAgLr30UgBmzJhBcnLyCYMNuwq3201paSnPP/88AJs3b5aBhir2PmzYMEaPHs2Pf/xj\n8Ua6yxpXCAQCIj8ej4empiYOHTokQwPLy8tJTEwkJiaGtLQ0AA4dOsRVV13FDTfc0OXvPnr0KAC/\n/OUv+fjjjwFCBiS6XC7Wrl3LBx98AOiTpCMpP+oOaG5u5vDhw9TU1MiU5piYGHbu3Mny5ctlL+67\n7z7y8/MjfhdomkZ7ezuffvopb731FqBP1HW5XOTk5Ejet6qqioaGBnw+n4zZePbZZykqKpIhrBDZ\nu8vr9fLJJ58A8Kc//YmWlhY0TSMxMRHQn2E4RoB8E3znFZTf76etrY09e/bIa7GxsXR2dtLW1kZZ\nWRmgj1K2Wq3k5+eLezp06FD69OkTsUtFue2LFy/mj3/8I9XV1Zx77rkAnH/++UyePDnkwg8EAqxd\nuxa73S4HKJy5AnWRrVq1iuXLl9PU1CT5iYKCAqZMmSKhPNBnxixcuJAdO3awdu1aAK644oqIhLQC\ngQBut5sFCxYA+qFsamqSCwf0Q2m1Wtm9ezfLly8HdKX+wgsvEBsbG7bD6/P5KCkp4d577+Xzzz8H\nwGazkZ2dzeTJk+Wgtra28sknn/DGG28wd+5c4IuQbHcYP2rPdu/eDcBf//pX9u3bJ2QXgPj4eFJT\nUxk6dKjIfWFhIbNnzw7LGtQ8MYfDwejRo/ne974n/23JkiWUlJTQ0tIi8jN9+vSIhYkDgYCEy//+\n97/z97//Ha/XK+HylJQU9u3bx+bNm+U9PXv25KGHHorIHRAcat26dSuPPfaYjFwHXWEaDAaMRqMY\npE1NTbS0tNDe3i7vvfvuu3nyyScZNWqU5IIjtYc+n48FCxbwz3/+E4C6ujoCgQAWi0VG0F999dXd\nNhvtO6mggq3GnTt38pvf/Ibt27fLAzWZTGiahtPpxO12A/rGBwIBrFarXB6pqam8+OKLTJkyJewC\n6nK5uPnmmwH45z//idFoZNKkSbz88suAngcwmUwh47nPO+88ysrKGDBggFwm4RLEQCAg492feeYZ\nDh06RCAQkO8ZPHgwgwcPJisrS75z5syZrFixgvLyct544w0AJk+eTHJycljWpOD1etm4cSPz5s2j\npqZG1qsOr7IuU1JS8Hq9NDU1iafV0dGBpmlhOTBKWa9atYqbb76Z9vZ2+vTpA+h7NmbMmJC8TUdH\nB++++y7t7e2sX78egIsuuoiYmBjMZnNED7GmabS2tvLII4+wcuVKQN/HjIyMEAU0duxYkpOTRdYU\nwsXASk9PB3RZueiii8jOzpazWVZWxo4dOwgEApSUlAD6uQhH7utkCAQCEhHYuHEjBw8exGQySS4n\nKyuLbdu20dbWJjLe3NyM3+8P+/lXJKRnnnkGgOXLl+N0OsnMzGTgwIHydxaLBavVKiSSgQMH4vP5\n2Lx5s5zXmpoa3n//fYYPHx5xmXr99ddZunSpPMPk5GTq6urweDyiHBMSEiK2huPxnVNQSjkpS+Th\nhx+muLgYTdMkAZuUlITRaMRqtcrlbzKZiIuLw+1209LSAuhC9NlnnzF58uSwrvHo0aPMnDmT/fv3\nAxAXF8ff/vY3Jk+efFKFoy6Oo0eP0tDQQHZ2tghDuOByueSwVFZWEggESEhI4LLLLgPg5ptvJi8v\nj7i4ODkEubm5nHfeeZSXl1NVVQXoZIFwKSh1CN5//32uv/562tvbxXhITEzkwgsv5JJLLmHw4MGA\nbnG+9tprvP7663KhzJw5UxRYVxCswG+44Qba29sZN26cKOa0tDTZF/W8EhISmDFjBgeWDlYYAAAg\nAElEQVQOHJBpvI2NjVx66aVCSAg31He3trbyve99jwMHDoiRceuttzJjxgz69esnZyHcoc/jocJn\n8+bNk5IERSxxu90i7+qMhVuug2EwGMTIaGlpwe/3YzAYaGxsBHRlpNam1jVixIiwK6dAIMCRI0e4\n9dZbJVRmNBqZOnUqN910k0xp7ujowOl04nQ65Rnl5eVhNpvJycnhzTffBPSzO2rUqBADIxLo6Oig\nqamJCRMmCBGoubmZxx9/HIvFQmZmJhA+4+abIEqSiCKKKKKI4ozEd86DAt0y+/DDDwGora3Fbrdz\n7rnnMnHiRAAmTZpEr169JLQBugWjaRoffvghjz32GAANDQ1icXUVyro5cOAA48aNw+l0MmbMGACW\nLl1KamrqSS1ZTdPE01KeQXBeLBzWr6ZpFBcXS5jFaDSSlpbGVVddxY033ghA3759TyjKDQQCDBw4\nkJSUFLHSw0WSCAQCQke+55576OzsxGw2c+GFFwJ6SK1Pnz6YzWbxtOrq6khISCA/P5/8/HyAL/VK\nTxVer5dp06YBunfSu3dv3njjjZPmAtXvNxgM+P1+Fi9eLPVkVquVIUOGkJub2+U1fdk6QfeW9u3b\nR1JSEq+88goA48ePx2q1dlt+wGAwiJyqEFUgEODYsWOAXlvn9Xrp0aMH1157LRD5MgXlTWdkZEj+\nbfz48YAeoThw4ECIJzJw4MCw71dbWxtPPPGERHZAz93+9re/FUIP6OfS7XajaZp4liaTCZfLhdFo\nFBnPyMiQwthIeMNqjSaTiblz5xIIBCQsWlxcTGxsLH6/X8LG3Ul3/04qqGCBGjRoECNGjODOO++U\n2K5iuxy/kYFAgPPPP18uD4/HE5b8UyAQkCK222+/HbPZzPz58/nd734H8KWMF03T2LNnD/fccw8A\nu3fvZtasWV+qzE4XLpeLd999V/JxJpOJSy65hEceeYS4uDgA2a/guhGbzUZGRgYdHR0S/gqHQg8E\nAjgcDh555BFAj7HHx8czbdo0/vd//xfQw6IqP6dCsuvWraO5uZkhQ4YwevRoAEkuw+kfnEAgwIIF\nC6ioqJDv3rhxI+np6Sf9TLU/Ho+H1atXs2jRImH4JScnR6yAUdM0/vGPfwDw4YcfkpGRwRtvvCEk\ngO6ulQmGCq85HA5++tOfAnD48GGsVitz5syRy7a71peZmcn48eMZOnQoU6ZMAeD5558nEAhgMplE\nblQoKxxQctHY2MihQ4cwGAxCinrqqadIT08/QRmqcFmw0iovL2fNmjUSPh0wYAC9e/cOySOGcx/V\nZ9ntdnr06IHf75fvycnJwW63k5KSwqhRo8L2nd8U30kFpWkaffv2BXSLaezYsfTp0yfkYX/ZA9y1\naxeffvopACNHjqSgoKBLa1EW489+9jNAv9yee+455s6d+5XWYiAQoLy8nBtuuIF9+/YBupIYMWJE\nWC2lQCBAQ0MDR48eFQU1e/ZsnnrqqZPmA5SSAl2xbdmyBZPJJMKpLOWuQNM0Dh06FJKjGz9+PL//\n/e8lAasKGj/77DPeeecdQGc4ZWdnU1BQwIQJE4AvLPKu7FdbWxvPPvusfMZPfvITsrOzv9TjdTgc\ngE57//3vf4/D4RAvr2fPnmE3MBTKysr4+c9/Dui/+/HHH2f48OHdTm0/GTRNo7OzkwULFghpw+fz\nCUsukrknhUAgQHt7O6AbV3369OHqq6+WnOm4ceNYvnw5brdblHokCofVGRo2bJjkfTMzM0/qqR2f\n12xsbOSOO+7gyJEj4rFMnz6d1NTUb1Q6c6oINkhVoX4wcWvnzp1omkZ2draQW6Ie1FdAXVxKsDIy\nMsRt/jrrwufz8dxzz0kIYuTIkcTGxnZpwzs6Oli4cKHQ2a+44gqmT59+AmsqeA2gX2633norBw8e\nFK8kKyuLoqKib/Rbvin8fj8VFRXU1NQwdOhQgK+9MNR6Nm/eLKHUkSNHAuGp8/H5fHR2dgoDLDU1\nlRtuuIGUlBQ5GK2trbz33nu8++67ooRGjRrFBRdcgM1mE8+vqySAQCBAWVkZra2tEh6aP39+yH8H\nfR+9Xi/79u0Tivv7779PWVkZmqbJGtPT0yOSRPZ4PNx2223U19cD0KtXL6ZMmdJt4byvg8PhYNmy\nZTz//POiwE0mE/fcc4/Q8iMNn88nNXNut5t+/fqRlpYmsu5yuSREGglvIFgOBw0axFVXXSUe1Fc9\np2D24Zw5c9i/fz+XXHIJd9xxB6DfcUp5hBNKphVUGsThcEj5wJ49e8SjUgauYth2B84M6Y4iiiii\niCKK4/Cd86A0TaOxsZHa2lpAp5Q7HA5iY2PFcrXZbCdoeE3T2Lx5Mx999JGEkb4uDPd1UFbFrl27\nJLTQp08f/H4/brdbKK2qp9s777zDn//8Z0Cnere1taFpmqx7zJgx9O3bN6yuvMPh4OWXX6aurk7y\nNhkZGV/695qmSUjtoYceorOzE7vdLrH6cKzL7/fT0dHBoEGDAL1IeMyYMfh8PrHcXnrpJZYsWUJW\nVpZQ4WfPnk1SUhIul0u8vHCsp6GhAa/XK9bkM888w//8z/9gsVjEY/n0009Zv349W7duFXKH3+/H\nbrfjdDrlGU6cODEiHlRHRwdWq1XIKuPHj+/WvnZfBmXVt7S08NZbb9Hc3CzewtixY/mP//iPblmH\n3++nqqqKuro6QM8tDRs2DJvNJjK1fPlyfD4fZrNZwn4qomE2m0NCXV2Rq+zsbK655hpyc3O/9n5R\n3pMqcN65cye9evXi/vvvp3///sAXUQtN07ocWVHnpq2tjbVr17J06VKJRsTFxWG328nMzKS8vBzQ\nSV/Hjh2jpKSERYsWAXDvvfd2mwf1nVJQmqbJZqkwQm5uLkajEYfDIWQEJWDBD7S1tZUf/OAHeL1e\nqavp0aNHl9ajFJTdbicvLw/Qk55Lly7lX//6F5999hmg1/T4/X4OHTokwjBhwgQqKyvZt2+fCPHc\nuXOJiYkJa9FpaWkpO3fulEMDXy7cgUCAPXv2cPvttwN6DURsbCwXXnghWVlZXV5TMKxWqyio0aNH\n4/f7KS4u5u233wb0QsuUlBTmzZvH1KlTAT0UqGmadM0GvStCV5lhJpMJs9mM0+kE9PYur732GpmZ\nmaLMTSYTTqeTlJQUCU0WFhaya9cuNmzYIGw/FUYNJ1Ql/y233CIhrLS0NCk4/Sa510hBydmGDRtY\nv349mqYJAeHxxx8nISEhIrmTYAQCAZxOJ5999hm9e/cG9JEQiYmJtLe3s2PHDkBnpKm9VDU9TU1N\nJCUlhe3cgf4cVN2cUoAnY+UGAgGqq6tZsGCBkJASExN55JFHGDp06EnD6acb5gsEArhcLp599lkA\n/u///k+a1ap7MyYmhvT0dPLy8qQjR3Z2NlarFa/XK8b17bff3m3Fut8JBaXyEqWlpbz33nuUlpbK\nZZuVlUVMTAxWq1UOS319PU1NTXR0dMhD/r//+z9qamqw2+3cdNNNwMk9rVOBwWAgOTmZn/zkJ2zZ\nsgXQczWNjY188MEHIW1JUlNTmT59Oj/4wQ8AcDqd/PKXv6S8vFxosJMmTQrpdNEVqI7Jixcvpqqq\nCrPZLK/5fD4hYgTPgyotLeWyyy4TD0HR0YcNGxbClusKNE2joaGB9957T5KugwcPZs2aNbzwwgs0\nNDQAMGvWLB544IGQuTMGg4Fjx46xYcMGea9ScqcLg8HApEmTmDhxIqtXr5Y1dnR0YDQahUTTo0cP\nsrOzmTVrlhxOv9/P008/jc1mk9f69esX9stYFZyOHj1aFGZ1dTVvvvkmw4cPp7CwEKBbOlicbG2g\nd9/o7OzEaDTy/e9/H9D7SJ6M8BNuJpqmabS1tWGxWCTno4gq9fX1PPfcc4B+JgwGA/n5+WKcKoUV\nbOR0dV0mk4mKigq2bt0qpQsZGRm0trZisVjk8n/++edZt24dLpdLnuuPf/xjrr766pMqp9M1QDRN\nY+vWrTz88MNs2LBBPisxMZG8vDzOP/98AK666ioSEhJoamqS1nFNTU0kJCTInQp6f8dBgwZ1C/Hl\njFdQfr9fNubxxx9n165d9O3bVy6O7OxsES5lrTQ1NVFdXU3Pnj2F/vv5559jMBhISUmRnlJdFUSD\nwYDdbqdXr15SHW6z2cjPz+ell16SyzYhIYGkpKSQw1pRUcHBgweJjY3l7rvvBnTKdDgul+Amnh99\n9BEul4v6+npp+GqxWEhLS8Pr9Uoft927d1NVVYXf75c1KKV6+eWXS0jkdEkSwZ7sq6++ysqVK2lu\nbgb0nn8HDx6USwb0sJ/qxqD2zOl0Ul5ejs/nE8XUVSMD9P1Yvnw5//3f/w3AwoULsdlsjB07luuv\nvx6AIUOGSIgteJRFz549iYuLEw9TechdRXAfN5/PJ9RoFTmoqKhg2bJlvPzyy8L2uuOOO7ptDILC\nkSNHAFi5ciWappGbm8udd94J6N5AcJ9JBcUUC1fvRKPRSGpqKhdeeOEJjWpbWlooLi4GEDLLf//3\nf8ulbDabw95b0uPxsGjRIkpLS2V/cnNz6ezspL6+Xs7cwYMHJQKjSl+uuOKKLy1LCTYoTwWtra08\n+uijIT0Ic3Nz+dvf/sbw4cNDvs/j8VBWViYG4LFjxygrK+Ojjz4SeXz11Vf5+c9/HjZj+qsQJUlE\nEUUUUURxRuKM96AaGxuljf++ffvw+Xwn1Dwpq0JZT1lZWdKPr7q6Wj7LbDZjMplC+pR1FcqLCu4u\nYDQaSUpKkmrs4HUqK+S1116jvr6e3NxczjnnHCB8ncv9fr90F6ipqZGwhopzl5eX09LSgtPpDBkG\nqIoYlRdw4403csstt5CZmRk2d76zs5Pt27dz+PBhoa2qruoGg0Go3p2dnTQ0NJCQkCDP2u/3k5mZ\nKfRhCN94C4vFwpNPPgno/R2rqqqw2WySb1Ld0tUege699evXD6/XK/3ewtUX0O12i+ympaVht9sx\nm81iaZvNZlasWMHBgwdZt24doOcwu9ODCgQC/PGPfwT05xUfH88dd9whDXZVpw2j0SihQNV3zmKx\nhE2mDAbDCZ+lRvD87W9/k5CawWCgsLCQK6+8MiJEluBuMo2NjRiNRpEVRTxQMgO6Z6zkSqUn9u/f\nT1ZW1pfeTadyZyni1q5du6iqqiIvL49Zs2YB8Itf/ELureAwf01NDSUlJZKjGzRoED/72c/IzMxk\nzZo1su5IdbU4Hme8glITJkFv85KYmEh+fr6E/VRjVZPJJMJgtVqxWCx4PB45GC0tLfh8PnJycrp8\nMIJj6MfXBHxdjYBiha1YsYKYmBgKCgrCPg7b5XKJO69pGqmpqVx22WUSAm1tbQ2ZoKvWrYobVYum\nn/zkJ6SlpZ0QAjmdOgj19z169GDOnDls375dQnwWi4W4uDhcLpeEG+rr63nvvfcoKioK6RDSs2dP\nbDab/F0kKurV/CSPxyMXmUr0Hz8u3Ww243A4JPQWjkLmtrY2SkpKhKl6ySWXSBPf4E78cXFxtLW1\nSaG3OifdBa/XKw1NfT4faWlpXHTRRbJGj8cjRojKf1ZWVhIfHy9khnDheDkwGAx4vV6WLVsml7/Z\nbGbx4sURb3ZqMBjIy8sjMzMzJOQ4cuRInE6nNBlITU2lV69erFu3TsLvr732GgUFBUL+Csbp1h+9\n++679OzZk2nTpjFv3jzgi6a9auoD6C3Z3nnnHSZPnixGc1ZWFtnZ2cyZM0cIQKNHj+62ydpnvIJS\n1hjoNOyysjKampok5q8Sn8EbZjKZ8Pv9WCwWsWidTicJCQkMHjy4ywVvxwuJoq6e7L8Fw+v1yjiJ\nzs5OYmNj+f73vx/2AxPc7UD1sistLZV8XHDnAxVr7t27N5dccgnTpk1jyJAhwBc5sXAqAYvFwuzZ\ns2ltbRVSgt1ux263s2/fPrH6du/ejdfrZeDAgXKY7HZ7xBlhCorZdzL2VXBbmsbGRgwGg8hjOAa5\n7du3jz/+8Y/SMmnMmDHExcVhMplEfl544QVWrlyJ3+8Xi/zcc8/tVhbf7t27JdcZCARITk6mpaVF\naN3V1dU4nU569+4tcjZgwACsVmtEZooFIxAIsH37dpEn0PMuilASCajfU/D/2jvz6Cir849/JpPJ\nZDJkshiWhJAQAgbCInvYQQjIIiJFKtiiR3HfPS7g0dpWTznacqyt4lL3AwhUpC0QsCiLyiqIgRhC\nAmQhC4GshEy2mXnn98f87nUSwpbMDIO9n3N6qhAz77zve++zfe/zJCbSqVMnjh8/Ltt0ZWZmEhcX\nx+TJk+U1iJZsqampcojhN998Q1lZGW+++aZ8/u61ussVmLh3g7j++uvZuXMnGRkZpKeny8+uqqri\no48+knWympoaxo0bx4wZM2SbJdFwICUlRR5u1jRNDSwUGAwGKeG+5557KCwsJCoqSqrK9Ho9TU1N\nzTxb4ekCsrh/yy23cOzYMQYOHOhR9YlIJQgP8UKFQ5He27x5M+A61zJ48GCGDBnilYUqFmZDQwNW\nq5W0tDT5OQ6HQzaoHDVqFOAatzF27FhCQkKayfU9jU6nIyQkhBkzZsgFVFhYSGVlJYGBgdLjLikp\nwWKxEBsbK6MSX7b0ESnH1nDvxXf48GGCg4OlgRKGvz1ERkZSXFwsPf/ly5cTGhrKkSNHOHz4MICc\niTVs2DCpUvPWnKXWcDqdrFq1qtmGWVFRweeffy5l3Y2Njdxxxx3nRRK+wOFw8Pvf/x6n0ymdxyVL\nlvjk841GIw888ACvv/66FGiIzT82NrbZvRBnAoVDWV5ezp49e3j33Xdl+zSLxSLPaV3J9Yv3YcSI\nEaxdu5bi4mLeeustwOUEiTlPQkF41113sXjxYukICkQZwz3b4iuUSEKhUCgUfonfR1ABAQEy4hk5\nciQjR44EfvamRaHRPWpyj6bcJ34eOHCAvn37ntdBuD1omtasA0Lnzp3PO68goqfy8nLZB7B79+4M\nHz68mRTXUwQFBTFz5kwAPvzwQ2pra5vVdwwGg+we/sc//hHAp15uQEAAnTp1krn4vLw8vvvuO+rq\n6qSIpFOnTqSkpNCtW7er0m9OiEZaO8MjPMjTp09TU1NDREQEWVlZAG3ydFsSExPDggUL5EFvh8PB\ngQMHKCkpkVHnyJEjGTZsGPfcc488g+Xr+yTOEIErfS3eM/FnCxcuJDw8/Ko8v7q6OnJyctDr9fLM\n07Rp03wSQRkMBmJjY3n++edl93mHw0FSUhJOp1Om2k0mEw0NDZSVlcnGyfX19WiaRn5+vhQ1DR48\nGLiydem+B/Xo0YNnn32W4uJi8vPzAVeX8vLycmbOnMmMGTMALvqs3H+fL3vx+b2Bgp8Xnjjz4h5i\ntlasa61e0KtXLyIiIjCbzR4t8AUGBuJwOOTGUV5eTnh4eLMDk+IUtxBqwM8FUvH3La+7PRiNRl54\n4QXAlQ//9ttvqaqqkvW4sWPHMn78eHr37u1RRePlItJ8QoyRlpZGUFCQnAkFrkV1yy23+Ewt1No1\nQvN0RsvrcDgcWK1WamtrpfJRdC9pzzWHhIRw1113SWWgmBbd2Ngo0zZC1Xe1Rmw4nU5uv/12VqxY\nASDrhbNnz5bjLcxm81W5NnA5D0ajEYvFItst+SoFKlSKnTt35uGHHwZ+7rhhs9malSLETChRIrBY\nLPTt25c5c+bIQ8dtTW2LzwkODmbcuHHNOuuIddWW5+PLZ3pNGKjWFDpX+t9aLBZZlPRkDlWIMURN\nLDc3l6ysLLp27SrrEuJlsFgs8oBgbW2tx9sHCcSBZHC1JRGti/wJvV4vZfjDhg3j+PHjBAcHy/5j\nY8aMoXv37le1W7eoL4rrbY3S0lJCQ0NlFFNUVCT7KbYVIbcXDow/otPp6Nevn6ypOhwOevfu7dGW\nQW1BrO1Dhw4REBBAjx49uPnmmwHfRpgthVsCo9HYbP8xGAz0799fzkGzWq3069ePmJiYditV3f87\nX6nuPI3OlwWvS+DVC2lqapJqO/Gieuqhud9DceofWn+xRKRls9nk+Y2r5WVebYTXWFlZyd69e3E4\nHFLhFBcXR3Bw8FUdwncpHA4Hp06doq6uju7/P5DvakV8iuacOHGC7du3Ex8fLydt+/OzcRcgtDWy\n8WPa/GWUSEKhUCgUfskvPoIS389ms6HX68+TTyoUil8mLYUuar1fNdp843/xBqrVD/KwKEGhUCgU\nF6TNG+21WTlrJ8owKRQKhf+jalAKhUKh8EuUgVIoFAqFX6IMlEKhUCj8EmWgFAqFQuGXKAOlUCgU\nCr/kf1LFp1Bcq4jWS3a7HavVSn5+Pt999x3gGsFxzz33XLUGu9cCLY/VeEPReyUzm8T/i3/21byz\nawX1FisUCoXCL/mfPKjrCzRNk6MjGhsbsdvtGI1GOXzPV56S0+nEbrdTW1srp2muWrWKvLw8jEaj\nHOg4a9Ysbr75ZtnxXOGfiAjq0KFDPPzww2RlZcmehuBqivzggw+yePFiwDMj6C8Xh8OB0+n0+XDC\nSyHuWXV1NYcOHaK8vJwhQ4YA0K1bN69P9xWIsTsVFRWAq7GwmHQgOq137NiRqKgogoOD/eL+2Ww2\nzp07x48//ihHBU2aNOlKxwSpThKX/OX/H0Y3NDTIMQZhYWGEhoa2Kx0iWqmI++hwOGhqaqK0tFRO\nFl21ahVFRUWYzWY5A2nBggWkpKR4dLqvO2KsR01NDatXr+bbb7/l4MGD8u/NZjPnzp2TzWsdDgdj\nx45l2bJl0mhdDdyvx32uF3BVWta4j0lwn4AKEBoaSlBQkE/TacIYPfLII6xevZr6+vrz0lZms5nn\nnnsOgEWLFsn5TN7E6XRSV1eH1WqVnd3FvbmaG63D4eD06dMAfPXVVxw8eFA6i+Dq9h8XF3fBSdjt\noWUT6cLCQvbu3SvnPIWFhdGzZ086duwo/2zXrl2Eh4fz3HPPyfvoS8Q+KZzrdevW8dFHH1FcXCwn\nDbz//vt06tTpSn6t6iTREofDQX19vTRGdXV1HDt2TEYPALfddhtPPPGEbGt/pbgbJ7GxNjU1kZ2d\nzYkTJ9i7dy8Ax48fp7q6mtraWmkk/vvf/3LnnXeyaNEir7yI4npWrVpFWloap06dkh23582bx4QJ\nE3A4HJw5cwaANWvWsHHjRsaOHcu2bdsA5ChoX9HU1MT7778PwBdffMHAgQNJSEggMTERcA2fTE5O\n9tmcIeFsgKvj+okTJygqKpIecEJCApMmTfKpt9vY2Ai43ilhxN1nVwkDKsbA33bbbSQlJXnk+lqr\nrbSso5w4cYK4uDjA9bwCAgLO64HpKzRNIyMjQxrr7OxszGYzERERcn3U1tby6KOP0qtXLxlteupZ\nuteWsrOzeeaZZzh79iyTJk0CYOLEiSQmJmIwGOjduzcABQUFrFu3jmHDhvlsTIi4Rk3TOHfuHHl5\neRQUFACwd+9eqqurqaqqkkbJlw7sL8pAuYfyP/74I1VVVcTExABII1BdXU1hYSHgMlptHQYmEC+h\n2MjKysowmUwkJSXJIXadOnUiKyuLnJwcjh8/DrgGG7711lucPXuWv/71r4Bnx380NDTI65kyZQqh\noaHMmTMHoFnU2LNnTwCGDBlCfHw8b775phyytnLlyjYb77Zc85o1a3jppZcAl3GIi4ujQ4cObNiw\nAYD8/HxeeOEF7rzzTp9dk3BwTp48icViYciQIdKo79mzh0GDBtG1a1efXA8go6G5c+fS0NBAdXW1\nNOD19fUcOnSIqqoqGeVt3ryZpKSkdn+uezRwIUNlt9vRNI2ysjLANSDQfco1uOZqiZE33tx4Gxoa\nePrpp1mzZg1WqxVwbawpKSmEh4dz4MABAA4ePMiWLVtkFCWutS1GquV9EQMJwbXv9O7dG6PRyL33\n3gu4RsqIeyBSfA899BCZmZns3r2bKVOmAN5N0zqdTun0fP/992zduhWDwcC4ceMAuPHGG8nKyqKo\nqIiJEycC+CQiFyiRhEKhUCj8kl9MBGW32zl06BAAn3/+OREREUyfPl1OJdXpdBQVFVFeXi6FADfd\ndFObvLiWY8A1TZMRVEREBB07dkSn0xEfHw/AhAkTsNvtMmoC+PTTT6mpqWHdunX84Q9/AOC6665r\n25e/COPGjcNkMtG3b1/ppbX2nYODg3nsscc4ceIEGzduBODrr79m+vTpHr+m1sjPz+fJJ5+Unn//\n/v159tlnKS0tlWm/8vJy1q5dy4IFC3ySUrPZbNJ7veGGGwgKCkLTNHn/CgoKqKmp8WkEJWonCxcu\nZO7cuej1eun519bW8sQTT/DFF1/IFJaov7QHUZMQv1PTNPR6PXq9vll6KCMjgzVr1shU8oQJE6ip\nqaGiooI9e/bInxs7dizDhw+X2Y32ZjHcEZmDefPmsXXrVsCVigWYOXMmN954I5mZmbLmY7fbCQsL\n84hQorX/XvxZfHw8qampDB48WE7Sbq2+GhoayuOPP87y5cs5duwYAP369fPK+65pGpWVlSxZsgSA\nwsJCevTowbx58+Q7/cMPP1BcXIymaYwaNeqC39Nb/CIMlKZpZGdns2rVKvnvkyZN4vrrr5dps7Nn\nz5KZmYmmaVLB079//yu+2e7zZcS/w88hutPplOmLlqF5SEgIv/vd7wBX6m3t2rXyPAtAZGSkRx6+\nTqcjJCQEgJSUFOx2OyEhIZc0xgaDgUWLFrF+/XoAPvnkE6ZOner1HHhTUxOLFi2itrZWjqr/5z//\nSbdu3bDZbFRXVwOu59qnTx+fLBBN09A0TT7DoKAguSmLZx0aGirH1vsK8d2NRiNRUVFA89RSdnZ2\nMweqV69e7bpfTqeT06dPk5mZKZ9DQ0MDZWVlxMXFSQN45MgRVqxYQW1trTSYXbt25frrr8dsNksF\nqc1mY+fOnTzwwAPccccdgGtdeOKZ2u12Xn31VQC+++47AgMDuffee2VKLSQkhJycHHJycmQ9ZejQ\noVK96p6aa7nO24JOp5Nrp2PHjkyYMAGTyXTR9aTT6ejbty81NTUsX74cgFdfffsxCeQAABTnSURB\nVNUrAo78/HzuuOMOWZOfM2cOTzzxRLPac25uLsXFxQwYMEAaVl9yzRsop9NJVVUVX3zxBfn5+QDc\nfvvt9O7dG71eL72+FStW8Nlnn2E0GmXR1BOSaveXUFzPhUY2uxst95+rrKwEkNGeJxCGWafTYTAY\nLstL1el0hIeHSwVPZmamLMR7A7GRZmRksG3bNvR6vTTgIvpcvny5zJEHBAQwceJEn3lwDodD3kdx\nD9yfbXFxscfqhm3F/TjDqlWryM7OBpD1z1tuuaVdv7++vp6lS5fSoUMHWY+rqanBaDRy7NgxiouL\nAfjxxx+lKlTUXUpLSxk6dCjx8fGyLpWbm0tpaSnp6elSLBAfH9/uKErTNNLS0vjHP/4BuIzo5MmT\nefzxx6UTcfToUdatW0dhYSHJyckA/OY3v6Fjx44ejeLccXcoLlfxaTAYqK6ubuaYeWoNijVXXl7O\nI488wvHjx5k6dSoAr732mhQgiTX36aef0tTUxPz5831aexL8IgxUVlYWeXl5REZGAq7Ugl6vp7Gx\nkYyMDAA++OADqqurmTx5stz82sLFwnhxPWKBuv+ZkCgLT3Lr1q00NjZKb9zTiGu6UgVVRUUFdXV1\ngEsx5s3o6dy5c4BLMl1XV0f37t25++67AZdBqKur49///rf8+ZCQEFJSUrx2PQIhmXZXLonCubsE\n12azSUPgK9yPM9jtdqqqqvjwww8BWLZsGU1NTQQEBDBhwgQAGZG2FZPJxJ133kldXV0zEUFCQgJ6\nvV4KRtLT03nmmWcoKiqS7/+YMWP429/+hs1mkxteWVkZVquV6upqioqKAIiOjsZoNLbrXausrGT1\n6tXy3bVYLNx8881UVlby5ZdfAvDNN9+wf/9+unbtKlVz4eHhF/xcT0RRggs5ra0hrqempgZwrUNP\nOULCYd+0aRP79+8nJCSE119/HXBlgcQ7LpzmgoICLBYLc+fOvSpKTCWSUCgUCoVfcs1HUHa7nQ0b\nNpCTk8PTTz8NuMLp+vp6srKyWLRoEeDy3CwWCyNGjGh28t7TCNFETU0N33//PeCKlhISEhgwYICU\ncJeVlaHX6xk3bpw8AOet67lcnE4nBw8elN7amDFjvJZOczgcvPfee4ArlRgcHMyiRYtkqsVms7Fn\nzx5KS0vln40aNQqz2exRz7Y1Ghsbefvttxk+fDixsbFA8/uYm5sLuCT6vpLhA82OM2RlZXH48GE2\nbNjA9u3bAVedVafTERUVxZ///Geg/QIEnU5Hv3790DStWY1G/E9EkHFxcYwdO5bFixfLmuqbb76J\nxWKhrKxMpvjq6+tJTk5mypQp8t6Jg7NtTWU5HA4KCgro0KEDw4YNA1wy/FmzZqHX62UHBJG96N69\nOwMGDAC4qDjCPb3t6+jBarXKIylNTU0eidSdTqeMeNeuXYvVamX06NEyBep+nm7z5s2AK4qbOnWq\nzE75ml+EgSosLKS+vl6+ROXl5ezbt481a9bIAmBQUBCDBw+mqqqKU6dOAa7Cpac3GPGQKysrpTrv\np59+AppvMIGBgcyaNYv33nvPJ2mi1jqGtFyYTqeTtLQ0+edz5szxmiEoLS3lnXfeAVwGwWKxEB8f\nL9Maubm5LF26FJvNJp/RvHnzWv0enkL87k2bNpGWlsaUKVPO+/6aprFjxw4AkpOTvVa7aA2bzSYP\nf69evZpvvvmGkpISmT4T9cbU1NRWDWtbudiZJfdUclhYGI899phUYgYFBVFbW8vSpUvZtWsX4Oqe\nMG7cOBISEujcuTPgWgvuLZKuFCEseOONN6RzZTAYZLpKGC2TyUSXLl1wOp3yXOSF3ifhBHmy9nO5\nz8Jms1FZWSmvTTxfTyDWV1FREQEBAXTo0EHuSWKdNTU1sXbtWsB1bx977LGr1nz4mjdQDoeD6Oho\nNE2Th+82b97MTz/9xKlTp+QL26dPH371q18RFBQkVSreuul6vZ7o6GjGjBkDQE5ODmfPnpUKP4DO\nnTtz3333YbFYvHINF6O1heJ0OqmpqWHfvn1SJtwWlePl4HA4eOutt5pJoM1mMytXrpSfV1RUxIED\nB9DpdFJaHRkZKfu9Xey7tBUhUV67di1nzpxh/fr1sobToUMHgoOD2bhxI1999RXgki37CqfTSW1t\nLVlZWYDLMauuriYgIEAWr0Wvx9TUVJ8XtEWNtby8XB73OHnyJFu2bOHLL7+U97F///4kJyeTmJjY\nzCtvz3MU78iF6sNCDCWippKSErkviIP2rSlzhZy+PYh6nIgOL2fPycvLIy8vTzquQpHrCcT3CQ0N\nxWQysWfPHp566ikAHn74YXr16kVxcTGlpaUAdOnShRtuuMFjn3+lXPMGKigoiN/+9rfs379ftuc4\ne/Ys9fX1GI1G+vbtC8DixYuJiYnB4XBIz82byp3g4GAWLlwIuNRDO3bsoKGhQS6EiooKdu3aJQUd\n3uJCrf+FmEMU/M+dO8dTTz1FXV2dVFd16tTJK+m0xsZG0tLSpOdmMBioq6tjy5Yt0lsMDAykqakJ\nTdOkgbLb7bLIeyXf9XIRvzs6Ohqr1cqmTZv417/+BfysiissLJSbbWxsrG/PhLgVygsKCqSwQAg5\nwsLCiI6OZuDAgXJj9OTzu1j0qmkaFRUVrFy5UqaHTCYTFRUVGAwGoqOjAZcRFaq99nZuuFICAwNJ\nTEwkJiZGfp7dbj9PSNTY2Iimae0WJggnAlz7lDA0F3tPGxoaePnll9E0jX79+gGeM1DuZzOfe+45\nPvnkE9LT06UQafv27XTv3p2QkBCZFk1NTfVpGrslSiShUCgUCr/kmo+gRKPFnj17ypPXf//73zEY\nDISHh8u+bdHR0YSGhmI0GmX6w5teW0BAgEyVvfDCC4SGhnL06FF5gr2+vp5ly5Yxbdo0hg4d6rXr\nEZ60u8xVRE9Wq1XWU9asWcOuXbsYNWqUPIvkjSaoTqcTq9Uqz9KAS+qblJREbW2tjKCio6PZt28f\nTqdTepBdu3aV3QsuJfdvCyJKWrJkiUxX7d+/H3BFvAUFBWzYsIHBgwcD+KzbtKhdlpWVybN+kZGR\nxMTEkJCQIDvkm81moqKicDqd0gOOiIhAr9d75DmKCNNut8uakYgGbDYbBw4cYNeuXbLOISLgDh06\nyN5u06ZNIy4uDrPZ3O4Uu/hsu92OTqe7ZMQjRB16vb7Z4Edxb8T3O3XqlKzhtRW73c7BgwfZtGkT\nAAMHDmTIkCHN+mSKlKzJZJLTBz7++GN2796N2Wzm2WefBTxbihDR0NSpU0lNTSU3N5dly5YB8O23\n35Keno7D4Wh2L+x2u89GkrTkmjdQAQEBGI3GZudTSktLsdlsDBgwQHZWDgkJwWQy+exgpXsn45CQ\nEJ588kkiIyNlG6E//elPVFdX88ADD/D1118DeFQpIxZgU1OTLJ67pxbsdju5ubn88MMPgGtBTZo0\niVdeeUV2KPDWC9nQ0EDv3r1lV/ARI0YwZcoUbDYbVVVVgKtu9/333xMUFMT48eMBV1cEUfz2Bu6H\nKsGlTBPvj8PhYOfOnWzdupVZs2bJn/PmohXPsKysjMzMTN5++21ZG+jRowcDBgxg8ODBUgUqUkrb\nt2+Xm+DUqVNJSkpqd5pI1CjB1TWitLSU3r17y80zPDycM2fOYLVa5WYrhEsDBgzg/vvvB1xp45Yp\nvfamZLOzs7FYLMTGxra6mbt3XBf7hftnnzlzhvT09GaNpa/kPautrW12ntHpdFJRUcHLL78sn1dR\nUZEUPggnIygoCLPZzMiRI+Vonj179hAdHU1DQ4Ns5OyNdywwMJDAwED69Okjz0Ht3LmTxYsXc+zY\nMfmZVqu1WcsvX3PNGyhBU1OTFEnU1tbSuXNnUlJS5AYTHh7uMw9A5ONFL7D4+HiSk5MxmUxyoaan\np7Np0yYKCgrkQcL58+d77BpFJHLw4EESEhKk0YGfR5GYzWbZp2zQoEGMHDnyku2W2lvnETLoJUuW\nSG8uLCxMPh9hoDIyMggMDCQsLIwZM2YALqGCp6KBtlz35s2biYyMlF2dvals0jSNzMxMwDXT6ciR\nI5w7d07eM71eT5cuXTh8+LCUDp84cYKKigp++uknaaDWr1/PG2+8wdChQ9td6xTOXWVlJTExMURH\nR8tsRFBQELNnz+add96R4hdN0wgNDeWll16SNShPPruTJ08CrpZcY8aMoUuXLhetlwjDKAQdAFVV\nVXz88ccUFBTIQ+C33nrrFdXFWlPhVldXk5mZKR2DgIAAamtr6dGjh3QoMjIyyMvL48svv5TR+NCh\nQzGZTERGRvqkjZZOp5P3bOjQocydO5fNmzdL9bPRaJTCJBVBtRFx8j8nJwdwWf0hQ4YwYsQIWdD2\n9s11P+GfmZnJypUr5XmZadOm0bFjR5miArj//vspLCzkyJEjshB/2223eawgKU7U79ixg5MnT8qm\nseCKYBoaGtDr9YwdOxZwnbwXikL3Yrh4Od29UCELbms0ajQa6d+/fzPBg4jqxKZTWFiIzWYjPj5e\nqq/cNxhfD9qsqqpix44dLF682CfHAqxWK7feeivwsyRYp9PJLMHhw4c5dOhQszSbGHnR2Ngo3zOn\n08nu3btJTk6WXn5b1oL7macbb7wRk8l0nsET01fdn8348eMZNGiQV9bfa6+9BsC+ffvQ6/VMnjz5\noul79/sj1JDvvvsuVqtVikvg556Yl3vNLX9Op9MRGxvL3LlzpWEeNWoUXbp0ITg4WF5jREQEw4cP\n57rrrpMzlg4dOkRFRQWjR4/2mbRbPK+6ujopJBGfHRERQV1dnewy0dr39SZKJKFQKBQKv+QXEUFp\nmsbXX3/Nli1bAFd6KykpiW7duvnE2judTurr6wHXGZqlS5dSWVkpa0qJiYlEREQQGRkpo5jExETi\n4uI4evQoR44cAVx1l+TkZI8Uj4WnXVZWRkZGBidOnJD92cxms+zULX7OfYSC8IxFjaioqEgW3VNS\nUqQn3dYISkQD7l6+EE/s3r0bcB3UtVgsTJw4UXqX7mkXX6cbVq9eTWBgIKmpqT6ZcHry5El5/0VN\nx/2fhdfr3ghYdD4wm81ydMuCBQuYNWuWR+pl4nNEQ9GWZ4eKi4tluhFcqdu//OUvXqv7ivVVWlrK\n4cOHKSgokHUbIYRwL/jn5ubyn//8h7S0NEpKSgCIiYlhxowZLFiwoNn4j/beq5CQEF555RX5vEQn\nDvezVSK97v5eBwcHExcXR1RUlKyVefNdd28eUFBQQF5eHhEREXJf0Ol0lJWVER4efl7E7JOjAV7/\nBC/jdDo5d+4c69evl0Xc5ORk5s+f7zNBhM1mk52e09PTqa6uRq/X06dPH8DVEicqKkqmF8CVo+7W\nrRtxcXGyNUxWVhY9e/b0SEFSbOqxsbEcOXKEzZs3y9pAly5dSEhIICIiQm4oMTExUlAhrjErK4vs\n7GxWr14tz09MnDiR2bNnExkZ2a7rbFkkdzgcVFRUyDRt165d6du3L7Nnz/boQcUrRWxuGzdu5L77\n7vNIB/zL/dyWBty9dgKuzax79+7NZp5Nnz6dAQMGyLRoSEgIBoPBo0a1Zfd+cNWAV6xY0azzx/jx\n49vVmPlSPPTQQ4DrYP7Ro0d58MEHpVpQOI01NTXyncrOzqampgaTySTriE899RRDhgwhKCioTc7P\nxRSlooM5uPYIIRpxnzTg/vPgEr9kZmbSuXNnn6T4nE6nfKdKSkqorq7GZDLJPSksLAybzSaVmy2v\n29tc8wZK0zS2bNnCtm3b5MuwcOFCn83pEXJb4c09+uijjB49mqioKLlxmEwmLBZLs9x8ly5dmD9/\nPqWlpbL1kqZp8jBqewra7t0X7r//fnr27MnOnTtl93CHw0FJSQl6vV5K4cWsl8LCQtk3LT09nYyM\nDEaOHCnrU7feeitxcXEeMaLusveGhgZKS0ulvHf48OEMHDiQXr16nedo+LJgK0Qb3bp1Y9asWT7Z\nNHQ6HYmJiYwcORKAbdu2yS7l4rmGhoZy991389BDD8nOKO7Rr/vv8ibinT5+/DirV6/GZrNJh+Km\nm27yqpMo3pUXX3yRDz74gJycHCmZFtJo93tiNBqZMGECzz//vJwJd6n5TJfiUvdX/L3BYJAd6Fv+\nvXsEZTQaCQsLo7q6Wqo4vX2Qv7CwEIB169Zx6tQpYmJipGijZ8+ehIeHn6d89BXXvIESQ8oaGxub\nTbD1VYFRvGDCs3aXJrvPEGqJ0+nEYDAwc+ZMWbD1ZPpIfGZoaCgzZ85k+vTpMg3p7i26X09gYCDx\n8fEyghIbTH19vZz707NnT49vOpqmcfr0aQ4fPixTNKNHjyYiIgKDwSCvRyxUXy0Qu90uBSxxcXHN\nhtp5m5CQED777DPANWuprKyM+Pj4ZqrU1p6jrxEGSnRvMRgMch3++te/9uo6FL979uzZpKamUlpa\nKnv+HTlyBJvNxqhRo6Q6r1OnTgQHB/usc4U7ortMS3FPy+swmUzExcWRn58v1X5hYWFeu1738UAN\nDQ3Y7Xauu+46acBFeeJy2zR5GiWSUCgUCoVfovO1XPcitOlCysrKGDRoEFarlRdffBGAxx9//KpM\nf7xSNE2TqS1wRTui28XV9ox9gfDcGhsbyc3NJSgoSBaqRXQgpMHwc7Tq7s1diRz4SnA6nRQVFclO\nEklJSSQlJV31Cbr+htg/SkpKmDZtGmfOnOHBBx8EXB1UroV16G+IHpniHffmPXQXSTQ1NckDvAIR\nbbYzemrzAr3mDZTdbufAgQN07NhRphbUJuL/uKcW6uvr5cwb93Ng7gV4gVis7gvGWwbK4XBINVN7\nJ77+0hEqTKfT2a7zVopfJP+7BkqhUPgHQg3myxlZimsCZaAUCoVC4Ze02UCpnIVCoVAo/BJloBQK\nhULhlygDpVAoFAq/xJ/kbkryo1AoFAqJiqAUCoVC4ZcoA6VQKBQKv0QZKIVCoVD4JcpAKRQKhcIv\nUQZKoVAoFH6JMlAKhUKh8EuUgVIoFAqFX6IMlEKhUCj8EmWgFAqFQuGXKAOlUCgUCr9EGSiFQqFQ\n+CXKQCkUCoXCL1EGSqFQKBR+iTJQCoVCofBLlIFSKBQKhV+iDJRCoVAo/BJloBQKhULhlygDpVAo\nFAq/RBkohUKhUPglykApFAqFwi9RBkqhUCgUfokyUAqFQqHwS5SBUigUCoVf8n/GLU9FtNIbMQAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8595dc3668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Interpolate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGW1JREFUeJzt3XuMXVXZx/EHvBdFVChUoEABRbkUFFqw5SLaUiK1oiQE\nNRVBbEADeAkoKeAlQbwEEqUajS1QJQpWI2gbFBAKSEGUkspdpFRLaQtYqlTwyvvHGx6+a3n2YWY6\nMz3nzPfz16+dOWdm9tp7n5X9rMtmzz77bEiSJEkREZtv6l9AkiRJncPOoSRJkpKdQ0mSJCU7h5Ik\nSUp2DiVJkpTsHEqSJCnZOZQkSVKycyhJkqRk51CSJEnJzqEkSZKSnUNJkiSlFw/jz3IT56G32SC8\nh+00tGyjzmcbdYeNbSfbaOjZRp2vZRv55FCSJEnJzqEkSZKSnUNJkiQlO4eSJElKdg4lSZKU7BxK\nkiQp2TmUJElSsnMoSZKkNJyLYPeMZ59tvS7nZpsNxtq5Um/h9dLXa8drSZI2HZ8cSpIkKdk5lCRJ\nUrKs3Ecsh/3jH//I/O9//zvzS17ykswvfelLi9dbJuu/Z555JvOSJUsyX3DBBZlvueWWzP/6178y\nv+IVr8g8c+bM4n3PPffczFtssUVm2+iF1WVhnv9/+9vfMi9dujTzz3/+88x33nln5te97nWZzzzz\nzOJ9991338y8riRpuPB+95///Kfl/2+++eYtc0R3f6b45FCSJEnJzqEkSZLSZk2zB4fAsP2gwfDf\n//63+PeaNWsy//rXv8781FNPZd57770z77nnnsXrX/ayl2UewkfNg/HGm6ydWJaMiDjxxBMzL1y4\nMDPLzX2ZCVuXJQ877LDMP/3pTzOPGjWqf7/wwHRdG7Gc8te//rX42rXXXpv50ksvzbxs2bLMf/nL\nXzKz7LLllltmPuKII4r3Pf/88zNvvfXWmYepTNN1bTQQ7a6dpuPcYWWyjf1lNmkb8ZhzSMw///nP\nzC960Yta5vrfdTmzL7rkWtqkbcR+wN///vfMTzzxRGa2A+9VEf87vOw5PPYdcE21/AV8cihJkqRk\n51CSJEnJ2crAmZecBRsRcfbZZ2devnx55l133TXzMccck/kNb3hD8XqWlfU8llBOPvnk4mtXXnll\nZrYNsWTMx/ss09RDBG666abM8+fPzzxr1qzMHfCof5Pi8bv55pszz549u/i+VatWZeYs/le96lWZ\nDzjggMyvfvWrM69cubLx5z/55JOZOat5pLfLQDSttPDwww9nvuuuu4rX7LLLLpm32WabzGwLlszq\nkmeHlc06Btvi6aefzrxo0aLMixcvzsyhNnXp/8ADD8zM489hHGvXrs283XbbFa+fOHFiZq4OwGt3\nJLRju6F1/Nx58Yuf7y5xCBKH2tTDbl7+8pdnbhoiwOuokzYD8MmhJEmSkp1DSZIkpRFfVuZj4699\n7WuZzzvvvOL7WAJgiXirrbZq+f+DWUauy6IDmZnWSfgY/49//GNmznytcRbYIYcckpklS85ivv76\n6zPffvvtxXvx+y6++OLMnB09Ehde5rXA2eGf//znM69YsaJ4DYdV7LfffplZpnrjG9+YecOGDZnv\nvvvuzJy5HFGWY3q1nDWUeI2tX78+87x58zJfffXVmR999NHi9Tz/eS9ju4wePTrzlClTitfzXNhp\np50yv+Y1r8lcl6J7UV2y5HCNG2+8MfOFF16Y+cEHH8zMGbL1zNff//73mXlP4+oCHLbD/4+ImDt3\nbubx48dnPuGEEzJPmDAhMzcMYIk1orfasi/7v3OTBbZL/bnPY862bxomNZTHtb/30e7uZUiSJGlQ\n2TmUJElSGpFlZT5q/+xnP5v5m9/8ZmbO6osoyyycccnFrllKGczSb7eXkWt8pM4ZyXXZhOXI733v\ne5nHjRvX8n05s4+vvf/++4vvY6mGMzbZ5iOlrMwhC3/6058yc8YkZyTvsMMOxesPPfTQzNOnT8/8\nyle+MnPTzGOWIuuSF68x9Q3LYTznTz/99Mycqc9znO0VUc5YXbduXcvM6+WXv/xl8XqW3SZPnpyZ\nQxS4ogPvcb00jIClxIiIOXPmZObQJV4jPBZsI5bxIyIOPvjgzBzesfvuu2fmzOV66MAdd9zR8uef\nddZZmbmxA1eT2H///aPbNJWL63sPh5DxNfx8Yhvxe+pzl2Vi9jvYLvw8qhfR5nXYtIczfyb/v91w\ntLp83Upv9TokSZK0UewcSpIkKdk5lCRJUhoxYw5Z7+dYj4suuigzx4fUY87e/e53Z37f+96XmeNm\nOBau3biZpjEKfZlC3ws4FuL1r3995uOPP774vg996EOZuRwGjwffi///2te+NnM9pqRpSYJ6jEav\n4nnGJTBuvfXWzHfeeWdmntef/OQni/fiUkK8fu65556WP49jasaOHZu5Hm/atGG9nlefrw888EDm\no48+OvNDDz2Umef729/+9swcvxZRLi30xBNPZL7vvvsycwwbl1+JKMewPfLIIy3fi9dlL42r5t/F\npboiIr7yla9k5jEiLoeyzz77ZK6vvalTp2ZuWvqp3WcHx7pxWZxzzjmn5e/IpWw4Lq/+Wie1Ja8R\ntgszj0NE8zIz1LRcUP1afo27p/A64vjgt73tbY2/P9uYuxZxfHa7sfJNO7Q0nSOd04qSJEna5Owc\nSpIkKfV0WZllLu5+sWDBgpbfw8f5xx57bPFeXPKGy3Fwxwc+aucj4IjyMW7TlPR2U+J7Cf9OLonA\nJS8iIrbffvvMTceDj/GfeuqpzEuXLs1clw3YFnvssUfmwdzVppOxVMGS3y9+8YvMY8aMyTxp0qTM\n3J0momzLP//5z5mXL1+e+cgjj8y84447ZuZyCnX7so1GynCLvmhqu4iI4447LjPLmVwOY/bs2Zln\nzpyZuV7KhsefP5P3y8ceeywzlz+JiLjhhhsys+TIYQW9unwNl/i59NJLi69xGAdL/FwiiktCzZo1\nKzPvhxHl0IuBHD+2C4eOTJw4seXvxe+pP986qZTM+wXPV7YL71VcqiuiHFLBzwSWb7kUDYdzcDei\niHLYDUvRvHZ5Hqxdu7Z4PXcUYhvz2uOx53VbL6NUnz8vpHNaVJIkSZucnUNJkiSlnior17uasBR8\nxRVXZGYpko/H+f3cfDyifATPnTgWLVqUmWXlegV5PtJlOW2rrbbK3LTZff3IvttLMPx7ePxZFo6I\nWL16deamGcZNM+6uvfbazHVZkmW20047LXMvz5DlMeAMOs4q5rFkOeSwww7LXM/8ZhllyZIlmblz\nSrtZya1+v/587Tndfk20w7+f5awvf/nLxfdxhxvO1v/2t7+dmSV+3m/6evx47+LwGt7HIiJGjRqV\nmbtIbbvttplZAut2bCPuusRZ/xHl9cPPFJb4P/rRj2ZmiXMoPwcef/zxzFxBYrfddsvcNDSq0zQN\nz+LfyGEXv/vd74rX89zlZw1XJrnrrrsy877JnbkiyuE5bMu99torM/sj9WxnDungTH9m3oM5u5xD\neOr3docUSZIk9YudQ0mSJKWuLCvzsTEfqXJT84iIefPmZeYjVT7O/+AHP5h52rRpmZctW1a8FzeW\nv/322zOvXLkyMx9Bz507t3g9H8Nvt912mcePH595xowZmQ888MCWr+0FfNTPdrnkkkuK7+OMV86m\nPOmkkzKz/M4S/x/+8IfMbJeIsp25EHAvlyY5S5KLInPha5abWZKoN4Mnzpo7/PDDM7Ns0pdZlfWC\nziy/cbgIr32WRpoWAe4FbBfOKOe9J6KcZcoFkzksoC+L3/YVhyTw/hhRth9n39azonsRry8udhzR\nvMAyS5ks9/N8r68Rfq3pM6JdG/O64v2SizWzLNqXUmQnaFoEnMeIs7DrWb0LFy7MzM96vtfuu++e\nmdfdlClTivfiDOemNmK71ucH25jXLoewMfO+zaEl7X5+k97qdUiSJGmj2DmUJElS6o7nxBWWWbgP\n5GWXXVZ8Hx+P85EqZ6tyht8nPvGJzJyNFPG/e0k+p2mx3nYlAJZLV6xY0fJ7uKcmZzn1ArYF2/KW\nW24pvo8lGZZduDAoS5YskbI0wxJCRMTJJ5+ceaQsfM0Z9vfee29mliZZAmGphOdfXVqaMGFCZrZr\nX2bCtpuFzOuHC81z8Vr+vDe96U2Z62EE3Yhl2TVr1mTmLMv6vsC2YPvRQBYUZ1twGA8X1Ob5FRFx\nxBFHZN57770z99IM5SY8XnUpr2k/aR5Xzkjn/a0uOfI+yOPKa7TpXhsRcdVVV2W+5pprMr/1rW/N\nvPPOO7f8GZ08dIO/G48FVwwZPXp05nrYDD/7OVSJ1yGHDrBNP/KRjzT+Lk3a9RuI5wLvd02zs9tt\nLNAXPjmUJElSsnMoSZKkZOdQkiRJqWvGHLKuzo2uv/vd72ZuGhdY4/dxnBrHzdS1f46h4niFeqzN\nc7hqekTzEgZcgoMr0/fa8jXEv43j3OqxYmwDLsXys5/9LHPTqv08rlzZPqJcZmWk4JhNjrPlLhdc\nZmSbbbbJzHGZ9TgWXhcbo35ftiV3zuG4VI7N4tgitm83XUe8x3Ec06pVqzKz7eqxUly6gkuT8F7E\n5Ut22WWXzDw/Isr7FcdFX3zxxZnvv//+zLx3RUR86lOfysyxcZ08Vm1j8O/i8mT1vYfj4NlevL9x\nlwuOE6x3oXn00Ucz8zpmW7K9v//97xev/9WvfpWZ1/jBBx+cuduXiGoaf8jPDS4tFxFxyimnZN53\n330zc4zm4sWLM3O5m9tuu614Ly4j1bQUUD0WlHh/bVoSrK/Lg7mUjSRJkgbMzqEkSZJS15SV+YiU\npeR6dXPi41Y+Hmf5kpkbxLMUElGWALi5Nh9P83dh+SWi3PSb7/Xe9743Mzdf5+/VjY/z2+HfwxL9\n6aefXnzfGWeckZmltdWrV2duKiWzzDJ27NjifZuWH+q140wsaXB1f5bop06dmrm/JYyBaPdebCOW\nuHmNsbTD4SGXX355Zg5b6CYsLV555ZWZ25WVeTzmz5+f+be//W1mlhmpHvbC47bTTjtlZimT9+Tj\njjuueD3PsW4q7Q8Glti//vWvF19jG7GcyPIzl1pjybNeioRt1rTMDIdgXXjhhcXree/7wAc+kJnD\nDXpp6aGm+xjvdRHldXXUUUdl5v1xwYIFmbk70B133FG8F68Dfu6z9N80NCqib8N2+Le0Wx6sv/fu\nkXXVSpIkqS07h5IkSUpdU1YmlnVZSqxn/XBnDD425wrw++23X2Y+Xq7LL3wk27RRNks+d999d/H6\nJUuWZJ42bVrmj33sY5n52LmXHufXmjaMf8tb3lJ837ve9a7MnCnGXTJ4/Jm32GKLzHX5jTNB+bu0\neyT/nG4tPfP35pAJHieW7lm+HY5dZOrjyn+zBDNp0qTMV199debf/OY3mVna4WzBVj+nk/C8/PGP\nf5yZQ1LWrVuXud75h8eJ78VZrnw9S9f1TErOXH/44Yczc3UGDsM5+uiji9c3DUsYCcM4eCx33HHH\n4mu8p7F8zGPBUmK7nYZ4jHlcOfOZO3ZwZ66IcrjGSSedlLkXdhjaGDzO/BxmX+OYY47JzGO/bNmy\n4r147XIIVVO7trsP9lf9edbfa88nh5IkSUp2DiVJkpS6pqzM8uP06dMzc1Hcuqx86qmnZmYpuWlj\ncpbS6lnQLNNw8V3+fC6GyVJMRFkyPeusszKz5NmrZZYaH29v2LAhM49lRNkGu+22W2aWtrioLMv9\nfG29UDmHDLBkynOh6RH8QDaf74R25Tn/5je/OfMee+yRmYsdc1bquHHjMm+K4Q78mZyFvmbNmsxs\n76YF5ztBu6ELa9euzfzggw9m5jCKgw46KPP73//+4vWcVcxjxuPBBbHvueeezPWMfl4jRx55ZGbe\nY3nu1OXTpuuk6Vro1XJzPduUJX6WhXnvGcjqAPx8uuiiizKzzFkviswZ0ttuu23L30XPY1vwc2Py\n5MmZr7nmmuI1/Devl6b2Hkwb246eBZIkSUp2DiVJkpS6pqxMnGX16U9/OnO9nzEf47ZbaLLV/9f7\nNN90000tM/enZLm53q+Rs5L5CL+XSijtsGzEUjAX+L355puL1/DR/cSJEzNzkVnu78pSKM8Flksj\nIq677rrMnLnL2eLr16/PzHIdy2r1bFHOZms3y3BT4LnNmagzZszIfM4552T+zGc+k3nOnDmZ65nf\nAymz9xfPneuvvz7zI4880vL7d9555yH5PQZDfYxYDuQQC/4/zz/OkuSqDRFlu/LnNJWwdt1118x1\nuZt7M3Nva76ee/DWCwn391zohGtkKLTb33Zjrx222X333Zf5vPPOy8zzqJ6FfNlll7X8XfTC2F5c\n8YF9g4jy2hmOUnK7IQk8F52tLEmSpH6xcyhJkqTUNWVlPgblzEuWaOvZyiztcTYjH6/ysTtnC3LP\n2YiIK664IjMXu+Z7sbRy9tlnF6/nLMOR+Aifx3nu3LmZf/CDH2SuS/EsJfNrbH+WnlkiZlmMJZeI\ncoHyphnOLJ++853vzMw9Z6dMmVK8L2ds1osKb2pNi+xyFj8X6L3hhhsyf+tb38pcz5Dl39w083sg\nJRSeL9wbmOVuzkLnzEv+Tt1UrhwzZkxm3teYeVzqhfpZ3urvbOG6/Hnaaadl5nXBn8Fj3k3Heag1\nrcYQUbZL00LhTe9V4/l/7LHHZmYpk+311a9+tXg9z7eRpi8bHvT19RzewmMfUd4H66EXw62/s5d9\ncihJkqRk51CSJEnJzqEkSZJSZw2M6iOOz+Dm5StXriy+jzsLcCzAvffem/nGG2/MvHTp0swcVxhR\njvXh2Kp3vOMdmbl5+aRJk4rXd9oYtOHGcUtLlizJzDbjUkAR5ZgqjvVbt25dZo794Bgc/n89Nqte\n8ug5XPqI46m4qwGX7xg1alTxeo7b4q4UnTbGlNcP/wYutzR+/PjMV111VeZzzz23eK9nnnkmM68F\njoHiWFCOe+HxWr58efG+559/fuYf/ehHmdl2PCfmzZuXmddnp+O5wbY44YQTMt96662ZeY3wnhRR\nnnM8zk07//D1vCYjynGevHdxp6L9998/s2MOn8d24L0qomzjpqWvqOn+FhExbdq0zBxXzXPqS1/6\nUuZZs2YVrx/JbdZuzGHTki/MvHa4G1r9mcDlooa7D1D/jf1tb58cSpIkKdk5lCRJUurKWicfj3L5\nmjVr1hTf98ADD2RmyZi7arDE/OSTT7b8GRHl42I+zmf5izu3dFopcVNjWZnHn2WyuqxcDxN4TtPq\n9Dz+LAuzFBxRlky5+wTbmKWF0aNHZ2YpqN3SAN1YsuEOCoceemhmHuMvfvGLxWtuu+22zIsXL878\njW98IzN38mBZmNdeXX5juZrX0pZbbpn5C1/4QuYJEyZk7sZjH1GeTyzFH3744ZlXr16dmTui1K9n\n2YtlTl6HLOXzPlbba6+9Ms+fPz9zXULT/+O9g7tB1Tgsomm4BYdGHXXUUcXreR9lyfLMM8/MfMop\np7T8GSMd7xG8Jup/N32Oc6gSh71wmFtE2VcYjj7Bxi7RQ54tkiRJSnYOJUmSlLq+rNw0+ysi4qGH\nHsrM2cosV/IxLMtqLCVGRHz4wx/OzEf1/Pk+tm/WtMMNS159fSTOdmLJcubMmZmnT5+emeXmiHJm\nYF9KkN1aptwYXM2fJds5c+YU3/ed73wn86JFizJztv+KFSsy97W9OUN86tSpmTmjmsMFeq2NePx5\nvYwbNy5zffxYDmNmKZ8zXrkLTn2NcCeaGTNmZB47dmzmXjvmg4XtxWEQEeWsYrYRZ7wuWLAg809+\n8pPM9bAbniNnnHFGZq4o4PCm1pp2jIpo3kHt8ccfz3zddddl5kz/Qw45pHivffbZp+XPHCrt7qnO\nVpYkSdKA2TmUJElS2mwwZ7e8gCH5QXwEXC9uvH79+szLli3LzHIzH/uzZLPnnnsW78WZgU2b2neA\nwfhlhqSdOPv04x//eObLL78889NPP128hsd5hx12yPy5z30u83ve857MTSV+22jw1PcLXn9cOYAz\nkblANWf58do74IADivflvzmjfJgWku2KNqrbommB66aZy2yLurTGf/Na4vHvgJLlxrbTkLQRj/dj\njz1WfO2SSy7JzHIkFx3fsGFDy8zhNBERs2fPznzqqadmrodXbWId2UbtNM02X7hwYWbe01atWpWZ\n7RARceKJJ2bmMIAO07KNfHIoSZKkZOdQkiRJqevLyvz967+FJRSWLFkm4T6sLJnUpcgOK0026cpy\nWJMuOeb91RVtNMJ1fRv1977e131YO+ya7MiSZbt93bnYOIfUsPzMcv3222+f+YILLijea/LkyZmH\ne9/efujINmr7A9F+7ENwj/Mf/vCHmbfeeuvMxx9/fPFenN0/3MMw+rG3smVlSZIktWfnUJIkSanr\ny8oqdH05bASwjTpf17dRX+7rHVYiHoiOL1nW+/ayzMwZypzxOmbMmMwHHXRQZg6Biuia9uv4Nmr7\nwxtWAOD/t1u9pJvbyCeHkiRJSnYOJUmSlOwcSpIkKTnmsLd0/VipEcA26ny2UXfo6vFsI0TPtFFT\nX6lLxhW245hDSZIktWfnUJIkSaljl1WXJEnaVFhK7oHycb/45FCSJEnJzqEkSZLScM5WliRJUofz\nyaEkSZKSnUNJkiQlO4eSJElKdg4lSZKU7BxKkiQp2TmUJElSsnMoSZKkZOdQkiRJyc6hJEmSkp1D\nSZIkJTuHkiRJSnYOJUmSlOwcSpIkKdk5lCRJUrJzKEmSpGTnUJIkScnOoSRJkpKdQ0mSJCU7h5Ik\nSUp2DiVJkpTsHEqSJCnZOZQkSVKycyhJkqT0f0vbteiaptPEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85965d6d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGfpJREFUeJzt3XuwVmX1wPFl9yjNTA1M09RCTMEbGhiSWZF56U6lk2lN\nNycrp7GZtGayccqii+nE1KiTl8wotaxkaDIQFUURNExTREEtNUHDyqys/P3F8vs8v7NPL+dw2fs9\n389fS87e73nPft693+1e61nPZk899VRIkiRJERHP2NRvQJIkSe3hzaEkSZKSN4eSJElK3hxKkiQp\neXMoSZKk5M2hJEmSkjeHkiRJSt4cSpIkKXlzKEmSpOTNoSRJkpI3h5IkSUrP2oi/y0WcN7zN1sNr\nOE4blmPUfo5RNwx3nByjDc8xar8Bx8gnh5IkSUreHEqSJCl5cyhJkqTkzaEkSZKSN4eSJElK3hxK\nkiQpeXMoSZKk5M2hJEmS0sZsgt03nnrq6b6cm222Pvrlaqg4Fk0co02LY9R07jhGktQePjmUJElS\n8uZQkiRJybQyMOX1z3/+s/jZ7bffnvEVV1yR8XOf+9yMp06dmvFee+1V7M/tNLA6RbxmzZqML7/8\n8ox/+MMfZrxo0aKM//73v2f87Gc/O+O3v/3txet+73vfy3jUqFHDeMfimP3nP//JeNWqVRnPnj07\n42uuuSbjXXbZJeOPfvSjxetuvfXWGT/jGf4/rKR26qW0qdaFMhqvupIkSUreHEqSJCmN+LQyHwk/\n/vjjGV911VXFdjNmzMj4D3/4w4CvtXjx4ozPOOOM4mejR4/OuAuPlDeW//73vxnfd999xc9OOumk\njOfNm5fxX/7yl4yZyuRrPfnkkxlfeumlxesyZfn1r38942c+85nr9N5HojqF8sQTT2Q8Z86cjFkG\nsGDBgox5jFevXp3xY489VrzuiSeemPGYMWMyNsU8PDxf6tIZYlnGs5719NeE164Ng+PCaxf/vT72\nHJdexsixW394Hfz3v/+dMb+DIspjzp9xvHhNbNMYeaWVJElS8uZQkiRJaUSmlflImGmx0047LeMf\n//jHxT5//vOfM+aj/he+8IUZM93M7SMiXvrSl2bcpkfHmwKPP1PJb3rTm4rt7r///oz5SJ6P4V/w\nghcM+O+cuVynQq+88soBt9t88817+wNGGB6/FStWFD879dRTM54/f37GLNHgObLjjjtm/Mgjj2R8\nxx13FK977rnnZvzpT386Y47RSD+PesU05d13353x1VdfnfFf//rXYp8pU6Zk/PKXvzzjLbbYImOm\n+JkmiyjHhts1jdlIXFiA6ch77703Y5Y08d+f85znFPtPmjQp4+222y7j5z//+RnzuPLfI8qxZDeN\ndR2vWr+OX1NnBl7rHn744WIf3gfwHNlmm20yZplT3dWkl7EYCn6fNpXq+ORQkiRJyZtDSZIkpRGZ\nVv7Xv/6V8Ze//OWMzz777IyZbowoU5ZMkz3vec8bcBvO9osYWqPMfsK/n8f2E5/4RMZModS22mqr\njKdNm5bxYYcdljEf6V922WUZs1F2RMQDDzyQMdPau+++e8b9mhrpFceL6f13vvOdxXZMMzPV8uIX\nvzjjcePGZcxx5HnIMYkoZ6dPnjw5Y6Y7bSzfjKnkhQsXZvztb38745UrV2Zcz7Lk+cPrGq93TEft\nuuuuxf5scH7AAQdkvNtuu2XcVMZRp7n6qYsAU8l33XVXxjNnzsz45ptvzpidGdjxIqKc4c9jxPOK\nv6/+TmIq+pWvfGXGr371qzNm6pnjVY9JXVbQL5rWhf/HP/6RMcehPg7cjmPBmNvssMMOxf4sJVjX\ndH99z8H9e+n64JNDSZIkJW8OJUmSlPrzWfAA+BiXjY+5zu7f/va3jOvHw5xtzNl7XP+X+9QNZply\n66c0Sa+Ytrr22mszvv766zOuH3W/7GUvy/jCCy/MeOzYsQP+jgcffDDjZcuWZbxkyZJiO85Q/+Uv\nf5kx058jMa3MNATTWUcddVTGS5cuLfZh2oPHb8KECRnzfGFqjL+Pay5HRDz00EMZM8U8fvz4jF1/\nucRU8plnnpkxU5acIc4UMWf9R5SzLHnusiSEv++mm24q9ufngulLppg//OEPZ7zzzjtnXM+q7fL1\nkt87ERGzZs3K+Fvf+lbGLKtoSt8yxVv/jOcYSzqoPkfYIeDnP/95xt///vczZgnV0UcfnfFBBx1U\nvFY/pZWbGlxzRj87k/DzWs8oHzVqVMYsF3j00UczZrr/rW99a7E/r5csoxnKOdHLDGXyiipJkqTk\nzaEkSZKSN4eSJElK/VMoMADm2FnbxpoKdjdnvQBbMUSUtQCsEbjzzjsz3n777TNmrUH9XkbiagCs\nT2L7k5e85CUZH3zwwcU+M2bMyJj1h8SaENbHsG6nHgsec9aLrGtNRj/gZ5F1sl/72tcyZp1hXffE\nVW0OPfTQjFn3xGP5ohe9KGOuErDTTjsVr8vfyVob1rZNnTo1Y47xSDmn6no2rurEumrWSnH8uMJG\n3UKD48TaTraLYp1c3QqHLaJ4XvLayWsCr711C46uXS95LOoVhc4444yMeSy33HLLjLmK0Cte8YqM\n99xzz+K1DjzwwIw5fjxGrAWsW9nwc7F48eKMee43nbusSY4Yfj3cplR/3jg/gJ9x1hnyGLMVDdsI\nRUT86U9/yviPf/zjgK/F48Wa4Bq/K/m91WuLGv6d1hxKkiRpnXhzKEmSpNRXaeU6tcEVAI4//viM\n2SaDj1e33XbbjLkSQ0TEvvvuO+Dv5IoPfOze6+PdLqRJhop/Z1M7DB7XD37wg8X+9YoAa/GxP1Oh\nfLzP1hhs81DjmI0UTZ3+L7744oy5QgbTgvUKKfxvph9ZrsH0MVtC8Ryp05pMed56660ZswUE03Is\n6ajTZ/2En/3f/e53xc9OPvnkjNlii8f8Qx/6UMbHHHNMxnX7k6a0Fc9jvpfVq1cX+8+dOzdjpua4\n2g1X4mBauevXRH72zz///OJnbBE0ceLEjA855JCMuVoJ2wAx9RxRfl81pXJ5LOvjyvOE1+F99tkn\n4/322y9jprXrc6ytZThNK5Px3+tU8D333JPxb37zm4xZqsS/n5/vVatWFa/F7x5eR9k6iiUx9ftl\nup7fm/ze47nDz0H9WnVJ0P/SzhGVJEnSJuHNoSRJklLn08p8dFo/0v3Upz6VcT1rbC2mhY844oiM\n6w7wfITPlQX4eJeP7QdbIYWzDJtmGg2WWuli2oXjxMfoTFOxg3xEmabkzDqmvDiDi5jK5KzWiDIl\nwPKBtqZG1rem2XhMoRBnJE+fPr34GVP/TGUyHca0Zq8pX35emObiIvdc0YipPJYKdPFcqfFYcLw+\n+9nPFtsxtcs08Xe/+92MX/e612U8WCq36bgxbcWZsDxXI8p05MKFCzMeM2ZMxkyZDZb+7AJexzij\n/uqrr27cjqUUnOnPaxqPUX194vEfyjHjPrwmMOXJ77HBZrt2Ycx47FlOU5dnsBSAfydTxExFM11b\nl8cwXc9x5fHiGDP1HFEec75PruzFTg933313xq961auK1+JKV72kmEfGt6EkSZJ64s2hJEmSUifT\nynzUyjTTl770pWK7efPmZczHyEy5cMYYUyNsbh0RsXz58ow5a4ipNKZcbrnllmJ/PpJmzGbbnHE5\nlJRP2zQ1Y2VamNucd955xf7nnHNOxmwMe+yxx2a8//77D/i7eSw58zWiPP7crivHdSiYUmEK9pJL\nLsl4yZIlGfO4fOQjH8l4t912K16X6X6m8plCYSp5KMeYZRwsyWAZCWcuM2XSD2PKv5mNru+6665i\nO6bT2Z2hl1TyUI7TYCU9s2fPzpjXW44l9cM4rcXZrmx2HFE2j+Z3zG233ZbxXnvtlTHPozqV2zQT\nt9djyVTysmXLMr733nszZlqU53EXS3CYImeKds6cOcV2LK/heE2YMCFjpmx53h1wwAHFa+26664Z\nN5VRcBx5PY0oPyOnn356xosWLcqYjdJZJlUvLFA3zf9fujfCkiRJ2mC8OZQkSVLqfFr5uuuuy3jW\nrFnFdkz/EtNq3P+GG27I+Iknnij2YVqUj4f5qJb71I/dmUrlzGeuJ8zZ1U1rCXcVjx9Tk0w/MTUY\nUR5PpgSY4mcj3aY1k+sGv5w9O9wZf13BdAVTEjfeeGPGTGExdc/x4kzGiPL48bzkvw8Xx4XnVVMj\n2H7DawxnK9czv/m55gx9Xu9YXsMUb69NjTnGLKm56KKLiu2YTt19990z7reZ5Gvxb2Eqdu+99y62\nu/baazNmc3euZ8z9x40bl3FdQsNm2U2NkPm+6rQiU9k/+9nPMmanAZYkNKVFu6JpPWTOLo8oP7u8\nh+A6yRxHHi82OY+IOO644zJuunZxzeVLL7202H/+/PkZ89znNfnII48c8PexbCCibOLdC58cSpIk\nKXlzKEmSpOTNoSRJklLnaw5ZK8Fp5/V2rDd48sknM3700UczZk1GXVPBOg6u8MHahcHqMJo6zfP3\ns6aA9UP9UE/Fegu2AWDLlHq1Ex4nLvrOtixXXHFFxhyXPfbYI+O6LRFrRDjmTat3dLG+psZj+eCD\nD2bMOqLx48dnzFpY1qrUtWhNC71zO45Xr+1T+FpN9TlcIYVtdJpafHQVjxNrPutaWl4nLr744oxZ\ny8trJOua2Q4jomzBwbpeniNsg8Ta7YhyVZ3JkycP+P774bxai38LayxPOeWUYruzzjorY7aPYX01\nz0/Wi9Z1vPzu4cocXPWL5x5bVUWUdXNse/LmN785Y15TuzheTe3Umlb6qX/WVL/JaxJb//D7KKI8\nR3ldeuSRRzLmqia8B4horiOeNm1axvxu5Aor9Xitax24Tw4lSZKUvDmUJElS6mRamfg4nY+AI8rH\nuExnNKU2+Jh+sG7ibMfA9Mvo0aMzXrp0abEPF+rmo2Ku/MGVO+q/pZ+wzcwJJ5yQ8Tve8Y5iO6bc\n2V7gjjvuyJgr0TBFetVVV2VctzTia73nPe/JmGPZtIoL06VdSrMw5ciUCFNYbJvB1FSv6Qieb0xn\nNZV3MK7bLvD8++1vf5sxVwihww8/PGOWKvTbecTU/4oVK4qf8frFFSDYMoVjz2NTH3+OedOKHWyD\nxM9LRNmOhedSV8+fdcFzjccuImLmzJkZP/TQQxnff//9GbPsien+enUZtk3hOcbSgeuvvz7juk0K\nU54f//jHB/yd/XT+8DO93377ZXzqqacW27373e/OmKst8Tt88eLFGTP1z7GLKFPG/D7jaitsS8OS\nq/q98NzjPUhTOzZeg4fCJ4eSJElK3hxKkiQpdTKtzEen7BLPtG5Emb792Mc+lnH9qH+tuXPnZszH\n8RFlKnqXXXbJeNKkSRkznfCLX/yi2J9pTi7Oze7mW2+9dcZNs6S6io+4meZlmomzImucccl0VtNs\nc64IwkfwEWX6lDPNODNvzJgxA77HpnRz/d9tGzO+H6YAmQbhTLmmlET9dzXNDG5KxRNTmfXrMIXD\nFV3mzJmTMceI5Rmvfe1rB/x9XdJ0XFlGw1mlEeVMRZZOjB07NmPOkF25cmXGLMmIiDjwwAMz5mx1\nXtd47tUdFTg263O1nK6pP/tMDfMzy1IIXscG+x5gmpH7sKTgq1/9asZc4SMi4uijj86Y32n9lEom\nHj/Ouuf5EVHONm+6dr3lLW/JmN8hV155ZbEdZ6HzdadMmZIx0/v1ClR8nzzHmr5feN0Y7jj65FCS\nJEnJm0NJkiSlzj/vZ4qQi4RHlOklzoTlo1o+jmdjyjpNwhQ1m8dy9h/TpatXry7250zk97///Rnz\nUXO/pV/4iJuP1xcuXJgxj9nOO+9c7M9UC1PD06dPz5jpT85qZVrr9a9/ffG6PP5MbbLB8po1azJm\nupmp/7oJMXEs2zCufA+cHceZdkzf8lhwXJrSLLVethtsG6Y5OUacXc2UOJtg9/oe24znDkskeFwO\nOeSQYh/OrOQ1jq/FY8km4iyXqPe/4IILMj7nnHMy5rWTs5Mj+nvG+HA0pQObGjQPhp9zpoy/+MUv\nZswFAFh2EBFxxBFHZMzxbltJzIbAv3GwRSaajgX34ffAqlWriu3uueeejCdOnJjxtttum/GWW26Z\ncV1OwjHu5bq2Pseu+1dRSZIkrTfeHEqSJClt+nzXMPExLB/VRpSP55m+ZPqFqSmmi+vGyWxmuWDB\ngozZvJRNTZkKiiibLbPRJR9P90M6rAlT9meffXbG/PvrWaaHHXZYxhzbgw46KGM2rmZDbD6qrxuL\nMkXNmWY33HBDxmxyypQLZ5NxBnxEOXO+bak0vh+mYPn33HbbbRkfe+yxGbN5br3+ddNsyl5m0/Hz\nXqdTeP78+te/zphp8KZx6YfzqKncgdelekY5j0dTY1yqmyoTZ4ufeeaZGbPRNtOUn//854v96zT1\nSDWUdb57OXciyu+uGTNmZMw1k/laXO86ImKfffb5n7+zX/FYDpbKbRo/nns8726//fZiO3aA4PcD\ny0N66eywKbTnnUiSJGmT8+ZQkiRJqZNpZT7q5RrGTBFGlGmT3//+9xlz9iXTJI899ljGN998c/Fa\nnGHLR8ecscdHwm9729uK/dlwlE1l2/QYeUNimpbHksevblb9hje8IWM+em961M81QXlcWRJQv5fr\nrrsuY36WOAud5QaTJ0/OuJ6F3ObUDI8HZ9cdd9xxGZ988skZc5bjF77whYyZvoooU4tMMfeSTuM2\ndRnHZz7zmYzZQJ6fl7333jtjlhr0wznVdGwuu+yyjOsyjGnTpmXcywz5wdZh/dznPpdxnSpbi838\n99hjj8bXHmk4dvy81tZ1oYP6nGJ3hp/85CcZc+zf9773ZfyNb3yj2L8NXRQ2Jh4/Xt9ZQhFRpnw5\nRk3Hi7P+60bj/D177rlnxrw3aeviCd2/ikqSJGm98eZQkiRJyZtDSZIkpU4WHbB2YKuttsqYqydE\nlIvPczFytjlhKxouJM99I/5/TdRaXLD8Xe96V8YnnHBCsR1rs/qhJmpdcVUOtrngSjJz584t9uGq\nKuPHj8+YY8FaUtacsvaDdaURZQsI1lptt912GbM+5DWveU3GrB3laik1fkbbVEcSUdbOsGUN62B+\n8IMfZLx8+fKMzz333OK1WPfHVYBGjx6dMdsVsQaLtWwf+MAHitddtmzZgPvwfGMtJOtV23a8h4Jj\nxLY2HAuu2hQRcfDBB2fc1CqDn0t+9mfNmlW81ne+850Bt+Pv/NGPfjTg+x2uuraua+PJ989rTf2z\nXloP8djzuyoi4r3vfW/GbBV2+OGHZ8w6Q9sLPY3nFOcaRJTfF/xO4FjwmsR7C7aziyjr4HmtXNd6\n001h5N2lSJIkqZE3h5IkSUqdTCs3dRSvU79cWYAprKYp7UwB1I96mXIbO3Zsxlw9gB3n68W82/ro\neEPi38x04JFHHpkx05ccr4gy1cUU1mDd7Qf63UzfRJQpyEMPPTRjlgVsv/32GTMlznRdXR7Q1Nao\nzZjeYNuLSZMmZcwU44UXXljsz9Y2XKWEq9KwpGLlypUZc0UWlhBElOP3xje+MePTTjstY55vbVuR\nZrj497OkgisFrVmzptiHJRpNLTh4veP5xdZBEWV7D66Kc80112TMkp71qevXyl7TyrxGNP3NPF94\nrap/tuOOO2Z83nnnZWwq+Wk8xjwnVq1aVWzHFD2vS/zeYNnS6aefPuD2ERETJkzImCUZXfiMd+Mb\nTJIkSRuFN4eSJElKmw1lYfAhWm+/iO+Zj4A/+clPFtv96le/ypgzkJgyaXrUXM9EnT59esannHJK\nxltssUXGLUhtrY9n1Rv8A/H4449nfPzxx2f805/+tNiOnefXNX3MGelTpkwp9jnxxBMzZspu1KhR\nGfeS8hnirMpOjBH/No7XzJkzi+3OP//8jB9++OGMmV7hLD/OEmyaKR4RcdJJJ2V8zDHHZMzU9QY8\n31o1RjxOnA15yy23FNstWrQoY17LmH7kzEqusFGnP7mK1OWXX54xywU2VOnEOpxXwx2nDXIe8f3X\nacb77rsv46bU5vz58zM+66yzMq67cfAax324Wk0L0pedG6MlS5ZkzPNi3rx5GbO84qabbsp44sSJ\nxWt985vfzJgp5o0xLsM9j3xyKEmSpOTNoSRJklIn08rENFXdJJSP2pli5qNipk84K3L//fcvXovp\nrF5SjptIq9JhveDsST7Oj4i44IILMmb5wDbbbJPxTjvtlPHUqVMz5uxozpaNKNPPm2BWcefGiJji\njChLNJgau/POOzNmpwDOfGWapU4rb7755hnXs803gtaOEY9/nQ7j7O/Zs2dnzPQzU5Ms22DZTERZ\notOy0hlqfcqS50dExI033pjxJZdckvGCBQsyfuCBBzLmGPG7KiLioosuynjcuHEZt6xTQivHiOpr\nGr+TFi9enDG/j1jCQfXiF0cddVTGTY3pW8C0siRJkgbnzaEkSZJS59PKPf/yHma79oHWpsOUHKP2\n68QY1dc0dmTgOvFMU/J6x/Wv6+4MLU6BUetTlvUscJY3sVn10qVLM+bx3nfffTP+yle+UrzWDjvs\nkHHLUsnU+jGqNZVu3HrrrRmvWLEiY84OZ5lTRHM5WsuYVpYkSdLgvDmUJElSGjFp5RGiE+mwEc4x\nar9OjlHTmuNN1/gWd13oVetTloPN7l++fHnGXCeb62dzzWSm+juk9WM06C/HucOx5PnSFHeIaWVJ\nkiQNzptDSZIkJW8OJUmSlKw57C+drJUaYRyj9nOMuqHT9WwjhGPUftYcSpIkaXDeHEqSJCl5cyhJ\nkqTkzaEkSZKSN4eSJElKG3O2siRJklrOJ4eSJElK3hxKkiQpeXMoSZKk5M2hJEmSkjeHkiRJSt4c\nSpIkKXlzKEmSpOTNoSRJkpI3h5IkSUreHEqSJCl5cyhJkqTkzaEkSZKSN4eSJElK3hxKkiQpeXMo\nSZKk5M2hJEmSkjeHkiRJSt4cSpIkKXlzKEmSpOTNoSRJkpI3h5IkSUreHEqSJCl5cyhJkqT0fx+p\nFPmOx/O+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8596495390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGg5JREFUeJztnWmwHVXVhhcqzogSIYBGAsigSRiCyCgqUxDUIAhaSEEE\nnMrSAv6gEhBLtEr5g1AECIOoRC2EAIaoSFCCARIJFIOMIYQYZEpQAg448v34Kstnb08fbrjT6Xuf\n59ebe073OadX7+6dftdae50XXnghREREREQiIl423F9ARERERHoHJ4ciIiIikjg5FBEREZHEyaGI\niIiIJE4ORURERCRxcigiIiIiiZNDEREREUmcHIqIiIhI4uRQRERERBInhyIiIiKSODkUERERkeQV\nQ/hZLuI8+KwzAPswToOLMep9jFE76G+cjNHgY4x6n44x8smhiIiIiCRODkVEREQkcXIoIiIiIomT\nQxERERFJnByKiIiISOLkUEREREQSJ4ciIiIikjg5FBEREZFkKJtgiww4//nPfzrqddb5b1/Pl7/8\n5UP6nURERNqMTw5FREREJHFyKCIiIiLJqLeV//GPf6SeN29e6lNPPbV43yOPPNJx+8mTJ6c+/fTT\nU++0007F+7Q2X5wXXiiX0Xz++edTL1++PPUll1yS+qc//Wnqxx57LPWrX/3q1CeeeGKx3+OPPz71\nK1/5ypf+haWA8fv3v/+d+l//+ldqjrdXvepVqdddd91iXy97mf9vFZF2w2tifX9bA1OgqIcbr8Ai\nIiIikjg5FBEREZFknaZHnYPAkH3Qi/G3v/0t9be+9a3U559/fuo//vGPxTa0xgjt4u222y713Llz\ni/dttNFGqQfx0fFA7HhI48QK45UrVxavnXXWWakvv/zy1I8//njqv/71rx33S5ty0003LV6bNWtW\n6l122SX1ED3Sb12MusHrB8fV73//+9QPP/xw6vXWWy/1+PHjO/49IuL1r399ao4xY7T2cIzR1mfa\nRn1cmZbBsdRjFlh/v8Cwxohj55///GdHzXjVqRdMieEYYUqGMRp8GEeOtYiIv/zlL6mfe+651K95\nzWtSv/a1r01dpzkNUXpNxxj55FBEREREEieHIiIiIpKMmmplPqpnteuFF16YetWqVY3bv+IV/z1U\n9aPjNSxdujT1nDlziteOOeaY1D3wqH9Y4fG76667Uk+bNq1434oVK1Kz+pWP5MeNG5eaj+dpPdeV\n4rfeemvqnXfeufF98r/U5/5TTz2V+pvf/Gbqa665JvXf//731O985ztTM3YTJkwo9rvHHnt03OZ1\nr3tdauPVDOP09NNPp/7Rj36UmmOvZp999km9zTbbpH7DG96QmpbZmDFjiu3XX3/91ByXfbE861Sn\nkXq9ZKoSUzLYdYHXMVb3R0S8+c1vTs20pb6kBEQ0W5Yj9XgPFry+LVy4sHjtiiuuSM1Utd133z31\n+973vtRbbrllsT1jPtRx8cmhiIiIiCRODkVEREQkGdG2Mu2Je++9N/WZZ56Zmo96WSn0xje+sdgX\nbRJaALRsWP133XXXFdsfeeSRqfnYf7TAWLCh+HHHHZf6gQceKLbhcdpiiy1SH3TQQan33nvv1I8+\n+mhq2pp33313sV9W0tJ+06bsDGPHYxwR8elPfzr1ggULUvO40hp56KGHUj/55JMdt42ImDFjRupd\nd901Nc8X/p2pBqMVxumZZ55JfcYZZ6SePXt26tWrV6euqyQXL16cmik11LSV60rasWPHpmbqxtSp\nU1PTru5WkT6c1tpAw/SYP/3pT6lpJfP6xJgypSKirH5lLHjd5LGrt2+y+5tSqLpZ1MPQUWBY4b3+\nJz/5SWqmrEWUqVG0/tlBg/FmB4eI/t2f6hSgta189smhiIiIiCRODkVEREQkGdG2MquIzj777NSs\nAOOjdtocrJaMKG0XWms/+9nPUtNu/u1vf1tsT5tn44037tsPGEHQ9jjvvPNS33///anrajw2qP7K\nV76Smo/k//znP3fUG2ywQera5qCdxir22hobzTRZyfvuu2/xPr5Gy2zDDTdMPXHixNQ8xowDG2VH\nlOkel112Wer58+enZjNzjlfaYiOZuqqX9tQnP/nJ1Lfffntqnu+0qdh0PKKMExuUN6XEsFlzRNm5\ngWkkXAt9ypQpqWk3czGBiP+9LrSJevEExuLaa69NTctv2223Tc30ptoW5P2G1z7+/dlnn03NSvOI\n0lbmYgI8L5g2xfSAt73tbcW+mprWtx2OMR6jiy66KDWvTxyDEeV9iOcxj+uSJUtS854XUY7Dta30\n76+975NDEREREUmcHIqIiIhI4uRQRERERJIRlZxT5+Bw9QbmNDHnj7lK733ve1NzVYaIsnR93rx5\nqenrMyegbg3BnEOWtA/RwtrDAvNtmIvBHAsei6OOOqrYnm1LmB/DlWx4XBmjt771ranZ8iSiXP2B\n8RvIfI02wt/PnJjPfe5zqZcvX15sw/N8zz33TM3VbtjWhPFiTvCDDz5Y7JetoNiKiO0/2K5op512\nSs084pEWR8aIeWYRZZzYiobjkMdm0qRJqXfbbbdiXwcccEDqur3GGphjWueZMef6V7/6VepFixal\n5jnGazLPkYiyTVEb8tl4vNnKJKLMUef9ifch5uvy/GWObkQZ/ze96U0d98sYsR1YvT/monLlHOab\nMm+8PveYT9eGGPUV5hn+4Ac/SM2WNTzebHkXUa5+whVteI7w/K7bz/Df1EMxbxi5MxMRERERWWuc\nHIqIiIhIMqJs5bqdwpw5c1Kzmzxb1rBtAh+7076KaF7MnCX8fDS/4447FtvTNhlpVheh7UXb8IYb\nbkhN2+EDH/hA6k984hPFvmhpsLUJbUba0rQWefxpDdTb0HYZjbYyfzNtptNPPz01Vy+p0yUmT56c\n+pRTTknNtAyOl3r7NdR2ysc//vHUX/ziF1Oz9RHtGI7veiWIkQTtKK7MEFG2SeEY23rrrVMfeOCB\nqTne2B4qov/WPK1R2mls7XHYYYelHjduXOraMuvV1BuOHZ6/bJU2c+bMYps777wzNa1kjhcee96H\naF9GROywww6peR9iGseYMWM6vqf+/kzV4Weyrc5mm22Wur7XttlKrtPReL/4+te/nnru3LmpV65c\nmZqWft0CjylNjCtbgPHvdesjfremexU1x0p/x01vjjoRERERGRacHIqIiIhI0npbmY9U68owdi7/\nwx/+kJqPW2l38tF63ZWfViShHUJr5thjjy3eN5KtriZoK7MykdYSrfwnn3yy2J5WMh/p8zjvtdde\nqbkCAK2w2rIkTY/eR4vFTHuI4+XGG29MTTuKqyRERJxwwgmpt99++9RMw+CKJTyWTZXiEWWF7PTp\n01PTlqMVN5JjxGOzbNmy1BdccEHxPlaQ0talNfae97wnNW35ehz093gy5uwiwJU1mro2DPR3GQqY\nUsRKe163IkrLkpYjV0LhvYd/r6uNOS6ZutFU7VzD84rpHqxK5jWY9ilTfl7sc3oR3hP4eyMivva1\nr6W+/PLLUzPG7IbBe33d5YTVyxwTtOF5reQ9M6Ic00ydofXPWDCOPA8iuo/3TvjkUEREREQSJ4ci\nIiIikrTeVmZ1z6xZs4rXaEHx0TEfz9Ju5oLl9cLiTQubswqZVWJbbrllsX1TZVuTzdaNXn6Ez99G\nO4mVVoSVjDfddFPxGqsvaSWzsSitZForpLYs63+/2Da9fLxfChwzd9xxR+rrr78+Na2KKVOmpD7i\niCOKffE8p1XStEj8SznHGeO3vOUtqTneaKfwHGxzFeUaeC4yDYaN/SPKY86ODBMmTEjNMTKQ53U9\npmhHchzTWqPN1fYxxt/P+0OdTsQq1Ysvvjg1709cjIFpHBwHEc2x7JauQXhNZvXt0qVLU9PWpq5t\n5TaMM14XHnvssdTf/e53i/dxHsG5Am39gw46KDVTYJ544oliX/wcxoX2MdMDrrzyymL73/3ud6kZ\nF36vd7/73akPP/zw1IxXRGlf9wWfHIqIiIhI4uRQRERERJLW28qEa0JGlJVhtNJoEdP65KN9NtuN\nKB/Jssquya6mrRJRVg41VS01VXW2CX5vHnNWd9Fq4ZrJdTUej+2+++6bmhYzrbT+WsHdqpo7fV6b\nYCxofVx77bWp+du41i7Xoq7TLVjBx3FF26nJYu4GbRdWVNNWZvUkx85Ig+c1ryv1b26q+L3llltS\ns6kxbScey4hyjPalor+u+LzqqqtSc131qVOnpl7b6sleg+cyK4wPPfTQju+JKCvM77nnntTf/va3\nU5977rmpP/axj6U+7bTTin01LazQlMJUN65mXHjv/NCHPpSaTe5pf/YlNafX4LWK1vH5559fvI8W\nO48rm5uzswMrmusFF3iv4rWLx2/JkiUdPyOiTFHgeGeKAa+VTak2EWtv/bdvRIqIiIjIoOHkUERE\nREQSJ4ciIiIikrQ+UYc+OlsARET8/Oc/T10vaL0G5mQ8++yzqZk/FVH698z7od/PnMOrr7662H7/\n/fdPzdwD5vYw76epLUibYI4KfxtXldlqq61Ss5t8RMQHP/jB1My34PFgOwbmlPD9deybtiH8DMae\nuUW9lhfaLQ+IuSs8Z5krxnhxLG222Wapn3nmmWK/zJG5+eabU3P1B+a5MReR53gdI64AwPHG79jU\nsqaN+VDd4HnG1ib1qk3MJWULjRkzZqTeZJNNUjPnb+utty72xdxSrqrC1lM8d773ve8V2/O8OOCA\nA1LvvvvuqUdSniivF4zLtGnTivdNmjQp9VlnnZWaLYp47s+ePTv1jjvuWOyLbaV4jeJ3Wb16dWq2\nrarfx9zCpvzwl9KSqpdgzuWll16aul6Zi/cHXld4DWVLIr6/Pqeb8tiXL1+emtdj1kDUMM/w/e9/\nf+qDDz44NecWa9u6pqadsw4RERERGRScHIqIiIhI0vrn+ny8vffeexevsYs5O43TGqM1wke69eNg\nPvrti501f/78Ynt+JleVmDhxYmpaRnVn/TZCm56WO1dsYKl/XXrfF+uiKRb8e91egDF/8MEHU9OK\noz3A78hu9LTleo362NEO52/mCkG0w3iO8u+0vCJKq+q6665Lfc4556RmSgFtxRNOOCF1vYLOnDlz\nUrPlRtMYoeXC86i2mNtohzXZyp/5zGeK9919992pOfZ4XvPvtJLr693ChQtTM35MMWCM582bV2xP\nK3qHHXZIzbSAtqbLvBj8XWzVFRGxxx57pGYseY5fdNFFqRmXel9cAYx2P9/H+1A9xvbbb7/UY8aM\nSd32FkNN8Fjy3F+2bFnxPl5X2F6N2/AY8ZxetWpVsS/OKbjN29/+9tQc32xrU38XpvoceeSRqdkm\nj9e+/l7rRk7kRURERKTfODkUERERkaT1tjLh492Isrv8Mccck5pVS7fddltqVtw99dRTxb6auuGz\n+u9d73pXatovEeUjbVpd/Jym1T7aao3RzuJxZiUkH9vXv6vpd/J40P7l31mFXNvVXLCen89VcWiD\ncxURvp92c8TwLz7f7bxgNTDtKFa00XaiNcXznfZTRGkTL1iwIDVXGeB5QFua1X/1OX7jjTd2/Hza\nNIwrrW9+97aMlW7wNzBeTJuJiNhzzz1T85jVaRVroB1Vp7E02YmLFi1K/cMf/jA1YxlRnhdNK3mM\nRnhcGUvajCeffHJqVtLWq24xxrze3Xfffal532FMIsr497eytQ3wukCLtj73Oa64MldTGhFTNXjs\nI8quJ7z28XrFLh1ML4goq80POeSQ1FzdqGkFKm1lERERERkwnByKiIiISNJ6W5mPzesGvbQc+ei4\nyYKi3bZ48eJiX7TcaCfysTObUdbVfw8//HBqVlbSpmMVLb97N7u112wafh/aWXxcTpuWTWE32GCD\nYl/9qZSjxVs3Jm2ydrbbbrvUtMwYO9pqbNJd73eo4tLXhs8cG7TMWW3M5uS0CZtSKiJKO+yrX/1q\natrvtLw+/OEPp6Z1zEbNEaUFQ/uLNg0b93IcdWuI3WvjpS80xbi2+Dl+eF735Td3ew/H8fTp01PT\nTqvHGDtHDGQF5UiFx4X2JS35euyx0fg111yTmulVhx12WOq6afpoiwuvz7x28x4eETF+/PjUTXY7\nxyTjVY9JzhuYIsCG9YxxfQ9kqlrTIhmDhU8ORURERCRxcigiIiIiSettZVpWv/nNb4rX2Piaaw7u\nsssuqVm5yjVA68fDTz/9dGpaW3y8y0fF9RqJtLW5L65j2dRou62NSNnsmr+HDY5p5R544IHF9rRH\nmmx2Pt7n3/v62J3b08qkfcx48T11U9nhoC8V3TVMX1ixYkXHv1988cWpv/CFL6SmtRFRjp/NN988\nNS1mnr88D2hd06KMKKur+VvYRJhVft3SMNoOfz87LTBeEeXx6G+KA+M0c+bM1LfffnvH78UOBBER\nhx9+eL8+X/4fWsm1rcw0KDag53WUzbHratvh7q4wnDQ1zY8oz9em+zD/zmsP7+ERZQrVQw89lJr3\nNlahszF6RDlvGep5QDtnHSIiIiIyKDg5FBEREZGk9bYy10X88Y9/XLzG6iA+rqXdyUqhrbbaKjUr\nVCNKO5H63nvvTc2G1u94xzuK7ceOHdvxNVY+0xbqVm3YyzYNvxutea4hzTVgzz333MZ9TZkyJTWP\nTV/skCZrIKKsvmSV3ymnnJKaj/oZL65HWts8vRSX+ruwUnGbbbZJTXuD6yzzuPD9H/3oR4v90pLp\ni5XJWNByqS1SWvZcz/rQQw9N3ZdGsCMB2rf33HNPaqbNRJSxqc/NtfmMiNI+PvXUU1PT7ueY/OUv\nf1lsv7afP1Kpj2u3xQ3W0GRf1qkXs2fP7vgazwOmM41mG7mG9i+7N0SUlnNTqhKvT9zXLbfcUuyL\n/+a8YbfddkvNNZtZnRxRjqOhvq755FBEREREEieHIiIiIpI4ORQRERGRpJU5h8zVePTRR1MvX768\neB9bzjCHpl5oew1sWXL//fcXr3GFCeaB8LuwlQdbbkSUeVtclYX0tWM9cxx6ObeHv+ekk05KzWO5\ncOHC1GeeeWax/aWXXpqaORrMy2AuHXNCli5dmvqSSy4p9nvXXXelZm4hWxIwlmeffXZqthaoV4Xo\nZfhdjzvuuNTMOWT7Gh5XnotsP1Pvl5p5UxwjPPazZs1KzZZUEWVrlC9/+cupmcfa1hZP/eGGG25I\nXceCbW6aWjwRXsfYZiOibOvF6yivXXPnzk3N8dLtM0cDPK51vjNzNpvy2Zrypa+++upiXwsWLEi9\n1157peaqReYZ/hceV173eN+PKGsV1ltvvdS8RrEm4de//nXqK664otgX8wzZtu6zn/1sarZzq9vq\nDCej7+oqIiIiIo04ORQRERGRZJ1uKykMMAP2QfzO119/feqjjz66eB8tw6ZH+HzMT11bVk2La0+b\nNi31sccem3rDDTcstm+yDZpoWgWk2/siYiC8nEE5IZpWIjnttNNSX3XVVcU2fLxPe4W/uek4sdVA\nt+NHe2HXXXdNzbY2tLTXNo4d6KkY8bjS7qdtwlVQ6oXomdbA17jN/PnzU3/nO99JTYu5To8455xz\nUk+dOjX1ABz/vtCzMWKqBdvaRJTXH9rytPs5Lm699dbUH/nIR4p9rVq1KjXjeuWVV6bef//9Uw+T\nxd/fOA36tY5tsyLKlCDGpSmliK1Q6jZSbP9Ei3ny5Mkd9zVM9EyMGBe2r6nHEa+DnEOw1RfHDldm\nY2pHRDkP4L2OseS1cpjoGCOfHIqIiIhI4uRQRERERJLW28pPPPFEai44HlGuxNFUYUz4mJ/WcURZ\nvfelL30pNav0+lptvLbU37fLvnvKDusLrIRkVXD975UrV6am/d8Eba66qnj8+PGpjz/++NR81M8F\n6wfYmmlFjDhenn/++dT1agKs9KNltnjx4o6aqQLsGrDPPvsU+z3iiCNSD8MqAT0VI45/2r20eCMi\nHn/88dSTJk1KzVWcWGF8xx13pGaMI8oqTVb7H3zwwal7oFq8ZyzLJuoq/BUrVqTm8WNcFy1alHr6\n9Ompea2MKK9X3//+91MPhU05hPekQYkR0ytoHUeUq/2wip9/f+SRR1LT3ue9JaKcKxxyyCGpe6kq\nObSVRUREROTFcHIoIiIiIkkrbWXCx/a/+MUvite+8Y1vpF6yZElqWmZs9suqyM9//vPFvtj8eIgq\nJl8KPWWHrS11w1hWftGOvPnmm1Ozmo8xmjBhQuptt9222O/YsWNT09ocIpus1TGqrxeMGa0axo5j\nlNYY7eK6MTzjMgxjrGdjxJSKuun/jBkzUrOa8oEHHkjN6lnua4sttij2xQblEydOTN1jjd970rIk\n9TWNFiabJ7PB9U033ZSanR14f4oo026YBjDC7kmDHqM6TYkpFkyJmTNnTmqmOW266aapP/WpTxX7\nGjduXGra/W2IkU8ORURERCRxcigiIiIiSettZSnoWTtMEmPU+/RsjHi9ZnV4RMTs2bNTX3bZZanZ\nbJx2PdfjPfnkk4t9bbzxxql7oCq5iZ63LGtbmeth33nnnalnzpyZetmyZalZdc716SPKNBpj1I8P\nqOZA/DfjxapkNoanrtMumhZs6DG0lUVERESkO04ORURERCTRVh5Z9KwdJokx6n1aESNWh9fQAmOF\nMi0wVriyA0OL6HnLsoY2M+OyevXq1KzcZxpA3dy6h21K0roYjUK0lUVERESkO04ORURERCRxcigi\nIiIiiTmHI4tW5EqNcoxR72OM2oH5bL2PMep9zDkUERERke44ORQRERGRxMmhiIiIiCRODkVEREQk\ncXIoIiIiIslQViuLiIiISI/jk0MRERERSZwcioiIiEji5FBEREREEieHIiIiIpI4ORQRERGRxMmh\niIiIiCRODkVEREQkcXIoIiIiIomTQxERERFJnByKiIiISOLkUEREREQSJ4ciIiIikjg5FBEREZHE\nyaGIiIiIJE4ORURERCRxcigiIiIiiZNDEREREUmcHIqIiIhI4uRQRERERBInhyIiIiKSODkUERER\nkcTJoYiIiIgkTg5FREREJPk/EA9sgzZr0Z8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85962a6cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGXlJREFUeJzt3XnwXfP9x/FP6EJalBJSBEGpILQkNLFUm4ipCGrGWCY0\nmmZoB12G1gRdZlSXYaYVnRoJgimamlqSUXtsoVoxqTVVpI1IIhpppXZ+f/wmb8/Pp/dc33zXc+/3\n+fjrlXy/59z7PZ97zj1z3p9lwHvvvZckSZKklFJap6/fgCRJkurDm0NJkiQFbw4lSZIUvDmUJElS\n8OZQkiRJwZtDSZIkBW8OJUmSFLw5lCRJUvDmUJIkScGbQ0mSJAVvDiVJkhQ+1Iuv5SLOPW9AN+zD\ndupZtlH92UatoavtZBv1PNuo/hq2kU8OJUmSFLw5lCRJUvDmUJIkScGbQ0mSJAVvDiVJkhS8OZQk\nSVLw5lCSJEnBm0NJkiSF3pwEW+p27733XsNMAwYMaPpvSZJ6Uke/n+rCJ4eSJEkK3hxKkiQp9Muy\n8uuvvx553rx5kS+44ILIDzzwQLbNW2+9FXn99dePPHHixMjnnntu5I997GPZ9nV9dFwn5WP3t99+\nO/J//vOfyPPnz4988803R3700Ucjf/KTn4x85plnZvvdY489In/4wx/uwjuWpM7h9e6dd95p+P/r\nrLNOw5yS3ymtgG35xhtvROZ3G7+DPvKRj2Tb92Ub++RQkiRJwZtDSZIkhQFVI2h6QK+9UCMsS550\n0kmRZ8+eHZnl5vK4VB0nPhI+8MADI//hD3/Ifm/gwIFr94Y7pzueQfdqO7Gc8u9//zv72e233x75\niiuuiLxgwYLI//rXvyKz7LLhhhtGPvjgg7P9nn/++ZE33XTTyL30CL/l2qgzqkaRVx3jmpXIWr6N\neMzZJebNN9+MvO666zbM5b/LcmZHtMi51Kdt9O6770b+73//G/nll1+OzHbgtSql/y1BrsFjX4Pz\nqqXbaG2xTVNKadmyZZHvv//+yK+++mrk3XbbLfKwYcOy7T/60Y9G7sG2bLhjnxxKkiQpeHMoSZKk\n0NajlVlCOfnkkyPfcMMNkTlqiMpRrHy8zzINHyPfe++9kWfOnJltP2XKlMg1eNTfp3j87rvvvshT\np07Nfm/JkiWROdJrgw02iLz33ntH3mijjSIvXry48vVfeeWVyBzV3N/bpTOqRuM9//zzkR977LHI\n2223XeTNNtssMtshpbxkxnOvZiWzWmFbvPbaa5HnzJkTee7cuZHZ1absNrPPPvtE5vFnN47ly5dH\n3mKLLbLtR44cGZmzA/Dc7Q9t2azbFr97PvSh97+K2QWJXW3Kbjfrrbde5KouAjyPXAygZ7Ady1lO\nzj777MjPPfdc5O233z7yUUcdFfnTn/50tj3Lyr3NJ4eSJEkK3hxKkiQptFVZuXyE//e//z0yR74S\nR4Dtv//+kVmuTCkfyXzXXXdFfvjhhxv+zmWXXZZtzxHS/XHiZT565wjxH/7wh5EXLVqUbcNH73vu\nuWdklql22mmnyKtXr478+OOPR+bI5ZTycoyllbVTnmOrVq2KPGPGjMi33HJL5BdffDEyP/ssmbBN\nUkpp0KBBkceMGROZn4Ntttkm8sYbbxy5HHnbrsq2YHeNe+65J/KFF14Y+ZlnnonMEbLlyNe//vWv\nkXld4+wC7LbD/08ppenTp0cePnx45EmTJkUeMWJEZC4awBJrSu3Vnh1ZX5eLLLBdyhIjjznbvqqr\nVE8e1/52HeUx/sUvfhH5vPPOy36P3TvYfp/4xCca/n93lpHLkdNrO+uATw4lSZIUvDmUJElSaKuy\ncvk4naOS+Xiepcgrr7wy8tChQyv3zZF93P7pp5+OzDINR2umlI/k7C9lZT7W/sc//hGZIyY5Inmr\nrbbKtj/ggAMijx8/PvLHP/7xyFUjj1mKLEteHNWsD8ZSGD/vKaV0+umnR+ZofX7G2V4crbpy5cqG\nOaX8fLn11lsjs+Q2evToyOyewBF/7bweLUuJKaU0bdq0yCxv8Rzh8WAbsYyfUkr77bdfZHbv2HHH\nHSNz5DK7DqSU0iOPPNLw9c8666zInPyXs0nstddeqdVUlYvLaw/LjNyG309so2YTyLNMzBI/24Xf\nSeUk2jwXq9Zw5mvy/5uVLMvydbvgMf7+978f+eKLL47M61ZK+TnG7x1Ods2uMp2ZcL5KV/flk0NJ\nkiQFbw4lSZIUvDmUJElSaKvOAWU/iE996lORTzzxxMgnnHBCZE6Hwf4V5b74s0022SQy+5RUTUfQ\naH/tin1kOAXGgw8+GPnRRx+NzH6e3/72t7N9cToh9q964oknGr4e+9QMGTIkcjlNR9WC9XofP68L\nFy6MfMQRR2S/9+yzz0bmZ/4LX/hCZPZf47RCL7/8cuSnnnoq2y/7sHH6FfZfe+GFFxrui+dkd/bh\nqQP+bZyqK6WUfvazn0XmcSJOlbH77rtHLs+9sWPHRq6a+qlZ/032deO0OOecc07D98ipbNgvr/xZ\nndqT5wjbhZnHIaXqaWaoarqgclv+jKun8FxiH+HPf/7zle+fbcyVi9hPrllf+aoVWlq9jy+PMfvx\nXnTRRZH53VQeo8MOOyzyV77ylcjsF83vwGbHq6r/aUemR+qM+pxpkiRJ6nPeHEqSJCm0VVm5fLzK\nKRE47cWWW24ZuerRa/kI/9VXX408f/78yCwb8HH6zjvvnG3flwto9yaWKlj2++Mf/xh58ODBkUeN\nGhWZK9SklLfnP//5z8hcwPyQQw6JvPXWW0fmdAplG7OdeuqRfCuqartjjjkmclnK5HQYU6dOjTxx\n4sTInMqGx56vV07L8tJLL0Xm9Cd33313ZJYb2aWgaiqOdsCpMq644orsZ+zGwRI/p4jilFBTpkyJ\nzGtiSnnXi84cQ7YNy2YjR45s+L74O+VqOXUqJfN6wc8s24XXKk7VlVLerYLfCSzfcioadungakQp\n5d1uWIrmucvPwfLly7PtuaoQ25jnHo89z93yfC0/P62MfxtXNps1a1bD32E7Hn300dm+OOUNp1rj\nal7sRlF+9nnMq6YbajbdUVfU56yTJElSn/PmUJIkSaGtyspl+YGPaFkWXrp0aeSqEcblKDOOuLv9\n9tsj85EuS2ynnXZatn07j5DlMeDoLo4q5vFkOeTAAw+MXK4mwDLKvHnzInPllGajkhu9v7X52Rrt\nVpok/v0sZ/30pz+NzNVtOFI/pZR+85vfRGaJn6P2OnL8ylUVWILhIvUDBw6MzFUGNt9888gsxbQD\nthFXXuKo/5Ty84dlXZb4v/71r0dmibMnV5JZsWJFZM4gscMOO0SuKp/VTVUJj38ju1785S9/ybbn\n55ffNxy9+thjj0XmdZMrc6WUd89hW+66666R2T2q7CrFbh0c7c/MazBHl7MLT7nvVlshpVzVhKXg\n6667LjL/Rt5b8PcnTZqU7YvnIVdZmzNnTmSWlcvVgViu53HlNZHXWpa4u3pO1/cslCRJUq/z5lCS\nJEmhtZ7/foDysSkfA19++eWROdqVIyknT54cuRxdzMfAf/vb3yKzNDBu3LjInAS40XtrJxwlyUmR\nOfE1y80sSZSLwRNHzR100EGRWTbpyKjKcgJylt9YUmDJiI/wqyYBbgdsF44oX7x4cWSOMC0nS2a3\ngO6c/JZdEm699dbIbDuOvOV53M54fpUTh1dNsMxSJktQ/LyX5wh/VlXmbdbGPK94veRkzSyLtkop\nsmoScB4jjsIuR/XOnj078sMPP9xwXzvuuGNknntjxozJ9sURzlVtxHYtPx9sY567LHMy87pddi+p\nc1eANfj3slw+bdq07PdmzJgRmceMJeLjjz8+Mr/3FyxYkO2L1y62N6+vvIeYPn16tj2P6xZbbBF5\n+PDhkSdMmBB5n332abhtZ9S/RSVJktRrvDmUJElSaI1n+R1UPkZlyeyBBx6IzHIMSy6cFLQc+coS\nKUszLCGcfPLJkfvLpNcp5aOwnnzyych8dM4SCEslHGVXlpZGjBgRmW3bkZGwzUYhs9TCyUg5eS1f\n7zOf+Uzkcs3sVsTS7LJlyyJzlCXbhe3Atiut7YTibIdygl9OqM3P18EHHxx5t912i9xuI5Sr8JiV\n17uqNaV5bDkinde4suTIayGPLc9RvgavtSmldOONN0a+7bbbIn/uc5+LvO222zZ8jTp33eB747Hg\nqNJBgwZFLrvNcCQyuyrxPGTXAbbp1772tcr3UqVZ1wHiZ4HXu6rR2c0WFqgrfka5xvfVV1+d/R67\nPvAzztlIOIPDt771rchs35T+d53wNaoWYmjWvYPd4RYtWtTwd7heOq/hneGTQ0mSJAVvDiVJkhS8\nOZQkSVJo6z6H7OfGvmKs63Malptuuily2YeC++bUJpzZnlOs9Cfst8m+GFzlglONbLbZZpHZN7Ps\nx8K+hV1R7pdtyZVz2C+VfbPYt4ht3ArTN6zBfinsx7RkyZLIbDv2leK0FZyWJKV8BQdOX7LddttF\n5ueDfdvYb+ayyy7L9vv0009H5qoa3/nOdyKzX1yd+6l1Ff82TmHBa09KeV8pthmvcVzlgn2wuOJC\nSim9+OKLkXkesy3Z3ldddVW2/Z133hmZ5/h+++0XudWniKrqf8jvDk4/klJKp5xySuQ99tgjMvto\nzp07NzKnP3nooYeyfXEaqaqpgMq+oMTra9WUYB2dHqyu10Je9xYuXBj50ksvjVzVL7DE3+MYBPaJ\nLo8LjzH7onIbKldmq5qeiucOr4/d2Q71bFFJkiT1CW8OJUmSFNqqrFw+Audj3NNPPz3yGWecEZll\ntaVLl0YuH8/yMS7LLEOGDIlcNTy90XtrJyxpcHZ/lunHjh0beW1LGJ3RbF9sJ5a4uZoBSzssIVx7\n7bWR2W2hlbC0eMMNN0SuKivzWMycOTPb15///OfI5QL2a7A0wmO2zTbbRGYZM6W8PHPMMcdE5uer\nrqWsnsQS0i9/+cvsZ2wnlhNZfuZ0HCx5lt1o2GZV08ywTHfhhRdm2/P6d9xxx0Vmd4NWmP6ko6qu\nY+WUaDyvDj300Mi8Ps6aNSsyV9h45JFHsn3xXGA3Apb+eYzL86Uj3Xb4tzSbHqyu32+8jrCUXK5c\nQ/xb+L3PrmnMw4YNi8yuLinl7bLTTjtFZrvwvZTda1asWNFwX0ceeWTkiRMnNnxfXW2T/nd1lSRJ\nUiVvDiVJkhTaqqxcPvbmY/TPfvazkb/85S9H5igxrpBRjhKqWoCbZQKOAi3fS7NH8mvU9dH8B+H7\n5mN1HieW7/kYvTdWkimPK//NEsyoUaMi33LLLZH/9Kc/RWZph6MFG71OnfCz+fvf/z4yyxYrV66M\nzJV/eIy4n5TyUa7cnqVrdjvgqPXnn38+cjl6j6WaI444InJVl4SqlRzaDY/l1ltvnf2M1zWWj3k8\nWEpsttIQjzOPLUc+c8UOrt6QUt5dY/LkyZHbYYWhruBxZmmR5cujjjoqMo/9ggULsn3x3GUXqqp2\nbXYdXFvNvt/qev6xrMvjXY7o5rWPXSK4us+ee+4ZmedK2bWGx4Ilbt5PsDvP448/nm0/b968yOPG\njYv8jW98IzLLzd3ZVcMnh5IkSQreHEqSJCm0dVl59erVkTnBMcuaO+ywQ2SWtjihbEr5I2Fuz234\nSLksl7LEXfUIvjOLz9fhET5LXbvsskvknXfeOTInPObI1KFDh0bui9GLfE2OQl+2bFlktnfVpKR1\n0KzrwvLlyyM/88wzkdmVYt9994187LHHRuao4majWjmB6xNPPBGZI/p5jhxyyCGRy9IOPzssn1ad\nI1XnQTvPGlCONmWJn6UuXns6MzsAuxJcdNFFkVnmLCf/5QjpzTffvOF70fvYFvzuGD16dOTbbrst\n24b/5vlS1d7dqVXake9z/PjxkXk/UF57Tj311MgsJfN7jvvl90M5CprnDhdW4OtzonN2tUkp7w53\n1llnRWZ3th5r4x7ZqyRJklqSN4eSJEkKLV9WZtmoLAVzgt/77rsvMh/bjxw5MjInmOXarinlpVCW\nz1guveOOOyI3mwxz1apVkVmy4+NtjpjiyKqUmo8y7At8xM7RqBMmTIh8zjnnRP7e974Xedq0aZH5\nqDylzpXZ1xY/P3fddVfkF154oeHvb7vttj3yProDj1E5qphdLPgzfv44SpIj+9imzdap5s+23377\nyDzGXJuZ61qX++UavFUjlDuiDudHT2m2vm1Xzx222VNPPRX5vPPOi8zPUTkK+eqrr274XvTB2F6c\n8eHee+/Nfo/nT2+Ukpt1SeBnsa7nHEfQf/e7341crmfMEn2zScQb/X+5TjPbjJlrj7PcXK7FzVHJ\n7J7RG8fYJ4eSJEkK3hxKkiQptHxZmaWN6dOnZz/77W9/G5mPa1lK5v9zNFI52phlYpbGWHLhBJZl\nyYejmFg+/dKXvhSZ686OGTMmcjnZLd9nHVRNssuRXpyg9+67747861//OjJHyKaU/91sj66WUPiZ\n4drALHdzFDpHXvI91bV80sjgwYMjszzBzOPCUcUsbZUlwo6MfuW5cNppp0XmOcHXSCk/5q10nHsa\ny73sKpBS3jYdKcU3G93Oz//RRx8dmaVMttnPf/7zbHt+3vqbjix40NHt2b2Fxz6l/DpYruHc2+o6\nepmffX5v8rpXjlbmdxivUbyO8VrJmSBuuummbF/XXXddZE52zX2x7c4+++xse84g0dvdM+rZopIk\nSeoT3hxKkiQpeHMoSZKkUK/Oa53APgFcpDqllBYvXhyZw8XZn4r9/FauXBm57DfCPjj8GfdVDokn\nDo9nfyquasDpOwYOHBi57L/IVSnqNk0E+3jwb+CQ/OHDh0e+8cYbI5977rnZvl5//fXIX/ziFyOz\nDxT7grLfC4/Zc889l+33/PPPj/y73/0uMtuPn4sZM2ZELvui1lX5uWBbTJo0KfKDDz4YmecI+9Tw\n81b2Lapa+Yfb87xkH0/2AeJKRSmltNdee0W2z+H72Ba8XqWUtzGnvypXUlmD7cXrW0opjRs3LjL7\nVfNz9ZOf/CTylClTsu37c5s163NYNeVL1TRUXDGD7ZtSPl1Ub/dDb8WVh/geN9hgg8i8T0gpXzWK\n/TyffPLJyPfcc0/k+fPnR2a/wpTytuR3B7/PJk+eHHnUqFHZ9n05vsAnh5IkSQreHEqSJCm0VVm5\nXNWEZTLm8jHyGlUz06eUz67OsjBLwSyXcuWJlKrLxIMGDYrMUlCzqQFa4RF+iSsoHHDAAZF5nH/8\n4x9n2zz00EOR586dG/lXv/pVZK7kwbIwSwBl+Y3lapbJNtxww8g/+tGPIo8YMSJyKx77lPLPE0vx\nBx10UOSlS5dG5qoo3LZceYVlTp6LLOWzjE+77rpr5JkzZ2Y/K0to+n+8dpQrQhG7RVR1t2D57NBD\nD82257WUpa0zzzwz8imnnNLwNfo7XiN4TpT/ruoSxK5K7PbCUmhKeem/N7oXdXWKnr7GduH0NcuW\nLct+b+HChZFZMuY5we+XV155peFrpJRfx9hevCby3qJO3cQ8oyVJkhS8OZQkSVJo+bJy1QzoKeUl\nr448Emfpk+XKlFKaOHFi5PHjx0fmI2GOCuxo+bFVy5RdwRnhWbKdNm1a9nuXXHJJ5Dlz5kTmiLBF\nixZF7mh7c4T42LFjI3NENbsLtFsb8fjznBk6dGhkHj+WwsoyGUv5HPHKVXB4jnAVmgkTJkQeMmRI\ntt92O+bdhe3FbhAp5aOK2U4c8Tpr1qzI119/fWR2u0kp/4ycccYZkTmjQJ1KYHVStWJUStWrbKxY\nsSLyHXfcEZkj/ffff/9sX7vvvnvD1+wpza6prXC+Vs2kwe5cKaX07LPPRuZoZXZH47HgfQO7iaWU\n0le/+tXI7IbB169rl4x6vitJkiT1CW8OJUmSFAb04gikHnkhjjz95je/mf3s2muvjfzaa69FZjlk\nq622ivyDH/wg8uGHH57tq+oxcM0ep3fHm+mzIWnlZ5ElGI4u40gxTlDNUX4sue29997Zfvlvjirv\npQlHW6KN2BZVk1uX/2ZZn23B0hozz6Py2PdxybK2bcTj/dJLL2U/u/zyyyNXTTy+evXqhpmlsZRS\nmjp1auRTTz01clmC62Ndbadev9ZVjTafPXt2ZF7TlixZEpntkFJKJ510UmR2A6iZWrYR26FcvGLV\nqlWRFyxYEJnlZn6/sDvOsGHDsn1x1gde01rhvsEnh5IkSQreHEqSJCm0fFk5e4EO/i01e6TbnWpb\nDlNo6TbqzPWC21SdezU7J2vbRlXruqeUT6zLLjUsP7O0teWWW0a+4IILsn2NHj06cl+u7/oBalmy\nbPqCaD92w+Aa59dcc03kTTfdNPKJJ56Y7Ysj/Hu7G8ZarK1cyzaq6jaTUt4u7I7GbjBcJ5nnR3kc\nanZdq2JZWZIkSc15cyhJkqTQVmVl1bccptDSbdRPum60RBuVE5KzzMwRyhzxOnjw4Mj77rtvZJbJ\nUmqZ9qtlybLDL14xCwD/v9kIV9tI3cSysiRJkprz5lCSJEnBm0NJkiQF+xy2l5boK9XP2Ub1Zxu1\nhrbpz1b1Pdwi/QqbaZs2amP2OZQkSVJz3hxKkiQp1Hbqe0mS2lVHVg6S+opPDiVJkhS8OZQkSVLo\nzdHKkiRJqjmfHEqSJCl4cyhJkqTgzaEkSZKCN4eSJEkK3hxKkiQpeHMoSZKk4M2hJEmSgjeHkiRJ\nCt4cSpIkKXhzKEmSpODNoSRJkoI3h5IkSQreHEqSJCl4cyhJkqTgzaEkSZKCN4eSJEkK3hxKkiQp\neHMoSZKk4M2hJEmSgjeHkiRJCt4cSpIkKXhzKEmSpODNoSRJksL/AfjttejZpwGEAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f85960aa550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Coming soon..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  },
  "nav_menu": {
   "height": "381px",
   "width": "453px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}