diff --git a/15_autoencoders.ipynb b/15_autoencoders.ipynb new file mode 100644 index 000000000..b483acfe2 --- /dev/null +++ b/15_autoencoders.ipynb @@ -0,0 +1,2077 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Chapter 15 – Autoencoders**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_This notebook contains all the sample code and solutions to the exercices in chapter 15._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# To support both python 2 and python 3\n", + "from __future__ import division, print_function, unicode_literals\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import numpy.random as rnd\n", + "import os\n", + "\n", + "# to make this notebook's output stable across runs\n", + "rnd.seed(42)\n", + "\n", + "# To plot pretty figures\n", + "%matplotlib inline\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "plt.rcParams['axes.labelsize'] = 14\n", + "plt.rcParams['xtick.labelsize'] = 12\n", + "plt.rcParams['ytick.labelsize'] = 12\n", + "\n", + "# Where to save the figures\n", + "PROJECT_ROOT_DIR = \".\"\n", + "CHAPTER_ID = \"autoencoders\"\n", + "\n", + "def save_fig(fig_id, tight_layout=True):\n", + " path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n", + " print(\"Saving figure\", fig_id)\n", + " if tight_layout:\n", + " plt.tight_layout()\n", + " plt.savefig(path, format='png', dpi=300)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A couple utility functions to plot grayscale 28x28 image:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_image(image, shape=[28, 28]):\n", + " plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n", + " images = images - images.min() # make the minimum == 0, so the padding looks white\n", + " w,h = images.shape[1:]\n", + " image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n", + " for y in range(n_rows):\n", + " for x in range(n_cols):\n", + " image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n", + " plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PCA with a linear Autoencoder" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Build 3D dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "rnd.seed(4)\n", + "m = 100\n", + "w1, w2 = 0.1, 0.3\n", + "noise = 0.1\n", + "\n", + "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n", + "X_train = np.empty((m, 3))\n", + "X_train[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n", + "X_train[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n", + "X_train[:, 2] = X_train[:, 0] * w1 + X_train[:, 1] * w2 + noise * rnd.randn(m)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Normalize the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn.preprocessing import StandardScaler\n", + "scaler = StandardScaler()\n", + "X_train = scaler.fit_transform(X_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Going to need TensorFlow..." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's build the Autoencoder:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_inputs = 3\n", + "n_hidden = 2 # codings\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "hidden = fully_connected(X, n_hidden, activation_fn=None)\n", + "outputs = fully_connected(hidden, n_outputs, activation_fn=None)\n", + "\n", + "mse = tf.reduce_sum(tf.square(outputs - X))\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "training_op = optimizer.minimize(mse)\n", + "\n", + "init = tf.initialize_all_variables()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_iterations = 10000\n", + "codings = hidden\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for iteration in range(n_iterations):\n", + " training_op.run(feed_dict={X: X_train})\n", + " codings_val = codings.eval(feed_dict={X: X_train})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure linear_autoencoder_pca_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEgCAYAAABb8m8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X+cXHV97/HXByThR8LGEAKB8CugmFJBd0MACSmivVQB\nvRKr3UKB+gsfqLddq2It6qNWaq1t4aq9XkRqS9UV0KBQtFotikYwsKuN3qJAIoFgTALBJUIIAT73\nj++ZyczJzOz8OGfOfGfez8djH7tz5vz47Jnd8znfn8fcHREREYA9ig5ARER6h5KCiIiUKSmIiEiZ\nkoKIiJQpKYiISJmSgoiIlCkpiIhImZKCiIiUKSmIiEiZkoKIiJRFlRTM7LfM7HozW2tmj5vZFjP7\nrpmdXXRsIiL94DlFB9CiI4BZwD8DvwT2BVYAN5nZW9z9MwXGJiISPYt9QjwzM2ASmOnuv1V0PCIi\nMYuq+qgWD1ntQWBO0bGIiMQutuojAMxsX2AfYAh4NfAKYLzQoERE+kCUSQH4e+Di5OdngS8D7ygu\nHBGR/hBrUrgCuAE4BHgdsCcws97KZnYAcCZwP/BkF+ITEcnT3sCRwDfc/ZEsdxx9QzOAmf07MNfd\nl9Z5/w+Bz3c3KhGR3J3n7l/IcoexlhTSvgz8XzN7nrvfW+P9+wE+97nPsXjx4q4GljY2NsYVV1xR\naAy9FAf0TiyKY3e9EoviqHb33Xdz/vnnQ3Jty1K/JIV9ku9Ddd5/EmDx4sUMDw93J6I6hoaGCo+h\nl+KA3olFceyuV2JRHHVlXh0eVZdUMzuwxrLnABcC24H/7npQIiJ9JLaSwlVmtj9wG/AQcDBwHnAs\n8E53f6LI4EREYhdbUvgi8EbgrcABwDZgAni3u99SZGAiIv0gqqTg7tcD1xcdRydGR0eLDgHonTig\nd2JRHLvrlVgUR/f0RZfU6ZjZMDAxMTHRa41EIiItm5ycZGRkBGDE3Sez3HdUDc0iIpIvJQURESlT\nUhARkTIlBRERKVNSEBGRMiUFEel7mzbBsmVw9NHh++bNRUfUu5QURKTvrVgBq1bBunXh+7nnFh1R\n71JSEJG+t3Fj49eyi5KCiPS9BQsav5ZdlBREpO+tXAmnngqLFoXvK1d257gxtmVENfeRiEg75s+H\n73+/+8cttWVAaM8499xi4miFSgoiIjmJsS1DSUFEJCcxtmWo+khEJCcrV4Yqo40bQ0LoVltGJ6JK\nCma2BLgIOB04EngEuAO4zN3vLSwwEcncpk2hTr7ygjp/ftFRtaaotoxORJUUgEuBlwA3AGsIj+N8\nBzBpZie5u57RLNInYmyk7QexJYW/B0bd/enSAjO7Hvgp8F7ggqICE5FsxdhI2w+iamh29zsqE0Ky\n7D5CUlhcTFQikod0o+yWLXH0849dVEmhgYOAh4sOQkSys3IlzJ696/W2bZ3PWRTjYLJuiz4pmNn5\nwKHAF4uORUSyM38+HHhg9bJOq5A0Md70ok4KZvYC4JPAKuDagsMRkTY0unvPup+/2immF1tDc5mZ\nzQduAR4Fft/dfbptxsbGGBoaqlo2OjrK6OhoPkGKyLQa9TLKup//ggXhGJWve934+Djj4+NVy6am\npnI7njVxLe05ZrY/8F1gIbDM3X8+zfrDwMTExATDw8PdCFFEmnTkkbB+/a7XRxwB99+fz7E2b949\nycQ29gFgcnKSkZERgBF3n8xy39GVFMxsJnAzcAzwsukSgoj0tq1bG7/OUoyDybotqqRgZnsA1wMn\nA69y99UFhyQiLUqPVB4aCj2LSubOLS42iSwpAP8AnAPcBMwzs/Mq33T3zxcSlYg0Ld2GUNntFGDh\nwun30Q9TYPSq2JLCCYATEsM5Nd5XUhDpcekeP3PnwvHHt9aYrCkw8hNVUnD3lxYdg4g0r9YdfboH\n0MKFrV/Qm+laqtJEe6JKCiISl1p39Fl0M22ma6lKE+1RUhCR3NS6o8+iB1AziaVeaaLZEsSgljSi\nHtEsIr0tjyePbdrU3FiDesdudqqLQZ0SQyUFEclNHk8ea7ZaqN6xm53qYlCnxFBSEJHc5DFYrNHF\nupkqn2anuohxSowsKCmISDQ2bQrPVahUebFuphTRbOklxucrZ0FJQUSisWJF9ejnWbPgqafCDKsL\nFsCGDdXr16ryabb0MqhTYigpiEg00hf5nTvhzjvDz7VGR9eq8hnUXkXNUlIQkWik6/nT6o2OrkwE\nW7bsKm1o/MLulBREJBrpev6dO2F1xbSY9UZHV7Y1pA1Kr6JmKSmISNOKrnpJ1/PXej5CLY0u/IPS\nq6hZSgoikevmhbrXpo5otjE4Xe00e3Z4/vMg9SpqlkY0i0SumyNv03fcd91V+9nKWWv0HOdmrFwJ\np54KixaF7/fdB2vXhoSiRuZqSgoikevmyNt0VcuOHbsno04v4LV0mvhKJQolgukpKYhELo/5heqp\nvOOeObP6vVIyyqPkUlQJZRApKYhELl01kmcdeeUd95Il1e+VklEeJZdmSigleZRUBklUDc1mth/w\nHmBp8vVc4CJ3v7bQwEQKVNTI23rTQOQxZ1DlsR56KCSFknTS6bXG8NhElRSAecD7gfXAj4HTC41G\nJHKd9Fyql4zymDOo8ljLllWPOUgnnUGd3TQrsSWFXwIHu/tmMxsB7iw6IJGY5XFXnXfJZbqk00lJ\npehxGL0gqjYFd9/p7qohlKh1WuedZZ15N++qs4p7up5EnbSxDOqDdSrFVlIQiV6nd+dZ3t1385kB\n3arr76SkoqqnyEoKIv2g0wtPlheubvZciuGC283uvb1KSUGkyzq98KTXf+gh2H9/OPLI1qtlujmo\nK4YLbjeTZK8ydy86hrZUNDRP2yXVzIaBieXLlzM0NFT13ujoKKOjo/kFKpJSaxK3Vi7GmzfDOeeE\n5wjU+vc99dTe7ILZ6e9dpCIboMfHxxkfH69aNjU1xW233QYw4u6TWR5voJLCxMQEw8PDXYlPJE/p\nbpmVFi0Kd/6SnfT5LjrxTk5OMjIyAjkkBVUfieSo2R43rfbM6depoHupZ1alGNpDsqKkIJKjZrs4\nttoVst6Ff+bMuOvBO+0SmleX0hjaQ7ISXZdUM3sbMAc4NFn0KjM7LPn54+6+rfaWIt1X6w6zVv10\nq3eipQFcd91VPeXDkiXx1NPX0ks9syrlMUq7V0WXFIB3AYcnPzvwmuQL4F8BJQXpGelxABs2wKGH\nwjPPhNel/vqtjhco9Rpq9sljseh03ERe4y6Kml+qCNElBXc/qugYRJqVvqN/6qnd19m4EW6/vb2L\ne79drDq9I+90e01zEWFSEIlB+uJy8MGwfn3tdRcs6L+Le7s6PQ+dbq8ZVpUURDKRTgI7d8Lq1eG9\ndevCM4FrmT07/iqffjJIvYzqUVIQ6UApGVQ2+K5bt/tTyebOheOPD20KW7eG1wsXDmb1RC/r5lxQ\nvUpJQaQDldUNjSxcOHjVEDEapF5G9SgpiHSgXvXC8cfDjBmDfXGJkdp2lBREWpJuO5g3r7q6wSyM\nFfi3f1O1kMRJI5pFWpAeMete3YjsHkoISggSKyUFkRakq4seeQQOPLDxOpK9vOY4EiUFkYbSF595\n86rfX7BgsObF6RV6bGZ+1KYg0kB6MNOJJ4Zpk9MNyIPeY6XbNJ4gP0oKIg3Uqi6q9ayCQe+x0m0a\nT5AfVR+JNKCqod5U67GZmzbB0qWw997h66ST1NbQDpUURBrQYKbeVGs8wbJl4RGlJatXD+bcRZ1S\nUhBpQIOZ4lGrXUFtDa1T9ZEMHHVn7E+1qvZU3de66JKCmc0ws4+a2QYze8LM7jCzlxcdl8RD3Rn7\n08qVoXfYzJnha+lSVfe1I8bqo2sJT1q7ArgPuAj4mpmd7u4/KDIwiYO6M/an+fN3TVcu7YuqpGBm\nS4HXAe919/e6+2eAlwHrgb8tNDiJhnoUidQXVVIAXgs8DVxdWuDuO4BrgFPM7NCiApPeMV2bQa3u\njCISxFZ99CLgHnf/TWr56or3H+puSNJrpnukonoUidTXUUnBzP7EzB4ws0fN7BNmNseCy8zs3VkF\nWWEBUKsGeCNgwCE5HFMiozYDkfa1nRTM7COEevxngV8BZwK3Ey7QL3H3j2USYbV9gB01lj9Z8b4M\nOLUZiLSvk+qju4E57r69tMDMzgPGgNFOA6tjOzCzxvK9K96va2xsjKGhoaplo6OjjI7mFa60Iv0A\nm3afX6xRyNJPxsfHGR8fr1o2NTWV2/HM3bPZkdmFwIXAa9x9Klk2E3gnMB84hnAn/353v73NY3wT\nOMTdfzu1/AzgW8A57n5Lje2GgYmJiQmGh4fbObR0wbJl1c87PvVU1f2L1DI5OcnIyAjAiLtPZrnv\nTHofmdnbgZcCZ5YSQuIvgC+4+5i7nwN8Gbg16Vrajh8DzzezWanlJwOevC+RStf9//CHGnE86DT6\nvPs6Tgpmdhmwj7tf5O47K5bPAv6MUHoAwN0/BWwF3tfm4b5EqPJ6S8VxZhAGsN3h7up5FLF03f/T\nT2vE8aBLjz4/++ww+2lpJtSlS5UostZp76O/An6UblQ2s7MIDdCb2b3xdz3wvHaO5+6rgRuAjyRT\nXbwZuBU4AnhPO/uU3lEaP/CcVEuXeg8NrvRnv2ZNGLW8Y0f4uvNO3TRkrZPeRx8AbknX4ZvZYcCb\n3P0Jdz/K3f88temRhOkp2vVHwJXA+cD/BvYEznL3VQ23kp5XGj9w0knVy9V7aHA189nrpiFbbfU+\nMrNLgTcCp5rZl4HPu/vjZvZi4DrgbXW2OxM4APhwm/Hi7k8BlyZf0ofUe2hwpXugXXUVXHzxrtc7\nd+4+v5FuGrLVclIws9OAFxKqgBYCq4ArzWwLcBjwaXf/jxrbzQQ+Bvy5u9+Zfl+kRCOOB1d6NPrF\nF1f/LWzeDOecA//1X+H18cfrpiFr7ZQU3gW8PrljX5e0H3yFUI3zPnf/mzrb/SPwT+5+ZXuhikiM\nWhl/Mt1o9PnzQ680yU/LbQru/mp3f7Li9aS7H+7uC+slBDN7H/CdUkIws6PajlhEek6jrqOtPL9C\no9GLl/ssqWb2R4RJ7D5XsfgNeR9XRLqn0YW/lbmoNINt8XKdJdXMXglcAHwraZyGUM00VH8rEYlN\nowv/ggUhWVS+rkftScXLLSmY2VxCT6R9gTNSb6vnkEgfaXThV2+yuOSWFNx9KzA7r/2LSO9odOHX\n3X9cYnvIjoj0IF34+0dsj+MUkYhpgrvep6QgudJFQCq10j1ViqHqI8nVOeeEScsgXAjOPnv3aQpk\ncOhRqb1PJQXpyHQlgTVrGr+W+HRS+tPgtN6nkoJ0JD1XzbnnqsGx33Xymat7au9TUpCOTFcdcMIJ\n1dVFJ5yQf0ySr06qgNRLqfep+kg6Ml11wM03V09bcPPN3YtN8qEqoP4WTUnBzA4G/hRYCiwBZgGn\nu/tthQY24KarDtCdYf9RFVB/iyYpAMcC7wbuBdYApxQbzmBLT4d8++31p0OW/qJE399iqj66CzjA\n3V8AXFF0MINO/c1F+lM0ScHdH3f3XxcdhwTqb97/NPBwMEWTFKQY9S4MamzsfyoNDqaY2hSkAPX6\npKuxsf+pNDiYCkkKZmbAjGbWdfcdOYcjiVrP0q13YVBjY/9r5eE40j+KKiksB25tYj03s8Xufk8W\nBx0bG2NoqPqhb6Ojo4yOjmax++jVKhXowjC4VBrsDePj44yPj1ctm5qayu145u657bzuQc0OAs5s\ncvUb3X1bavsVwPXAS5sZp2Bmw8DExMQEw8PDLcc7KI4+ujoBLFoUupqmLwzqeipSrMnJSUZGRgBG\n3H0yy30XUlJw903AtUUcW+qrVSpQNdFgqVWFqJuAwaKGZilTdYFogkOJKimY2WWAA8cBBlxgZqcB\nuPvlRcbWD1QqEPU4kqiSAvAhQlIg+f7HFT8rKYh0SB0LJKqk4O4abCeSI1UhSlRJQVqnhkNphaoQ\nRXfefU5TFQwWzVcknVJS6HNqOBwsugmQTikp9DlNXDdYmr0JUIlC6lFS6GObNsHOnTBzZvg68UQ1\nHPa7Zm8CVKKQetTQ3MdWrIDVq3e9njFDjcz9rtneQ6pWlHqUFPqY/vEHT73eQ+leaAccoPEIUpuS\nQh/TQCQpSU9fsXQpnHqqxiPI7pQU+pgGIklJupT48MOwdm0xsUhvU1LoYxqIJCUqNUqz1PuoT6iL\noTSycmWoLlq0KHxXqVHqUUmhT2jKY2lEpUZplpJCJCp7j8ybB+7wyCPTP0tZRKQVSgqRSJcESvQs\nZdGkh5IltSlEotGd/4MPauTyINPoZMmSkkIkGt35P/poGLm8Y0f4+tnP4JRT1OA8KFR1KFmKJimY\n2Rlmdo2Z/dzMHjeztWZ2tZkdXHRs3VDZe2Tp0lAaKPUkmTu3et1t23bdNR5+eOvJQT2Z4qJJDyVL\nMbUpfBR4LnADcC+wCHgHcJaZvcjd+/rS1aj3yLJlsH597fd27NhVpdBs7xP1ZIqLBilKlmJKCmPu\nXnVpMrNvAN8F3g58oJCoekDlRWHLllBSSGulSkHVEXFRd1PJUjTVR+mEkCz7HrAVWNz9iHpH6aKw\ndi3cd1+oUpo5s3qdVqoUVB0hMriiSQq1mNl+wCzg4aJjyUIWdfmlBPHAA+2PYNXoV5HBFVP1US1j\nwF7AF4sOJAtZ1uV3UqWg6oj8aWyB9KpCkoKZGTCjmXXdfUedfSwntCNc5+7fzTC8wqguvz/VSgBq\nzJdeVVRJYTlwaxPruZktdvd7Khea2QuAlcAa4M3NHnRsbIyhoaGqZaOjo4yOjja7i1xpVHJ/qpUA\nHnywep0NG7ofl8RhfHyc8fHxqmVTU1O5Hc/cPbed1z2o2UHAmU2ufqO7l/vTmNlhwCpgB7DM3Tc1\ncbxhYGJiYoLh4eF2Qu6KzZvh7LNhzZrw+oQT4OabVa0Qu6OPrk72ixbt3kts9mx47LHuxyZxmpyc\nZGRkBGDE3Sez3HchJYXkQn5tq9uZ2Vzgm4R2hNObSQgxmT8/PEd5R1Jhtnq1qhX6Qa0S4DPPVCeF\n9ABEkaJE0/vIzPYFvg4sAF7h7uum2SRKWbUraFRy76jVm2vhwup10q9FihJT76MvACcC1wDHmdlx\nFe/9xt2/WkxY2cqqXUENmb2jVm8ujUKWXhVTUjgBcOANyVel9UBfJIWsLhbqydTb1O1XelU0ScHd\njyo6hm7I6mKhnkwi0o5o2hSkNRqVnC+12Ui/iqakIK1R9US+1GYj/UolhR6iu894qM1G+pWSQg/R\nYxXjoZlkpV+p+qiH6O4zHupSKv1KSaGHqMdQPNRmI/1KSaGH6O5TRIqmpNBDdPcpIkVTQ3PG1INI\nRGKmkkLG0v3XjzkmPDdZ01+LSAxUUshYusfQtm3qWioi8VBSyFitHkPqWioisVBSyNjKleEpWpXU\ntVREYqGkkLH580MbQieT0amxWkSKoobmHHTatVSTrYlIUaIpKZjZaWb2VTN7wMy2m9lGM/u6mb2k\n6NiypukuRKQo0SQF4PnAM8CngEuAjwEHAbeZ2f8oMrCsabI1ESlKNNVH7n4N4fnMZWb2KWAd8KfA\nN4uIKw+a7kJEihJNUqjF3beb2RZgTtGxZEnTXYhIUaJLCmY2G5gBzAMuBI4DLi80KBGRPhFdUgCu\nB85Mfn4KuAr4cHHhiIj0j0Iami2Y2cxXjc0vBX4XeANwO6HUsFc3469FYwtEpB+Yu3f/oGa/A9za\nxKoOLHb3e+rsZy9gErjb3V/X4HjDwMTy5csZGhqqem90dJTR0dGmY69n2bJdYwsgDFpTu4CIdGp8\nfJzx8fGqZVNTU9x2220AI+4+meXxikoKB7GrCmg6N7r7tgb7upxQetjP3XfUWWcYmJiYmGB4eLjl\neJtx9NHVT01btAjWrs3lUCIy4CYnJxkZGYEckkIhbQruvgm4NqPd7QsYMBuomRS6QY/SFJF+EE1D\ns5kd6O5bUsvmACuAB9z94WIiCzS2QET6QTRJAfi6mW0AfghsBo4ALgIWAHXbE7pFYwtEpB/ElBSu\nAf6AMHp5DvAooffRx9z9B0UGJiLSL6JJCu7+KcK8RyIikpOYJsTLhcYXiIjsMvBJofTsgnXrwnc9\nT1lEBtnAJ4UNGxq/FhEZJAOfFLZubfxaRGSQDHxSmDu38WsRkUEy8Elh4cLGr0VEBkk0XVLzopHI\nIiK7DHxS0EhkEZFdBr76SEREdlFSEBGRMiUFEREpU1IQEZEyJQURESlTUhARkTIlBRERKYs6KZjZ\nZ8zsWTO7qehYRET6QbRJwcxGgAuA7UXHIiLSL6JNCsDHgX8hPK85GuPj40WHAPROHNA7sSiO3fVK\nLIqje6JMCmZ2AXAc8BdFx9KqXvmj6pU4oHdiURy765VYFEf3RJcUzGwW8BHgcnePqpQgItLroksK\nwAcJ7QhXFh2IiEi/KWyWVDMzYEYz67r7jmSb5wP/C3i9u+/MMTwRkYFU5NTZy4Fbm1jPzWyxu99D\nKB2scvevtHisvQHuvvvuFjfL3tTUFJOTk0WH0TNxQO/Eojh21yuxKI5qFdeyvbPet7l71vts7sBm\nBwFnNrn6jcCJwLeA1wA/Lu0G+B7wM+BNwFZ331bjWH8IfL7TmEVEesx57v6FLHdYWFJolZldCPwT\nIRFU8mSZA2Pu/vEa2x5ASED3A0/mG6mISO72Bo4EvuHuj2S545iSwkJguMZbVxMu9h8Gfuruv+hm\nXCIi/SSapFCPmf0C+Im7v6roWEREYhdjl9Q0T75ERKRD0ZcUREQkO/1QUhARkYz0XVIwszPM7Boz\n+7mZPW5ma83sajM7uMntP5hMx53+eqLbsST7OMTMrjezR81sysy+YmZHtRjHwWb2N2b2n2b2WPL7\nLG9h+0zOSadxJPvo+HxU7GvIzD5tZpvN7DdJXC9uctvP1jkn/11n/Rlm9lEz22BmT5jZHWb28iaP\nldnv3EksWf5vJPvbz8z+0sy+bmaPJPu6oIXt2/78sorDzC6sc06eMbP5LcaxxMw+aWY/TX6f9WZ2\nnZk9r8ntMzkfRQ5ey8tHgecCNwD3AouAdwBnmdmLmpwvyYG3Ao9XLHum27GY2X7Ad4DZhN5VTwPv\nBL6TbP9ok3EcC7w7iWENcErrv0om56SjODI8H6UR9V8DXgj8LfAIcEmyr2F3X9vEbp4E3kh1N+mp\nOuteSxhjcwVwH3AR8DUzO93df9Agzsx+505jSWT1vwEwD3g/sJ4w9uj0ZjfM6PPrOI6EJ9vfn1r+\n6xb3cynwEsL1Yg1wMOF6MWlmJ7l7zRsOyPh8uHtffQHLaiw7DXgW+FAT23+Q8Ec+twdieU8Sy3DF\nsmOBncCHW4hjP2BO8vOKZJ/LW9g+k3OSQRyZnI9ku9cln8NrKpbNA7YCn2ti+88CjzV5rKXJscYq\nls0kJMfvd+t3ziCWzP43kv3tBcxPfh5J4rqgG59fhnFcmP58OjgfJwPPSS07hnDzcW23zkffVR+5\n+/drLPse4eQsbmFXe5jZ7IJjWQHc6e7lcfXu/nPg24Q/gmbjeNzdW71rqaWjc5JBHJmcj4p9/crd\nb6zY18PA9cCrzWyvZnZiwaxpVnst4Q7/6opj7QCuAU4xs0OniTOr37nTWEo6/t9IjrvT25/pOJPP\nL4M4ysxslpm1fU119zvc/enUsvuAnzL99SKz89F3SaGWpAg+C3i42U2AdcCUmW0zs39ttX6w01iS\n4uDxwF013l4NHJ3sq1tyOydNHTz78/FioNYkNquBfYHnN7GPfYFtwGNJXfQn68TwIuAed/9NjWOV\n3t9NTn8DbcVSGRYF/h1UyOLzy4oRqvgeA54ws6+a2TEZ7v8gpr92ZXY++rFNoZYxQhHxi02s+yjw\nCeB2YAehuuftwIlmtqTGP1NescwlFOs31nivtOwQQrE/b3mfk2ZkfT4WAN+dZl//r8H2vyTU3U4S\nbq5+j1CHe3xSN/9s6lj14rbkWLXk8TfQbizQG38HJZ1+fll5glCVeCshKYwAfwasSuryH+pk52Z2\nPnAocNk0q2Z2Pno6KSR3Si1Nr11jH8uBDwDXuXutk5beT3rupBvN7E7ChHrvMLN/6FIs+yTfq/aV\nnJNSEXN/M5vZahyt6vScZBRHzfORKM1n1fB8pGLZp8G+rOJ49faTfurf9WZ2L6Ex+LWEYntJo2OV\n3q+lmd+5YZx19tnW/qb5O7iEkCS7paPPLyvufgOhYbjkJjP7JnAb4cmQl7S7bzN7AfBJYBWhc0Aj\nmZ2PXq8+Wk54oM50X09YeNZCleSkriS05L+53SDcfZzQDvDXXYxle/I9fZFbDlxF+OzubDWODP2S\n8Mc23TnJKo565wNCj4vpzkc6lu119rU3oTfJ9hrvTeeKZNt0985Gxyq9X0uj33m6betpN5aakv+N\nX7H775y3PD6/TLj7KuCHdHBOkiq5Wwils9/3pOW4gczOR0+XFAhTYl/U5LpVRWIzOwz4JuGknuXu\nj9fcqnnrCfXHH+xSLFsJmX9BavnPgJuAVwEXU/vuoGYcGfsZoW55D6Y/J1nEUe98wK6pTqY7H5Wx\nbKyzr9KyX7YaoLs/aWaPEKp90sesVS0z3bEa/c6lZa2e23ZjaeRBdv+d85b555exB2mzXcPM9ge+\nAexP6MH4qyY2y+x89HRScPdNTF9s2o2ZzSVchPcCTk/206mFwKS7txRPu7G4u5vZT4AlqeWbzGwf\nYK27X1176/wlccyijXPS5vFqno/Eb9P6+fgxsKzG8pMJ9cT3tBpjcj7mAVtqHOt0M5uVqnc/mZDM\nfkwN0/zOJwHr2qjHbyuWaRxJ7UbOPGX++WVsEbv/HUwrqf68mdAV9WVJT7NmZHY+er36qGVmti/w\ndUKGfIW7r2uw7mFmdmxq2bwa610CHJjst2uxAF8iNOINV6x3LHAG1XXWmcn7nHQSB9mejy8BB5nZ\nuRX7mkdoD7jJKx73amaLzGxRxeuZdbqhfiD5nj4nXyLcgL2lYh8zCKXgO0qNkV36G2g7liL+DpJj\nHGxmx5rZnhWLm/788oyjzjl5JaHBudXrxR6Ez/Rk4LXuvrrOermej76bEM/MvkKoWrmG0E2s0m/c\n/asV6355+47ZAAADpElEQVSHMHhqj4pljwPXAT8hNNKcBrwe+BGhKNf0Q3oyiGVWctzZwN8RGpjH\nCHX5L/YWHq5hZpcR7gSPA/6A8MCiXwC4++XTxJHlOekkjizPxx7A95M4/o7Q5e8S4HBgibvfW7Hu\n/cCz7r4oeX1EEsc4oRoNQu+jVwBfc/ezaxzvOuB/Eh4pWxpFvAQ4I6mDzv13ziCWzP4OKvb5NmAO\noYfNWwntbj9K3v64u28zs38GLgCOdPcHku2a/vxyjuOeZL27CKPZR4A/Bh4Clrp706UFM7uS8Az6\nm6huvAbA3T+frFcrjuzORysj3WL4IlxgnqnztS617q3A06llVxH+6H9N+MP/OXA5sF+3Y0mWH0L4\nR3yU8Ef3FWBRG7E8WyeO9O+f9zlpO44sz0eyryHg08BmQnvRtwkX2lqf49rUdv+SnIdthOL5GsLo\n4z3rHGsGYdqTh5L17wBe3u3fuZNYsvw7aPJ/5PBknc8SkuHh7Xx+ecYBfAiYILT/PJns5xPAgW3E\ncGuDGJ6pWC/X89F3JQUREWlf37UpiIhI+5QURESkTElBRETKlBRERKRMSUFERMqUFEREpExJQURE\nypQURESkTElBRETKlBRERKRMSUFERMqUFEREpExJQUREypQURESkTElBRETKlBRERKRMSUGkTWb2\nJ2b2gJk9amafMLM5FlxmZu8uOj6RdujJayJtMLOPAO8ENgLbgT0Jj018LjDp7q8sMDyRtj2n6ABE\nInU3MMfdt5cWmNl5wBgwWlhUIh1S9ZFIG9z92lRCuBB4I/Ayd5+qXNfMRszsh2Z2aLfjFGmVSgoi\nHTKztwNLgDPdfWfF8hXA2YAn7+9ZTIQizVObgkgHzOwyYIe7f6zBOr8D/CdwlLs/0LXgRNqg6iOR\nNpnZXwE/SicEMzuroJBEOqakINIGM/sAcIu735JafhjwpmKiEumckoJIi8zsUkKj8l+a2VvMbL9k\n+YuBbwP/p8j4RDqhhmaRFpjZacALgecBC4FVwJVmtgU4DPi0u/9HgSGKdEQlBZHWvAt4k7s/5e7r\ngLOAhwk9i97n7m8tNDqRDqmkINICd3916vUkcHhB4YhkTiUFEREpU1IQyd+egKH/N4mA/khFcmJm\nv2tmNwCfJYxq/nczGzezFxUcmkhdGtEsIiJlKimIiEiZkoKIiJQpKYiISJmSgoiIlCkpiIhImZKC\niIiUKSmIiEiZkoKIiJQpKYiISJmSgoiIlCkpiIhI2f8HOS1wl9Pev4cAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig = plt.figure(figsize=(4,3))\n", + "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n", + "plt.xlabel(\"$z_1$\", fontsize=18)\n", + "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n", + "save_fig(\"linear_autoencoder_pca_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Stacked Autoencoders" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's use MNIST:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train all layers at once" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders). We will use ELU activation, He initialization and L2 regularization." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import fully_connected\n", + "\n", + "n_inputs = 28*28\n", + "n_hidden1 = 300\n", + "n_hidden2 = 150 # codings\n", + "n_hidden3 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "l2_reg = 0.0001\n", + "\n", + "initializer = tf.contrib.layers.variance_scaling_initializer() # He initialization\n", + "#Equivalent to:\n", + "#initializer = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "with tf.contrib.framework.arg_scope(\n", + " [fully_connected],\n", + " activation_fn=tf.nn.elu,\n", + " weights_initializer=initializer,\n", + " weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n", + " hidden1 = fully_connected(X, n_hidden1)\n", + " hidden2 = fully_connected(hidden1, n_hidden2)\n", + " hidden3 = fully_connected(hidden2, n_hidden3)\n", + " outputs = fully_connected(hidden3, n_outputs, activation_fn=None)\n", + "\n", + "mse = tf.reduce_mean(tf.square(outputs - X))\n", + "\n", + "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n", + "loss = mse + reg_losses\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's train it! Note that we don't feed target values (`y_batch` is not used). This is unsupervised training." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train MSE: 0.0405902\n", + "1 Train MSE: 0.0167411\n", + "2 Train MSE: 0.0117727\n", + "3 Train MSE: 0.00848352\n" + ] + } + ], + "source": [ + "n_epochs = 4\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = mnist.train.num_examples // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch})\n", + " mse_train = mse.eval(feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " saver.save(sess, \"my_model_all_layers.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function loads the model, evaluates it on the test set (it measures the reconstruction error), then it displays the original image and its reconstruction:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n", + " with tf.Session() as sess:\n", + " if model_path:\n", + " saver.restore(sess, model_path)\n", + " X_test = mnist.test.images[:n_test_digits]\n", + " outputs_val = outputs.eval(feed_dict={X: X_test})\n", + "\n", + " fig = plt.figure(figsize=(8, 3 * n_test_digits))\n", + " for digit_index in range(n_test_digits):\n", + " plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n", + " plot_image(X_test[digit_index])\n", + " plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n", + " plot_image(outputs_val[digit_index])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure reconstruction_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAApkAAAJNCAYAAAB6AG5rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3VmMngXZPvC3nX2mnU4LbWlLoSAChbDZYquAshjcIsSF\nA+FAAyYSE01sogYSE/XEQ+SEoMGAogcYAxqNG2iqbAWsUEAKllVsbQq0M9PZl8538P+ffObjvW8z\nN9Np+/udvlee53nXXn0OrlkwMzPTAACASgsP9wUAAHD0UTIBACinZAIAUE7JBACgnJIJAEA5JRMA\ngHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQLnWw30Bb2PmcF8A8I5acLgvgHfW\nwMBA+Ds+MxP/1C9YEH9UDh06FGZaW+N/7qanp0vOFWlpaQkzmdcmI3OchQvj+03j4+NhpqOjI8xk\nXuPM9VTJvJ8V15P5HGdem8xnJ3OczPVkMn19fU1D7mQCAFBOyQQAoJySCQBAOSUTAIBySiYAAOWU\nTAAAyimZAACUm687mQAcwSr2JLMyO4aTk5NhJrMLGJ0rs0uZeW2qtj+rriezgZmRuZ6pqakw09bW\nVnKuudrkrNrjnMvnVPEddicTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACgnJIJAEA5\nY+wAlMsMm2fGnqvGp1taWkrOVTFQnXltMkPr09PTYaZqmLvqfcg8r4y5fF7R65z5bFX9cYLMZyfz\nucgcJ/O8Iu5kAgBQTskEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAoZ4wdgHKZ8enM\nIPRcDaQ3GrmB74oR8MwxpqamwkzVa1N1ro6OjpJzZQbbqz5fFcfJHGNycjLMtLe3h5mqofWKEfoM\ndzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJQzxg5AuczgeNVodNUoeSYT\nPa/Mc8qoGiTPjJ+3tLSEmcyY+NjYWMm5MkZHR8NM5r2oGNfPfP6qPhcV19to1H2vIu5kAgBQTskE\nAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAoZ4wdgMOiYuy50agb3c5kMgPoFaoG5ru7\nu8NMZvh9aGgozIyPj4eZ4eHhMNPZ2RlmMuPwmc9FT09PmIlen8y1ZGSuNzNmnxngz6gYkHcnEwCA\nckomAADllEwAAMopmQAAlFMyAQAop2QCAFBOyQQAoJySCQBAOWPsAJTLjIlnxp6rxs+np6fDTGZU\nOzpOZgh7dHQ0zExMTISZ3t7eMLN69eowk3mvFi9eXHI9mcH7zOuT+Vxkhsvb2trCTDTGnhmqzwze\nZ16bqhH6zGtT8ccS3MkEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFDOGPsc\n2bZtW5i59dZbw8yaNWvCTFdXV5j53Oc+F2aWLVtWkgGOPZlh6arB9sxxxsfHw8zg4GCYefXVV5s+\n/tJLL4XHeO2118JMT09PmNmwYUOY2bNnT5jJDIVnMsuXLw8zfX19YWZsbCzMZJ5XZrg8MyAfPff2\n9vbwGJl/KzOf40wmM1Sf+V5l/rBAxJ1MAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCA\ncgsym0uHwby8qNk444wzwsyuXbvm4ErylixZEmY2b948B1dyZFq3bl2Yuemmm8LMSSedVHA18048\n0sYRrb+/P/wdz2z1ZbYOM5ucmQ3MvXv3hplo8/i5554Lj/HMM8+EmdWrV4eZzG5y5vXL7HZmrFq1\nKsxk9jb7+/vDTGZL87TTTgszGVFP+tCHPhQe47zzzgszmW3UzAZmW1tbmMl8LlpaWsLMkiVLmn6J\n3ckEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFAuXkWlxC9+8Ysw89RTT4WZ\ns88+O8z8/e9/DzOPPfZYmPnlL38ZZn7/+9+HmVNOOaXp46+88kp4jCqZIeDMoPDrr79ecTmpwfZv\nfOMbJeeCI1HmO5sZlq4614knntj08QMHDoTHWLt2bZjp7OwMM5lB8ldffTXMdHV1hZn29vYwMzAw\nEGaefvrpMJN5Pzds2BBmMgP8mWH83t7epo+fe+654TEyw+YZmTH2qj9gMD09nbqmpueZ9REAAOA/\nKJkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCAckomAADlFszMzBzua/i/zMuLOtZUDf1GY+wv\nv/xy9pJmLTMonBljj55To9FovPHGG2HmvvvuCzNXX311mDkCLTjcF8A7q7+/v+R3PPNvVCYzPj4e\nZiYmJsJMNFA9NTUVHmPBgvjjnxndzgy/Zwa1M0PhQ0NDYSZj3759Yaavry/MnHTSSWFmx44dYeaW\nW24JM0uXLm36+JYtW8JjXHzxxWEmo2IgPSsz2N7b29v0w+xOJgAA5ZRMAADKKZkAAJRTMgEAKKdk\nAgBQTskEAKCckgkAQDklEwCAcq2H+wKYvzo7O8PMmWeeOevzrF+/ftbHqPTYY4+FmTfffDPMbNq0\nKcxceeWVqWuCo1FmlDwzFJ4ZQM/8IYbu7u4w09bWNuvzTE5OhpnMePyKFSvCTEZmdDszDp95P5MD\n32FmdHQ0zDzyyCNhJnPNxx9/fNPHV69eHR4j8znOvMYZmXNlRt0z36uIO5kAAJRTMgEAKKdkAgBQ\nTskEAKCckgkAQDklEwCAckomAADllEwAAMoZY+eYMjw8HGY++clPhpnMaO73vve9MNPV1RVm4EiU\nGbnOZDKD0JnjdHR0hJnMiHVmTLxC5jllRt0zmWhgPpvJXHNmrD7z+7p79+4ws2PHjjCT+VxcccUV\nTR/PjLFnXr+xsbEwk3mNM0PrVd/PiDuZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA\n5ZRMAADKGWPnmHLXXXeFmb1794aZ4447LsycfPLJmUuCo1JmUHtmZibMZMbPq86VEY2bT0xMhMfI\njG5Xmcth7qr3Yc+ePWHmjjvuCDN/+ctfwszGjRvDzEUXXdT08WXLloXHyDzvuRpIzzLGDgDAvKRk\nAgBQTskEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlDPGzlHjpZdeCjNbtmwpOdejjz4aZk44\n4YSSc8HRKjO0XnWczBj26OjorK8lc57MGHtnZ2eYyTzvlpaWMNPd3R1mMjLj3ZnX57XXXgszzz//\nfJjJ/AZfeumlYWbFihVNH888p2jEv9HIjdmPj4+HmY6OjpJzVXAnEwCAckomAADllEwAAMopmQAA\nlFMyAQAop2QCAFBOyQQAoJySCQBAOWPsHDV+9atfhZnMIO4111wTZk499dTUNcGxKjP2nBnvzsiM\nkleNT4+MjMz6GJmh9cyg9tTUVJhpa2sLM5nXJjPqXvV+btu2Lcxk3oeNGzeGmfe+971hJnq/MgPp\n09PTJZmqz7oxdgAAjlhKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDlj7BwRMiPq\n9913X5jJDBx/97vfDTOZYWI4llUNc1cdJzN0nfmdaW9vb/p4a2v8z2rmdygzup35HRodHQ0zMzMz\nJefKPPddu3aFmWeeeSbMZN6rCy+8MMysWbMmzETvRWaMPXO9czmcn5H5DIbHKLgOAAD4X5RMAADK\nKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCAcsbYOSL88Ic/DDMPPvhgmLn22mvDzKmnnpq6\nJmB2MmPPmaHwzEB1Zgy7YnA8Mx6fed6Z51T12kxNTYWZzID8xMREmPnDH/4QZrZv3x5mLrnkkjCz\ncePGMNPd3R1mDh482PTx4eHh8BiZ169K1WB75vMVcScTAIBySiYAAOWUTAAAyimZAACUUzIBACin\nZAIAUE7JBACgnJIJAEA5Y+wcdk899VSY+fKXvxxm+vr6wsx3vvOd1DUBs1M1Jl41XN7e3h5mMqKx\n9cz1Zsa7o9H37LkyI+BVQ+uZEfU//elPYaanpyfMfOxjHwszJ5xwQpiZnp4OM9Fzb2trC4/R1dVV\nci1VI+qZPxqQ+V5F3MkEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFDOGDvv\nuNHR0aaPf/aznw2PkRmpve6668LMqaeeGmaA2csMhVcNrWfGpzPHyYxqR2PYmd+qycnJMFOls7Mz\nzGTeh507d4aZ22+/Pcw8++yzYeaGG24IM5s3bw4zmcHx8fHxMBN9vhYtWhQeIzN4X3Etjcbc/iGE\n8BizPgIAAPwHJRMAgHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCcMXZmJTP6+vGPf7zp\n4y+88EJ4jPXr14eZb3/722EGmBvRaHmjkfv9yMiMbmdMTU2FmWjEOvOcWlvjf3ozo+7d3d1hJjMw\n/9Zbb4WZn/3sZ2Hmr3/9a5j54Ac/GGY+8YlPhJnMAHpm3DzzfrW3tzd9PPP5y3y2MjJD/pn3PPO8\nK75X7mQCAFBOyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBAChnjJ1Z2b9/f5jZunXr\nrM9z9913h5lly5bN+jxAjWi0vNFoNBYujO9zZEbJM8PvmTHszHGige/M9Ubj3o1Go7F48eIwkzE6\nOhpmdu7cGWaeeOKJMJP5Db722mvDzBlnnBFmMq9zZow9M1ze2dnZ9PGqPyqQ+YxmvjMZmaH1inO5\nkwkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFDOTiZva2BgIMxs3rx51uf5yU9+EmYu\nuOCCWZ8HmDuZncyMql3AzAZm5pqj42Q2MDPXktkxzOwq/uMf/wgz99xzT5jZu3dvmLn++uvDTObf\njGiXstFoNPr7+8NM1TZq5jgVx8i851WfnYyK5+1OJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskE\nAKCckgkAQDklEwCAcsbYeVt33nlnmHn55ZdnfZ6LL744zFSNywJzI/OdPXToUJjJjLFnMtPT0yXH\naWtra/p4Ziw7c57M63fw4MEw87e//S3MPP7442FmxYoVYeb8888PM62tce0YGxsLM8PDw2Fm8eLF\nYabis5N5TnOp6rNe8YcQ3MkEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFBu\nfi2IMmd27doVZr71rW+98xcCHLMyw+UZmVH3jIqR9JmZmZLzTExMhJkXXnghzNx///1h5qmnngoz\nGzduDDPd3d1hZnR0NMy0t7eXZKpG7yOZ9zwj8znOZDLPKXPNFd9PdzIBACinZAIAUE7JBACgnJIJ\nAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJQzxn6MevDBB8PM4OBgybnWr1/f9PGurq6S8wDzR2YQenp6\nOsxUDbZnhrkzQ9eZ40SmpqbCzMDAQJjJjKiPjIyEmU2bNoWZD3/4w2HmxBNPDDMdHR1hJiNznIqh\n9UYjfs/nciA9o7U1rnbj4+NhpuL1cycTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACg\nnJIJAEA5Y+zMyvvf//4wc//99zd93Bg7HH2qhqWrVA1zRzLD3Jkx9kzm3HPPDTNV4+fnn39+mOnp\n6QkzExMTYSbzXrW1tYWZzGs4VzLfh0wm88cAJicnw0xmsL2CO5kAAJRTMgEAKKdkAgBQTskEAKCc\nkgkAQDklEwCAckomAADllEwAAMotmG+Duf/fvLwooMzcLGNz2Bw4cCD8Hc+MbmfGzTP/jrW0tISZ\nzNB1dK7M9WbOMz09PetryV5PZiA980czMufKvA9Vw/mZc2Ve5yiTud6Kz1ZW5noy58ocp6+vr2nI\nnUwAAMopmQAAlFMyAQAop2QCAFBOyQQAoJySCQBAOSUTAIBySiYAAOXm6xg7AABHMHcyAQAop2QC\nAFBOyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMA\ngHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCAckomAADllEwAAMopmQAA\nlFMyAQAop2QCAFBOyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACg\nnJIJAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKBc6+G+gLcxc7gvAHhHLTjc\nF8A7a8+ePeHveGtr/E/Q9PR0yfVkzjU5ORlmZmaaP63M9ba1tYWZlpaWMDM+Ph5mFiyIv2qHDh2a\ns+N0dnaGmarXcGxsLMxkXudI5rM1NTUVZjKvcdX7UGX16tVNL8idTAAAyimZAACUUzIBACinZAIA\nUE7JBACgnJIJAEA5JRMAgHLzdScTgKNcZjtw4cKaeyGZDcyMaKewvb09PEa0tdlo5F6bio3HRiP3\n2nR3d4eZzD5j1YbjXG1gNhrxe555Tpntz6ot16rnnXmNI+5kAgBQTskEAKCckgkAQDklEwCAckom\nAADllEwAAMopmQAAlFMyAQAoZ4wdgHKZEfXMiHU0hJ3NZAbQM4PZFePwVc87M9heNXifGQHPjIlX\nDZdnrifznleM52fGz6ve88y5qt7PCu5kAgBQTskEAKCckgkAQDklEwCAckomAADllEwAAMopmQAA\nlFMyAQAoZ4z9GPXTn/40zAwPD4eZ7du3h5kf/OAHqWtq5pvf/GaYufzyy8PMpZdeOutrAWJtbW1h\nJjMIPTExUXE5qYHqikxm/DzjjTfeCDMHDhwIM3v37g0zmaHwzPX09fWFmYzjjjsuzKxduzbMLFq0\nKMxkBtA7OjqaPp55z6uG/jPXW6Wzs3PWx3AnEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFBO\nyQQAoJySCQBAuQUzMzOH+xr+L/Pyoo4UX/rSl8LM97///Tm4krl11llnhZmHHnoozCxZsqTicmhu\n7haFOSx2794d/o63tLSEx8mMT2dG3TND1xmjo6NNHx8aGgqPMTg4GGaefPLJMPPII4+EmccffzzM\n7N69O8xEg+SNRu79XLp0aZj56Ec/GmY+85nPhJl3v/vdYaa3tzfMRJ+vzOcvM3ifeY0z58oMvx88\neDDMZP4tXLVqVdMvqDuZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5VoP9wXw35lv\nG5gXXHBBmPn0pz/d9PFdu3aFx/jRj34UZp577rkw8/Of/zzM3HDDDWEGaC6zb5mR2XLObGBmMpl9\nwfHx8aaPZ/YHM/uWW7duDTPbtm0LM8uXLw8z11xzTZhZt25dmDlw4ECYefbZZ8PMv/71rzCT+b0/\n7bTTwszU1FSYibYpM9uoVRuYmc9x5jvT3d0dZjLbnhF3MgEAKKdkAgBQTskEAKCckgkAQDklEwCA\nckomAADllEwAAMopmQAAlDPGPo/885//DDN33HFHybkuvPDCMPO73/0uzGQGXdvb25s+nhlAfvHF\nF8PMww8/HGbefPPNMAPMjcxg+8TERJhpa2uruJzU+HQ0dJ0Z5s78DvX394eZD3zgA2HmqquuKjlO\nZ2dnmMn8Bu/ZsyfMZMbYM6/hW2+9FWYyn6/oczE6OhoeIzOi3tLSEmYyg+2Z71Umk7meiDuZAACU\nUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKGWOfRzLjstEQcKORG1p/4IEHwsyi\nRYvCTIW77rorzDzxxBMl57r66qtLjgM0l/mtymQyg9CZTGYMOyMasY7++ESj0Wj09PSEmYsvvjjM\nZEbUzznnnDCzdOnSMJMZHN+7d2+Yee2118JM5v1cs2ZNmFm1alWYyXwuRkZGmj6eGarPZDLD8BmZ\n1y/z3csMtkfcyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUM4Y+zzynve8\nJ8xkBtszY8BdXV2pa5oLd9xxR5ipGqkF5kZmyPnQoUNzdq7MGHZmxDr67cwMra9fvz7MHH/88WFm\n3bp1YSZzPQcOHAgz+/btCzOvvvpqmBkaGgozmden6rlnxtgPHjzY9PHu7u7wGBmZz3HmM5o5TiaT\nGWyPuJMJAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCcMfYjzJIlSw73JfzX\n7r777qaP79ixo+Q8V155ZZh517veVXIuoLnW1vifl6mpqTCTGcuenp4uyWTOFT2vVatWhcdYuXJl\nmMmMbmcymdc4M8y9c+fOMPP888+HmbfeeivM9Pb2hpmqkfmxsbEwEw35Z46Rea8y4+eZP2CQ+Rxn\njmOMHQCAeUnJBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJRTMgEAKGeMnVl58sknw8wXv/jF\npo+Pj4+Hx8gMHN96661hpq2tLcwAs5cZAc8MVFcNS1cNv3d3dzd9vL29PTxG5jdveHi4JJN5jfft\n2xdm7r333jCzbdu2MHPeeeeFmauuuirMZP5NGB0dDTMDAwOzPlfm35WJiYkwkxnFz5yrarC9gjuZ\nAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKGWNnVh599NEwkxkejtx4441h\n5vTTT5/1eYAambHnTGZ6ejrMtLbG/5TNzMyEmcnJyTATDV1nrndwcDDMZEbC+/r6wszBgwfDzK9/\n/esw8/DDD4eZFStWhJnLLrsszGzYsCHMZEbvM6/hypUrw0z02cl8tkZGRsJMZ2dnmMmM62dkBtsz\n4/ARdzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKKZkAAJQzxs7buv7668PMPffc\nM+vzfPWrXw0zX//612d9HmDuZMaeqwahM3/wITPYXnHNExMT4TGGhobCTE9PT5jJjMdv3bo1zDzw\nwANhZvny5WHmyiuvDDOXX355ybkyr2Hms9PV1RVmIv39/WEm84cHMgPzbW1tYWZsbKzkeiq4kwkA\nQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFBOyQQAoJwx9mNUZsj2t7/9bZjJjL6uXLmy\n6eM333xzeIzMSC0wf2TGnmdmZsJMZlA7k8mca2RkJMxEzyszxp4ZWs+Mbu/atSvM3HvvvWHm3//+\nd5i57LLLSjLr168PM4sXLw4zmSH66N+eRqPR6O7uDjP79+8PM5GOjo5ZH6PRyH2Oq4bWM9+riDuZ\nAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKGWM/Rl1zzTVhZt++fSXn+spX\nvtL08WXLlpWcB5g/Dh06FGYyY8+Z42SMjo6WHGdqaqrp45lx78zzzgzD//nPfw4zDz/8cJg5/fTT\nw8ymTZvCzFlnnRVmMkP0w8PDYSZ6HxqNRmPRokVhZnx8PMwMDg42fbyzszM8RmtrXLcyn4vMCP30\n9HSYyYzDZ16biDuZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJKJgAA5exkHoW2b98eZrZu\n3Vpyrk996lNhZsuWLSXnAo4cCxfG9zAyG5hVx8lsELa0tISZtra2po9ndjIz53n++efDzI4dO8LM\nmjVrwswll1wSZjZs2BBmurq6wszQ0FCYeeWVV8LM2rVrw0xmS7O/vz/MRNfc19cXHqO9vT3MZDYw\nZ2ZmSjKZ1yaz7RlxJxMAgHJKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDlj7EeY\n0dHRMHPTTTeFmYmJiYrLSQ30ZkZogaNL1Wh0Zmi9atS9o6Nj1pnMtezcuTPM3HvvvWHmiSeeCDOn\nnHJKmLniiivCTGbUvaenJ8wMDw+HmWjwvtFoNKanp0vOlRmHX7FiRdPHMwP8mT8GkBljzzynxYsX\nh5nMdy/znQmPMesjAADAf1AyAQAop2QCAFBOyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyhljP8Lc\nfvvtYeaPf/xjybmuv/76MLNly5aScwFHl8zYc0tLS8lxWlvjf8oyI+mZMfHoegYGBsJjPPLII2Hm\nN7/5TZjJ/KGLTZs2hZkzzzwzzGTeh8wfC8m8VyeffHKYyTz3zPUcd9xxYWbJkiVNH88Mwx88eDDM\n7N+/P8xknvfIyEiY6e3tDTOZ70zEnUwAAMopmQAAlFMyAQAop2QCAFBOyQQAoJySCQBAOSUTAIBy\nSiYAAOWMsR9hbr755jk71y233BJmMsOwwLEnM96dyVSdq6OjI8x0dnaGmampqaaP79u3LzzGiy++\nOOvzNBqNxubNm8PMFVdcEWbWrFkTZoaHh8NMZlw/829GZrA9836OjY2VnCsadR8aGgqPkflcDA4O\nhpmlS5eGmRUrVoSZBQsWhBlj7AAAzEtKJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkA\nQDlj7LytzMDswoXz5/8pmXHezFjw9PR0mBkfH09dUyQa+W00Go1bb7215FyRzGuT+WMAbW1tFZfD\nES4zkJ4Ze86MRmd+hzKfy8z1TE5ONn18ZGQkPEbm9yMzEh5dS/Z6Xn755TCTGTbP/J5lrjkzOJ4Z\nzn/jjTfCzKJFi8LMK6+80vTx7du3h8fIfP5WrlwZZs4+++wwkxm8z1xPxR9LmD8NAQCAo4aSCQBA\nOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUM4YO29rzZo1h/sS/is33nhjmFm9enWY2bt3\nb5i57bbbUtd0tMl8Jr7whS/MwZUw32VG1DOZjKrh90xmYmKi6eOZ5xQdo9FoNAYGBsLM008/HWYG\nBwfDzLJly0qOkxn4XrduXZjZv39/mMn80YzM8HvmOC+++GLTxw8cOBAe45xzzgkzH/nIR8JMb29v\nmMkM1WdblacXAAAFA0lEQVS+M8bYAQCYl5RMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDkl\nEwCAcsbYjzDXXXddmLnzzjvn4Ermn9tvv/1wX8L/0toaf71aWlpKzvX5z3++6ePve9/7Ss5z0UUX\nlRyHo9/ChTX3MDLj5u3t7WEmMyw9Pj4+6+tZuXJleIzzzjsvzLz00kthZtu2bWFmz549YWbx4sVh\nJjO03tHREWYyzyszxp75fe3p6QkzmTH26Hf6xBNPDI9xwQUXhJklS5aEmczz7urqCjOZPzyQec8j\n7mQCAFBOyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACi3IDNQexjMy4s6Uvz4xz8O\nMxMTE3NwJf/Pjh07mj5+2223zdGVNBpf+9rXwsxpp51Wcq6rrroqzKxYsaLkXEegeGGbI9rrr78e\n/o5nxrunpqbCTGY0OvOHDzLD79EY9uTkZHiMzED6Qw89FGZ2794dZg4cOBBmMuPdIyMjYSbz3Pv7\n+0sy69atCzO9vb1hZtGiRWGms7Nz1uc56aSTwszy5cvDTF9fX5jJvJ+ZP2CQeT/Xrl3b9EvjTiYA\nAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHLG2IHDwRj7UW7fvn3h73hm7Dkz\nop7JTE9Ph5nMOHw0/J45z/DwcJjJHCczAp4Zqh8cHAwzAwMDYSbzvDKZxYsXh5loFD97nEOHDoWZ\naKQ/816NjY2FmcznLxqGbzRyQ+uZ7pd5bVatWmWMHQCAuaVkAgBQTskEAKCckgkAQDklEwCAckom\nAADllEwAAMrFQ1MA8F+amJgIM5l9y8zOY2ansGoXMNp5HB8fD4+RyYyOjoaZjMxrvGjRojDT3d0d\nZrq6usJM1fuQ2VjN7E5m9iunpqZmfS19fX1hpuo9z7x+mfehYkfdnUwAAMopmQAAlFMyAQAop2QC\nAFBOyQQAoJySCQBAOSUTAIBySiYAAOWMsQNQrr29PcxUDXNHY9lZmVHt6FyZ682Mx2dG6KvGuzOD\n5Jn3asmSJWGms7MzzGTeh8xrmHkvFi6c/b22zGtTMfreaOS+V5kB/swfBDDGDgDAvKRkAgBQTskE\nAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlDPGDkC5qkHoqkHtqpH06FxdXV3hMRYsWBBmMqPb\nmfHuzPPOvFcZmesZGBgIM5kh+qpR98zgePS5yDzvzGd0Lofqq97ziDuZAACUUzIBACinZAIAUE7J\nBACgnJIJAEA5JRMAgHJKJgAA5ZRMAADKGWMHoFxmaL21Nf4nKDNiPTExUXKczDB3NKSeOUZ3d3eY\nyYxuZwa1M8PmmWuuev0y15wZkB8aGgozmWvOiJ5XxYh/5jyNRu71q/rOZL6f4bXM+ggAAPAflEwA\nAMopmQAAlFMyAQAop2QCAFBOyQQAoJySCQBAOSUTAIByCzLjnwAA8N9wJxMAgHJKJgAA5ZRMAADK\nKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCAckomAADllEwAAMopmQAAlFMyAQAop2QCAFBO\nyQQAoJySCQBAOSUTAIBySiYAAOWUTAAAyimZAACUUzIBACinZAIAUE7JBACgnJIJAEA5JRMAgHJK\nJgAA5ZRMAADKKZkAAJRTMgEAKKdkAgBQTskEAKCckgkAQDklEwCAcv8DAQ3cHIIUuWcAAAAASUVO\nRK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_all_layers.ckpt\")\n", + "save_fig(\"reconstruction_plot\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training one Autoencoder at a time in multiple graphs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are many ways to train one Autoencoder at a time. The first approach it to train each Autoencoder using a different graph, then we create the Stacked Autoencoder by simply initializing it with the weights and biases copied from these Autoencoders." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create a function that will train one autoencoder and return the transformed training set (ie. the output of the hidden layer) and the model parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size, learning_rate = 0.01, l2_reg = 0.0005, activation_fn=tf.nn.elu):\n", + " graph = tf.Graph()\n", + " with graph.as_default():\n", + " n_inputs = X_train.shape[1]\n", + "\n", + " X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + " with tf.contrib.framework.arg_scope(\n", + " [fully_connected],\n", + " activation_fn=activation_fn,\n", + " weights_initializer=tf.contrib.layers.variance_scaling_initializer(),\n", + " weights_regularizer=tf.contrib.layers.l2_regularizer(l2_reg)):\n", + " hidden = fully_connected(X, n_neurons, scope=\"hidden\")\n", + " outputs = fully_connected(hidden, n_inputs, activation_fn=None, scope=\"outputs\")\n", + "\n", + " mse = tf.reduce_mean(tf.square(outputs - X))\n", + "\n", + " reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n", + " loss = mse + reg_losses\n", + "\n", + " optimizer = tf.train.AdamOptimizer(learning_rate)\n", + " training_op = optimizer.minimize(loss)\n", + "\n", + " init = tf.initialize_all_variables()\n", + "\n", + " with tf.Session(graph=graph) as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = len(X_train) // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " indices = rnd.permutation(len(X_train))[:batch_size]\n", + " X_batch = X_train[indices]\n", + " sess.run(training_op, feed_dict={X: X_batch})\n", + " mse_train = mse.eval(feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n", + " hidden_val = hidden.eval(feed_dict={X: X_train})\n", + " return hidden_val, params[\"hidden/weights:0\"], params[\"hidden/biases:0\"], params[\"outputs/weights:0\"], params[\"outputs/biases:0\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's train two Autoencoders. The first one is trained on the training data, and the second is trained on the previous Autoencoder's hidden layer output:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train MSE: 0.0129656\n", + "1 Train MSE: 0.0129532\n", + "2 Train MSE: 0.0130242\n", + "3 Train MSE: 0.0133689\n", + "0 Train MSE: 0.0030644\n", + "1 Train MSE: 0.00334272\n", + "2 Train MSE: 0.00350276\n", + "3 Train MSE: 0.0036409\n" + ] + } + ], + "source": [ + "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150)\n", + "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can create a Stacked Autoencoder by simply reusing the weights and biases from the Autoencoders we just trained:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28*28\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n", + "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n", + "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n", + "outputs = tf.matmul(hidden3, W4) + b4" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3FuMnWXZP+BnOvtdp7SAU6CgBYECoihYVMJnjOFEoxHD\nAXKgQROJiSaSqIHERD3RM+WEoNG4PzDRqEHcE4mIiIJsZSMVgSoUWrqd/fZ/8B19//jcD66b6Uyn\n13V6537Xu9a7uubXN3l/XcvLywUAgM5tWO0TAAA41glUAABJAhUAQJJABQCQJFABACQJVAAASQIV\nAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJPWs9glULK/2CcA60rXaJ8DRt2fPHr+j8AoZ\nHx9v/o66QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJK3V2gQAOGqWl+stE11dnTePLC0thfPW\nsaN5dM4trdddyWOvV+5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahMAOGoyj+Nnjp2pCWhV\nH0S6u7vD+YYN8X2N6Lxbu/Pz8+E8Eh17cXEx3M1URWRqJFabO1QAAEkCFQBAkkAFAJAkUAEAJAlU\nAABJAhUAQJJABQCQpIcKgDVhtTqqWvNWl1S0u7CwEO729MR/hqNOp1YPVXTerd3M67b6r1r7x6r1\n+a4AAI4igQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJbQIAR01XV1dHs1La1QfRfqueIHqUv/WYf3Re\ns7Oz4W5fX184n5iYqM6OHDkS7g4ODlZno6Oj4W5vb291lqlrKCWukpibmwt3W9dxNblDBQCQJFAB\nACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAElrt9ABgHWn1SWV2Y36jTIdVvPz8/GJBQYGBjp+3VJK\nGRoaqs5a7ynquIo6qkqJe6haoutQSnzemZ6pbI9ZljtUAABJAhUAQJJABQCQJFABACQJVAAASQIV\nAECSQAUAkKSHCoD/YyX7eiYnJ6uzqampcLd1XjMzM9VZpt9oeHg4nG/Y0Pm9idHR0XDe399fnS0t\nLYW7rT6oTk1PT4fz1ucRvadW51f0nhcXF8PdqKeq1WH1crhDBQCQJFABACQJVAAASQIVAECSQAUA\nkCRQAQAkqU14hfzpT38K5zfddFM4P/XUU6uzwcHBcPeDH/xgdbZ58+ZwtzUHjj+tx/Gj+oK5ublw\n96WXXqrO/v3vf4e7rcf1o3mrciGqGGhVLpx88snVWasGYGxsLJz39fVVZw8//HC4G1VUbNmyJdwd\nGhqqzrZu3Rrujo+Ph/PotXt7e8PdqFqj9b3t7u6uztQmAACsAQIVAECSQAUAkCRQAQAkCVQAAEkC\nFQBAkkAFAJDU1ermWCVr8qQi55xzTjh/8sknj9KZ/F+tjpNLL730KJ3J2vDqV786nN9www3h/PTT\nT38Fz+aoyRescMzZs2dPx7+jre6kiYmJ6izqmSqllP3791dnzz77bLi7Z8+ecB69dtQzVUrcYdTq\n1tq3b1/Hu1GvUilxt1br2P39/dVZ1H1YStzZ1OqZuvzyy8P561//+uos043Y6qHasKF+DymalVLK\n+Ph483fUHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIKlntU9gvfjJT34Szh944IFwfv7551dn\nf/vb38Lde+65pzr76U9/Gu7+6le/qs5e85rXhLv//Oc/w3lGT0/9q7l169Zwd/fu3R2/bqtW4TOf\n+UzHx4ZjRas2Iaog6OqKny7v7e3teLf1aHt03q3d6Dfn8OHD4W5fX1911npPUQVFKaUMDg5WZyMj\nI+Huxo0bq7MDBw6Euw899FB1dskll4S7b3nLW8J5dJ1a9RbRdYquQyntWoUsd6gAAJIEKgCAJIEK\nACBJoAIASBKoAACSBCoAgCSBCgAgSQ/VK2THjh2peeTCCy8M51dffXV19qUvfSncffrpp6uzVg/V\nU089Fc4zoj6RVg9VdN579+4Nd88999z4xGCdWF5ers66u7vD3ahbaXR0tOPd0047Ldw98cQTw/n2\n7dvDeWRycrI6i7qgSillenq6o+OW0v6st2zZUp319/eHu/v376/ObrvttnA36sdqdWu1+qCiLrLo\ne1lK3CXV6pmK5q2espfDHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBI0kO1zg0MDITz\nTO9Splsr45577gnn+/btq8527twZ7l5xxRUdnRMca6IuoVY30ubNm6uz+fn5cDfqdIqOW0opc3Nz\n4TwyMzMTzqPupLGxsXD34MGD1dmRI0c6ft1SShkfH+949+67767Oop6pUko59dRTq7NWX9jJJ58c\nzqPPs9UlFfVUtXZb/VlZ7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAElqE1iTJicnq7P3ve99\n4W706OxXvvKVcDd6pBuOF63ahOjx89aj6UNDQ9VZq3Jh06ZN4fzw4cPVWevf9pYtW8J5ZMOG+r2J\n4eHhcLdVyRDNW5/XP/7xj+ps79694W5UfXDhhReGu6Ojo+E8+o3u7e0NdxcWFqqz1ndPbQIAwBon\nUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASXqoWJO+9a1vVWd79uwJd6M+mTPOOKPTU4LjxvLy\ncjiP+nxaHVY9PfU/O/39/eFu69gbN26szqKuqFJK6evrq86i3qRS4u6kVvdR67yi3qXnnnsu3H38\n8cers5mZmXD3rLPO6mhWSikjIyPhPLrO0XUopZSpqalwHom+169ER5U7VAAASQIVAECSQAUAkCRQ\nAQAkCVQAAEkCFQBAkkAFAJCkh4pV8Y9//COcX3/99R0f++67767OxsfHOz4uHC9anTxRL1OrSyrS\n6mRq9VBFPUOtLqmo72nv3r3h7pEjR6qz1ntqdTZFvXu33XZbuHv//fdXZ63PcseOHdVZ1PVXSrtL\namBgoDqLrkMp8Xm3+tNa1yLLHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltAqvi1ltvDefz\n8/PV2VVXXRXubt++vaNzAv5XqzYhejy99eh69Eh967H2VvVB9Nqt9zQxMdHRcUsppaen/qd048aN\n4W7rvHbt2tXRrJRS/vWvf1VnO3fuDHff+ta3VmebN28Od0dHR8N5dB1bn3XmGq80d6gAAJIEKgCA\nJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSQ8VKybqkvrxj38c7vb391dnX/ziF8Pd7u7u+MSAFbO4\nuBjOZ2ZmqrPsv92oh6h1Xq15ZNOmTdXZ2NhYuHvw4MFwfu+991Znd911V7gb/Y5eeuml4e65555b\nnW3ZsiXcbV3H6Dq1uqQyuyvdYeUOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpDaBFfONb3yj\nOrvzzjvD3Q984APV2fbt2zs+J2BltR6Zn5ubq842bIj/j7+wsBDOZ2dnq7PWY/HRa4+MjIS7o6Oj\n1Vlvb2+4+8wzz4Tz3//+99XZU089Fe5edtll1dk73/nOcHfbtm3VWatiYmlpKZxH1RktmXoDtQkA\nAGucQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJOmhomMPPPBAOP/4xz9enW3atCnc/cIXvtDR\nOQErL+rsaXUMRR1FrQ6rVv9R1Pk0NTUV7kY9VK0uqcnJyY5f99Zbbw3n999/f3U2Pj4e7r73ve+t\nzk4//fRwN+r8irrESmlfp+jzanWRRd+Rvr6+cLd17Cx3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIE\nKgCAJIEKACBJDxVV09PT4fzqq68O51EXyTXXXBPubt++PZwDqyfqKIpmpZSyvLxcnbX6i/r7+8N5\ntD8wMBDuRl1Trd2DBw9WZ7///e/D3R/96EfhPOqxuvLKK8Pdt7/97dXZ8PBwuDs/P1+dzc7Ohrut\n70DUF9X6DkQ9ZqvNHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltwnEuegT1Xe96V7j7xBNP\nhPMdO3ZUZ5///OfjEwNWTVRt0NJ67D06dldXV7jbely/p6f+J631uP3g4GB11npPe/bsqc5+9rOf\nhbu7d+8O5xdffHF11vqN3rJlS3XWqjaYnJyszlqfR+s6RqJKhbXOHSoAgCSBCgAgSaACAEgSqAAA\nkgQqAIAkgQoAIEmgAgBI0kN1nNu/f391dscdd6SO/d3vfrc627x5c+rYwOrZsKH+f/He3t5wN+oo\nanUjDQwMhPOow6i1Oz8/X50dOHAg3L399turszvvvDPc3bRpUzh/5zvfWZ1FHVWllDI6OlqdRb/9\npcSdX1HfVyml9Pf3h/NI9N0qJddjlulXezncoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAktQm\nrHOHDh0K55deemnHx/7e974Xzi+66KKOjw0cm7q7u8N5VG3Qeuy9dexMJUP0W3nvvfeGu7/+9a+r\ns+np6XD38ssvD+fvfve7q7MTTzwx3I0+ryNHjoS7UY1Eq9og2i2lXa0Ria7xStcitLhDBQCQJFAB\nACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEl6qNa5b37zm+H8qaee6vjYl112WThvdcoAa1Pr327U\nQ9TaXVpaqs56euI/Sa3+o6iHaGJiItx95plnqrO//vWv4e6uXbuqs6GhoXB3586d4Xzbtm3VWdTp\nVUopMzMz1VmrsynqsIquYSnt6xh9R1rHbn0HVtPaPTMAgGOEQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJAkUAEAJOmhWgeefPLJ6uxzn/vc0TsRgIRWN9L09HQ4j3qXDh06FO5GnXz//Oc/w92xsbHqbOvW\nreHuSSedFM7n5+ersyNHjoS70ee1sLAQ7i4uLlZnvb294W6rhyrTcbWWuUMFAJAkUAEAJAlUAABJ\nAhUAQJJABQCQJFABACSpTVgH7rzzzurs8OHDHR93x44d4XxwcLDjYwPHrq6uro53o0fmo0f1Syll\ndna243lrd2pqqjo74YQTwt3/+Z//qc7Gx8fD3dNPPz2cRzUCExMT4W5UjRBdh1JK6e/vD+cZx3I1\nQsQdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjSQ3Wce+tb31qd/eY3vwl39VAB/62o\nG2l+fj7cXV5eDudRd9LmzZvD3fPOO686O+2008LdjRs3VmfDw8Ph7tDQUDiPOr9avV1R31Nvb2+4\nG33WrdfdsCG+VxO9p0zH2WpzhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASOpqPYa6StbkScEx\n6th9DpmO7dmzZ03+jkZ/c6JKhZcjely/9Sh/VNnQepQ/87qtY7eqJDqVqU04lqsNOjU+Pt580+5Q\nAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQNJa7aECADhmuEMFAJAkUAEAJAlUAABJAhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUA\nQJJABQCQ1LPaJ1CxvNonAOtI12qfAEff7t27/Y7CK2Tbtm3N31F3qAAAkgQqAIAkgQoAIEmgAgBI\nEqgAAJIEKgCApLVamwAAR01XV/2p+OXltdlAsWFDfE9kaWmpOovebylr9z2vZe5QAQAkCVQAAEkC\nFQBAkkAFAJAkUAEAJAlUAABJahMAWPdaNQGLi4sdzUoppbe3t6PZyxG99vz8fLgbzVu7AwMD1Vl3\nd3e426pciPaP5ToHd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSQ/VOvD973+/Opuc\nnAx377vvvursa1/7WsfnVEopn/3sZ6uzd7zjHeHu29/+9tRrA+vP0tJSOI86iqanp8Pd2dnZjs6p\nlPi8WscdHBwM51FfVKsfK9rt6+sLdzdsqN9vaZ1zqysq+rxaHVdrmTtUAABJAhUAQJJABQCQJFAB\nACQJVAAASQIVAEBSV+vxxlWyJk9qtXzsYx8L51/96leP0pm8cs4777xw/oc//KE6Gxsbe6VPZ73r\nWu0T4OjbvXv3mvwdjR6Zb9UATE1NhfNDhw5VZ7t27Qp3d+/eXZ3t378/3H3hhReqsz179oS7J5xw\nQjjfsmVLdTYyMhLu7tixozp77WtfG+6ecsop1dnGjRvD3dZ1jOZR1UMppfT29obzlbJt27bm76g7\nVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJDUs9onwP+KuqZWsmfqoosuqs7e//73h7tP\nPvlkOP/2t79dnT366KPh7g9/+MPq7MMf/nC4C6yeVrdh1DM0MTER7r700kvh/P7776/OHn/88XD3\nkUceqc5avUp79+6tzjZt2hTutrq1on6sbdu2hbtRl9Tg4GC4G3VcDQ8Ph7ut9xR9BzLfn1ZHVXTs\nrq58XZ87VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAktqEo+TZZ58N51//+tc7PvYll1xSnf3y\nl78Md4eGhqqzvr6+cLf1KPGuXbuqs7vuuivc3bdvXzgHVs/S0lLHu3Nzc9XZ4cOHw90XX3wxnB86\ndKijcyqllHPPPbc627JlS7h7/vnnV2ejo6PhblTXUEopDz30UDiPHDx4sDpbWFgId/v7+6uz6enp\ncLf1/YjqC6JahFLiv0ut130lqhEi7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkh6q\no6TVqxT1ckQ9U6WU8tvf/rY6GxkZiU8s4Vvf+lY4/8tf/tLxsd/73vd2vAusrOj3KpqVEncBtTqb\nTjrppHAe/Vbu3r073D3nnHM6mrXO6/nnnw93H3vssXAe7e/duzfcjc470zN44MCBcDfqsCol7sdq\n9RtOTExUZ8PDw+Fua57lDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk6aE6St74xjeG\n86inqtUXMjg42NE5ZX39618P53Nzc0fpTICjqadnZf50dHd3p+bT09PV2VlnnRXunnLKKdXZxo0b\nw93ot67VQ/XCCy+E8+g9b968OdwdGxsL55Go76nVNba0tBTOoz6o2dnZcHdmZqY627AhvkcUnXfU\nj/ZyuUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTVgjMo+3rpTvfve74fzBBx/s+NhXXHFF\nOD/zzDM7PjaQ03r8PHrEvPVIfVS50KpjaM1HR0ers1b9TG9vb3W2sLAQ7u7evbs6u+uuu8LdQ4cO\nhfOogmDHjh3h7sUXX1ydtaogou9A6/vR+ryiKojoOpQSf/dWqs7j5XKHCgAgSaACAEgSqAAAkgQq\nAIAkgQoAIEmgAgBIEqgAAJL0UB3n7r///ursox/9aLg7Ozsbzrdu3Vqd3XTTTeFuq4sEWDmtLqmo\nG6nV9zQ/P1+dTU1NhbtRB1EppQwPD1dnAwMDHZ9Xqyvqd7/7XXX28MMPh7tRh1UppVxwwQXV2ete\n97pwd9u2bdXZ0NBQuLu4uFidtT7L6PtRSnwd+/v7w93oO9Lqx1pp7lABACQJVAAASQIVAECSQAUA\nkCRQAQAkCVQAAElqE45zd999d3XWqkVoue6666qzs88+O3VsYOW0Hj+PHqmfmZkJd6NKhlZdQ6tO\nJXocf3JysuNjt6oN/vjHP1Znd955Z7h78cUXh/OTTz65Otu5c2e4G9UbtCooFhYWqrNWNUbrOkXf\nr6i+oqX1nlrzLHeoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEkP1Tp37bXXhvMf/OAH\nHR/7k5/8ZDj/9Kc/3fGxgdWztLS0Ysfu7u6uzkZHR8Pdnp74T1amZ+jw4cPV2c9//vNw989//nN1\n1upkirqiSillaGioOuvv7w935+bmOpqVEn/W0WdVSvs6ZkTfHz1UAADHOIEKACBJoAIASBKoAACS\nBCoAgCSBCgAgSW3COjAxMVGd/eIXvwh3Z2ZmqrNXvepV4e6NN94Yzvv6+sI5sDYtLy+v2G70u9Cq\nGGjVBMzOzlZnk5OT4e7f//736uyee+4JdxcWFqqzM844I9w9//zzw/kb3vCG6qz1G3vkyJHqLPq7\nUUopY2Nj1VnrOs3Pz4fzSHQNW6JKhaPBHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBI\n0kO1Dlx11VXV2YsvvtjxcT/xiU+E882bN3d8bGB9GhgYCOdLS0vVWU9P/Cepq6srnEe9es8++2y4\n+/DDD1dnBw4cCHdPOumk6mznzp3h7tlnnx3Ox8fHq7PBwcFw9/Dhw9VZ67OM3vNpp50W7i4uLobz\nqE+stRt1YLXe00pzhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS9FAdA+67775wfscd\nd3R87CuvvLI6u/766zs+LrB+dXd3V2cbNsT/T4+6pqKOqpcz37dvX3X20EMPhbt//OMfq7Pp6elw\n981vfnN1dsYZZ4S70WdZSikjIyPhPDI0NFSdTU5Ohrtbtmypzubm5sLdhYWFcL68vFydta5x1FPV\n+ixXuqfKHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltwhoRPZZ7ww03hLutR1gjb3rTm6qz\nvr6+jo8LrF/Ro+3RY+2lxL8rrcfeW/UFTzzxRHX26KOPhrv79++vzlrVB69//eurs7PPPjvcHR8f\nD+dRvUF/f3+4G/1tiCoVSilldHS0Opuamgp3Z2ZmOp63qg3Gxsaqs1ZlRzRv1TW8HO5QAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIeqjXilltuqc5uv/32jo977bXXhvPrr7++42MD61Or\nzyfq7BkeHg53l5eXq7NWh9Xzzz8fzp9++unq7PHHHw93o96l17zmNeHumWeeWZ2ddNJJ4e7AwEA4\nj67FoUOHwt2RkZHqrHWdoj6o+fn5cLfVQxV1fg0ODoa7UX9WqztxYWGhOmt9518Od6gAAJIEKgCA\nJIEKACBJoAIASBKoAACSBCoAgCS1CWvEjTfeuCLH/fKXvxzOW4+ZAsefqBahlFK6u7urs1b1QfR4\n+tzcXLj73HPPhfOoNuHAgQPh7tatW6uzV73qVeFuVE+wadOmcDd6lL+UUqampqqz2dnZcPeUU04J\n55G9e/dWZ08++WTHu6XElQynn356uBtVNrQ+y97e3nCe5Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBA\nkkAFAJAkUAEAJOmhWucmJibCedQJs5L6+/vDeabnptXNEpmeng7nN910U8fHjkTvt5S4p2ylu1Xg\n/7e8vLwix2392231Yx0+fLg6279/f7gb/Sa1epV27doVziOtfqxHHnmkOhsdHQ13o88z6oIqpZQ7\n7rijOnvmmWfC3db346yzzqrOzjzzzHA3+r1r/Y6uNHeoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQq\nAIAktQnr3Kmnnrrap/AfXXfddeH8lFNOqc727NkT7t58880dndNaFl3Hj3zkI0fxTCCuL+jpif+s\nRFUtrQqQffv2hfMXXnihOhsYGAh3o/qCe++9t+Pdu+66K9x98cUXw/nMzEx1NjY2Fu5G9QatipjH\nHnusOmtVPZx//vnhvK+vrzobGhoKd6N6i9Z3r1W7keUOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJA\nBQCQJFABACTpoVojrrnmmursm9/85lE8k6PjlltuWZXXbfWUdHd3d3zsD33oQ+H8LW95S8fHftvb\n3tbxLvy3urq6wnnUJRX1JpUS90G1/n2edtpp4fyss86qzqJzLiXug9q1a1e42+qaikSdTC3nnHNO\nOF9cXKzO9u/fH+5OTU1VZ2effXa4e8EFF4TzM844ozprfR7RdWx9b1eaO1QAAEkCFQBAkkAFAJAk\nUAEAJAlUAABJAhUAQJJABQCQ1LW8vLza5/CfrMmTWi3f+c53wvnc3NyKvO6DDz4Yzm+++eYVed1S\nSvnUpz5VnUVdMy3vec97wvnJJ5/c8bHXsNUtZ2FV7N69e8V+R6MuoKhnqpS46y3qTSqllOeeey6c\nv/DCC9XZ008/He7u27evOnvggQfC3WeeeaY66+3tDXdb3UmnnnpqdTY4OBjuLi0tdXxe27dvr85O\nPPHEcLfVF7Z58+bqbGRkJNwdHh6uzlpdYxnbtm1r/o66QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAk\nUAEAJKlNgPVPbcJxaCVrE6JH/VuPrmcqF1q1ClFNQF9fX8fHPnjwYLh7+PDh6qz1N7b1eR06dKg6\nm5qaCnd7enqqs6GhoXA3+rxan2VUjVFK/J5bNRJRVYTaBACAY5xABQCQJFABACQJVAAASQIVAECS\nQAUAkCRQAQAk1UsqAOA/iPqeWqKeoZmZmXB3ZGQknPf29lZnmT6o0dHRjl+39Vm13vP8/Hx1Nj09\nHe7Ozc11fF6ZjspMF1nrdVs9VavJHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltAgD/lejR\n9dZj75nH8ScnJ8N5pjYh8zj+7OxsdTY0NBTuLiwsdHzs1ntaXFyszlrVBgMDA9VZq3Ihet2WtVyL\n0OIOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACTpoQLgqIk6jFrdSPPz8+E86j9q9RtF\nnU6tXqXoPbXOuacn/jMcdWvNzc2Fu93d3dVZ6/OIzjvTJdaihwoA4DgmUAEAJAlUAABJAhUAQJJA\nBQCQJFABACSpTQDgmLCS1QcZUd1Dq2JgYWEhnEf7rWNnzitTb3G88qkAACQJVAAASQIVAECSQAUA\nkCRQAQCsKOmNAAAAtUlEQVQkCVQAAEkCFQBAUleriwIAgJg7VAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJ/w+NWuXmfmiQvgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training one Autoencoder at a time in a single graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another approach is to use a single graph. To do this, we create the graph for the full Stacked Autoencoder, but then we also add operations to train each Autoencoder independently: phase 1 trains the bottom and top layer (ie. the first Autoencoder) and phase 2 trains the two middle layers (ie. the second Autoencoder)." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28 * 28\n", + "n_hidden1 = 300\n", + "n_hidden2 = 150 # codings\n", + "n_hidden3 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "l2_reg = 0.0001\n", + "\n", + "activation = tf.nn.elu\n", + "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "\n", + "weights1_init = initializer([n_inputs, n_hidden1])\n", + "weights2_init = initializer([n_hidden1, n_hidden2])\n", + "weights3_init = initializer([n_hidden2, n_hidden3])\n", + "weights4_init = initializer([n_hidden3, n_outputs])\n", + "\n", + "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n", + "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n", + "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n", + "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n", + "\n", + "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n", + "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n", + "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n", + "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n", + "\n", + "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n", + "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", + "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n", + "outputs = tf.matmul(hidden3, weights4) + biases4\n", + "\n", + "\n", + "with tf.name_scope(\"phase1\"):\n", + " optimizer = tf.train.AdamOptimizer(learning_rate)\n", + " phase1_outputs = tf.matmul(hidden1, weights4) + biases4 # bypass hidden2 and hidden3\n", + " phase1_mse = tf.reduce_mean(tf.square(phase1_outputs - X))\n", + " phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n", + " phase1_loss = phase1_mse + phase1_reg_loss\n", + " phase1_training_op = optimizer.minimize(phase1_loss)\n", + "\n", + "with tf.name_scope(\"phase2\"):\n", + " optimizer = tf.train.AdamOptimizer(learning_rate)\n", + " phase2_mse = tf.reduce_mean(tf.square(hidden3 - hidden1))\n", + " phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n", + " phase2_loss = phase2_mse + phase2_reg_loss\n", + " phase2_training_op = optimizer.minimize(phase2_loss, var_list=[weights2, biases2, weights3, biases3]) # freeze hidden1\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training phase #1\n", + "0 Train MSE: 0.00779163\n", + "1 Train MSE: 0.00745319\n", + "2 Train MSE: 0.00768598\n", + "3 Train MSE: 0.0077875\n", + "Training phase #2\n", + "0 Train MSE: 0.00224432\n", + "1 Train MSE: 0.00251813\n", + "2 Train MSE: 0.00247135\n", + "3 Train MSE: 0.00259297\n", + "Test MSE: 0.00284198\n" + ] + } + ], + "source": [ + "training_ops = [phase1_training_op, phase2_training_op]\n", + "mses = [phase1_mse, phase2_mse]\n", + "n_epochs = [4, 4]\n", + "batch_sizes = [150, 150]\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for phase in range(2):\n", + " print(\"Training phase #{}\".format(phase + 1))\n", + " for epoch in range(n_epochs[phase]):\n", + " n_batches = mnist.train.num_examples // batch_sizes[phase]\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n", + " sess.run(training_ops[phase], feed_dict={X: X_batch})\n", + " mse_train = mses[phase].eval(feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " saver.save(sess, \"my_model_one_at_a_time.ckpt\")\n", + " mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n", + " print(\"Test MSE:\", mse_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3FuMnWX5N+BnOpvOpu1stC22ILSKShVMaQwomGDdQKLR\nqCERSdSgicZEE0nUQGKinuiZckLQaETBAxKMGjXYCEICaCvWgpFttdJitUBt6X72/Q7++ZLvy9/n\nfuq6O51hel2nd+53vetdq2t+fZP313Xy5MkCAEDnlsz3CQAAvNwJVAAASQIVAECSQAUAkCRQAQAk\nCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACT1zPcJVJyc7xOARaRrvk+AM+/Q\noUN+R+E0GR4ebv6OukMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQt1NoEAHjZW7Ikd99iZmam\nOuvqip/kj+azs7Md7/KfuUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTQBgUcg86j9fuydP\nngzn3d3dHR87qkZo1SZEWlUQZ2vlgjtUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkKSH\nCoAFodVflOlOah076oNq7c5lD9XExER1tnTp0nA3Oq/W605NTVVn09PT4W5PTxwt+vv7w3lkIXdc\nuUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTQBgUYgeqc88bh9VCJQS1wS0qh6OHj0aznt7\nezt63VJK6e7urs5atQnReUfndCrHjmoXliyJ7/NEn2P0fk/lvLLcoQIASBKoAACSBCoAgCSBCgAg\nSaACAEgSqAAAkgQqAIAkPVQALAgzMzPhPNOd1BLtZvqeWu9pZGSk4/MaHx/v+Lz6+/vD3eg9tzq9\nJiYmwnmk1XEVifqtSsl1kZ0Kd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSQ8VAP+V\nqA+q1RU1NTXV8e6hQ4fC+eTkZHXW19cX7i5btqzj3agranBwMNxtdVxF16TVq5Tpx4peN9PLVUr8\nObXeU+b7EzkdHVXuUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASWoTTpOtW7eG81tuuSWcr127\ntjobGBgIdz/+8Y9XZ2NjY+Fuaw6cfVqPn09PT1dnR44c6Xh3z5494e6zzz7b8XxiYiLcXb9+fXW2\nbt26cHdoaKg66+/vD3dbv+9RnUN0LUuJqwCi6oJSSjl8+HBH51RKKSMjI+E8ql1oVVRkKhd6e3vD\neZY7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJDU1eobmScL8qQir3/968P5zp07z9CZ\n/P+Gh4fD+eWXX36GzmRhuOCCC8L5TTfdFM5f/epXn8azOWPichYWpUOHDnX8Ozo7OxvOjx8/3tGs\nlFL27dtXnT311FPh7mOPPRbO9+7dW50tXbo03B0cHKzOot6kUkp5/vnnq7MlS+L7Fq2/wdGxx8fH\nw93Xve511Vmr/yo6r9HR0XD3iiuuCOdXXnllddb6nDKiHqrWd354eLj5O+oOFQBAkkAFAJAkUAEA\nJAlUAABJAhUAQJJABQCQFD8Lyin72c9+Fs4fffTRcP7GN76xOnv88cfD3W3btlVnP//5z8PdLVu2\nVGfr1q0Ld//+97+H84zoMeVXvepV4e5zzz3X8eu2ahW+/OUvd3xsWEiix+K7u7vD3dY8EtUTrFq1\nKtxt/fvs6+urzlrVBydOnKjOouqCUko5cuRIdTY2NhbuRlUPpZSyZ8+e6qz1Oezfv786iz6HUkrZ\nvXt3dTY0NBTurl69Opy//e1vr85atQnR97ZVfTAzM1OddXXl22XcoQIASBKoAACSBCoAgCSBCgAg\nSaACAEgSqAAAkgQqAIAkPVSnyUUXXZSaRy655JJwft1111Vn3/zmN8PdZ599tjpr9VDt2rUrnGdE\nfTKtHqrovF988cVw9w1veEN8YrBIZHp3+vv7q7OoJ6iU+N92q4NoZGQknC9ZUr9HMDAwEO6+8MIL\n1dnBgwfD3ahrqre3N9xtXa8nn3yyOhseHg53oz6o3/72t+Hu008/XZ2tX78+3F25cmU4jzrBWt1a\nres1V7unwh0qAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASNJDtchFfTGl5HqXMt1aGdu2\nbQvn+/fvr84uu+yycPc973lPR+cEZ5OoK2j58uXh7uTkZHUWdVSVUsqqVavCeabjKupWinqTSill\ndna2Omv9Br/00kvh/OKLL67OVqxYEe5OT09XZ1u3bg13JyYmqrOod6uU9t+G6Hq2uqKiax2935ZM\nL9v/5Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkNoEF6dixY9XZBz/4wXA3eqz229/+drg7\nMDAQnxicBVqPrkeWLIn/nx7NBwcHw93o33Yp8b/f8fHxcDd6bL5V59A6r8iyZcvCeXS9ent7w92/\n/vWv1dmOHTviEwu06nbOO++8cB5dz1b1QfQ5nY7qgwx3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIE\nKgCAJIEKACBJDxUL0u23316d7du3L9x9xSteUZ2df/75nZ4SnDUyPVStHqFo3t3d3fHrllLKxMRE\nx7tRN1Krwyo675mZmXC3NY+6pg4ePBju/vCHP6zOHn300XD3sssuq842b94c7q5YsSKcR31RrR6z\njOh7fTo6rNyhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCQ9VMyLv/3tb+H8xhtv7PjY\nv//976uzc845p+PjAv8j01M1OztbnbU6rAYGBsJ5tN/qGZqcnKzOWt1I0bz1upljb926Ndz9yU9+\nUp21Or/e9773VWdr1qwJd5cuXRrOI63rMdddUhnuUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAA\nSWoTmBe/+MUvwvnU1FR1du2114a769ev7+icgP+Refw8qkUopZT+/v7qrFWbMDMzE857e3urs9Z7\nis47s9uqAWhVDLz44ovV2Z///OdwN7qe7373u8PdK6+8sjpbvnx5uNuq1YiuZ+szjuZqEwAAXuYE\nKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSQ8VcybqkvrpT38a7kbdLN/4xjfC3e7u7vjEgJSo\nZ6inp/M/K1GPVCnxb0oppUxMTFRng4OD4W7UYRQdt5S4G2lkZCTcbb2nZ555pjp75JFHwt0NGzZU\nZ1dffXW4e+6551Znra6x8fHxcB59R1pdUlGvV6v/aq65QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAk\nUAEAJKlNYM58//vfr84efPDBcPejH/1odbZ+/fqOzwnIix5Pb9UAtKoROn3dUuK6ldZ5RY/jR7NS\nSlm2bFl11nq/e/fuDed33XVXdbZr165w993vfnd1dvnll4e7Uf1M61pGNRKl5KoPWp9FJDp2q67h\nVLhDBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEl6qOjYo48+Gs4/97nPVWcjIyPh7te/\n/vWOzgnIy3QBTU5Odrzb6hiKupFKic+7dezZ2dmOd6enp6uziYmJcPfee+8N5/fff391Njw8HO6+\n4x3vqM7GxsbC3ehzbPVMZbrGMp9xqx+rdewsd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSB\nCgAgSQ8VVSdOnAjn1113XTiPukquv/76cHf9+vXhHJg/0b/tTN9TV1dXuNvT0/mfrNaxoy6pvr6+\ncDd6z3/4wx/C3Z/97Gfh/ODBg9XZRz7ykXB306ZN1Vmrkynqz5rLHqro+1FKrmtsrrlDBQCQJFAB\nACQJVAAASQIVAECSQAUAkCRQAQAkqU04y0WPqL73ve8Nd59++ulwftFFF1VnX/va1+ITA+ZNq2Ig\neuQ+qh8oZe4qF7LHjh71b9UA/OMf/6jOfvKTn4S7v/rVr8L5NddcU51dffXV4e7o6Gh1Njk5Ge4e\nP368OhscHAx3W/UW0efU+u5NTU1VZ1GlwqkcO8sdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAg\nSaACAEjSQ3WWO3DgQHX2wAMPpI59xx13VGdjY2OpYwPzJ+r7aXVFRbsTExPh7tDQUDhv9R9Fop6q\nVmfTI488Up3dc8894e75558fzj/2sY9VZ1HXXynxe4q6oEopZfny5eE80voco8+p9f2JOtBa72mu\nuUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTVjkDh06FM4vv/zyjo995513hvONGzd2fGxg\n/nR1dXW829vbG86j2oRoVkq7viCa9/X1hbvRo/x79+4Nd3fs2FGdRdU0pZRy/fXXh/PoN3pgYCDc\njeoLWhUT0fWanp4Od1vfn6mpqY53o1qFqFLhVI6d5Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJAkUAEAJOmhWuR+8IMfhPNdu3Z1fOwrr7wynM915wcwN1p9UBlRv1HUMVRK+zclOnaro+j555+v\nzn75y1+Guz/60Y+qs5UrV4a7b37zm8P5qlWrwnmnMn1PMzMz4W6mp2q+u6Qy3KECAEgSqAAAkgQq\nAIAkgQoAIEmgAgBIEqgAAJIEKgCAJD1Ui8DOnTurs69+9atn7kSAs16rJ2hiYqLj3VY/1pIl9XsE\nBw8eDHcffPDB6uyhhx4Kd1944YXqbNOmTeFuq6fq2LFj1dnQ0FC4G13PufycWvOoayrTgTbfHVXu\nUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASWoTFoHocd/Dhw93fNyLLroonA8MDHR8bGBxaj32\nHj0ynxUdO6pUKKWUycnJ6qy/vz/cveaaa6qzq666Kty98MILw3lvb291NjU1Fe5G1QeZzyk6p9Zu\nKfNfbzBX3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJD1UZ7m3ve1t1dlvfvObcFcP\nFXAmtbqkpqenq7Px8fFwd82aNdXZW9/61nD3ggsuqM7e8IY3hLujo6PhPJLpkmrttrqm+N/coQIA\nSBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkrpaj07OkwV5UvAy1TXfJ8CZd+jQIb+j/4/W37qoVmF2\ndjbcjeoJ+vv7w91WFURXV/2fb+s9RfMF+rd/wRoeHm7+jrpDBQCQJFABACQJVAAASQIVAECSQAUA\nkCRQAQAkCVQAAEkLtYcKAOBlwx0qAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS\nBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS\nBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS\nBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCApJ75PoGKk/N9ArCIdM33\nCXDmHThwwO8onCZjY2PN31F3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCApIVamwAAi97Jk523\nW8zOzobznp76n/jW60bH7uqKGwRa88XKHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltAgCL\nQvSof6smoLu7uzqbz5qAmZmZ6qxVmxC959bu5ORkdbZkSXwvpnU9ent7w3nm2JFMRcWpcIcKACBJ\noAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkvRQLQI//vGPq7Njx46Fu9u3b6/Ovvvd73Z8TqWU\n8pWvfKU627x5c7h71VVXpV4bWHymp6fD+cTERMfHnpqaqs5av6Pj4+PV2ZEjR8Ld1jlH89WrV4e7\nK1asqM6i3q1SSunr66vOWn1OrZ6pnp7Oo8dcdn5luUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFAB\nACR1tR5/nCcL8qTmy2c/+9lw/p3vfOcMncnps2HDhnD+0EMPVWfDw8On+3QWu4X7nDFz5sCBAwvy\nd3R2drY6m5ycDHf3798fzg8cOFCdPfnkk+HuX/7yl+rsiSeeCHd3795dnR0/fjzcHR0dDecrV66s\nzlr1Mu9617uqs/PPPz/cHRkZqc6WLInvxUQVFKXEtQmtaoyoNqF1XjMzM+E8MjY21vwddYcKACBJ\noAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkuplEJxRUdfUXPZMbdy4sTr78Ic/HO7u3LkznP/w\nhz+szlq9LnfffXd19slPfjLcBeZPq9sw6hk6fPhwuPuPf/wjnG/fvr0627ZtW7j7zDPPVGdRd1Yp\npbzxjW+szs4999xw98iRI+H8n//8Z3X21FNPhbsrVqyoztasWRPu9vf3V2etz7jVQxV1SbW6orq7\nuzvejT7HVofVqXCHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIUptwhuzZsyecf+973+v42G95\ny1uqs1//+tfh7uDgYHXW19cX7rYeUf3rX/9anT388MPh7v79+8M5MH9a//Yj0SP1rcftjx8/Hs6P\nHj1anS1fvjzcveaaa6qzq666KtzdtGlTddZ6HL9V53DnnXdWZydOnAh3o4qKiYmJcPfgwYPVWVSp\ncCrHjs47qlQoJf6ORJUKpZyeaoTw+HN6dACAs4BABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk\n6aE6Q1q9SidPnqzOop6pUkq59957q7Nly5bFJ5Zw++23h/NHHnmk42N/4AMf6HgXyIl+j1qOHTsW\nzoeGhjp+3Z6e+E/W2NhYR7NSStm8eXN1duGFF4a7w8PD1dmhQ4fC3ag7q5RSdu7cWZ21ft+j9zw6\nOhruRl1TrT6wjFbHWdQl1fr+6KECAFjgBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEkP1Rly\n6aWXhvOop6qvry/cHRgY6Oicsr73ve+F88nJyTN0JsB/K+rsafX5RPOlS5eGu1EX0Nq1a8Pd3t7e\ncB7tt85r1apV1VnrN3h6ero6O3jwYLi7bdu2cD4+Pl6dtbqkXvva14bzyOzsbHXW6nOKdkuJ/2a1\nesyiY3d1dYW7rXmWO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJLahAVieHh4vk/hf7njjjvC\n+WOPPdbxsd/znveE89e85jUdHxvIaT0WHz1+3qoYiKoPWrUIrfmhQ4eqs1YVRHd3d3XWqgE4evRo\ndfa73/0u3H344YfDeVSbsG7dunB3aGioOouqHkop5ciRI9VZq4Ki9TnNVaVOqxZhZmamOos+/1Pl\nDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk6aE6y+3YsaM6+/SnPx3uTkxMhPNXvepV\n1dktt9wS7rZ6TICcqLOn1dnU01P/09HqAor6flq7rT6owcHB6qzVfRSd1/Hjx8Pdp556qjr74x//\nGO4ODAyE8w0bNlRn73znO8PdqKcq6mQqpZR9+/ZVZ6tXrw53W51O0ferv78/3I2+A61+rNZ7znKH\nCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIUptwlvv9739fnbVqEVo+85nPVGeve93rUscG5k/0\n2HtUqVBK/Nj7kSNHOn7dUuLH9cfGxsLd8fHx6mzXrl3h7k9/+tPq7L777gt316xZE843b95cnV1y\nySXhbvRZtK716Ohoddb6HKampsJ59B1oVWdENROtWo255g4VAECSQAUAkCRQAQAkCVQAAEkCFQBA\nkkAFAJAkUAEAJOmhWuRuuOGGcH7XXXd1fOwvfOEL4fxLX/pSx8cG5k+rCyjq+5mZmQl3lyyp/z8+\n6pEqpZQTJ06E8/7+/uqs1Z104MCB6uyBBx4Id++///7qbOnSpeHu2972tnAedfaNjIyEuy+99FJ1\nFl2rUkoZGhqqzlo9U63vQPQ5T05Ohrt9fX3VWfTdKqX9vc5yhwoAIEmgAgBIEqgAAJIEKgCAJIEK\nACBJoAIASFKbsAgcPXq0OrvnnnvC3fHx8eps9erV4e7NN98czqPHW4GFq1UxED1+3tqNfq96e3vD\n3WXLloXzqKJg//794e7u3bursz/+8Y/h7vDwcHX2mte8Jtx905veFM43btxYnbUqGQ4fPlydLV++\nPNyNqjFaWrUK0Xn39MSxJFO5EO22vrenwh0qAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIA\nSNJDtQhce+211dkLL7zQ8XE///nPh/OxsbGOjw0sXFHPVGve6i+K+n5aHURLlsT3AKIeon379oW7\nDzzwQHX2xBNPhLtr166tzi6//PJwt9VDNTg4WJ0dOXIk3I36nlqf8fT0dHX2z3/+M9w955xzwnmm\nS2pmZqY6a3VJRfPW9TgV7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkh6ql4Ht27eH\n86g/peVDH/pQdXbjjTd2fFxg8Yo6e1o9QplupL6+vnA+MTFRnf3pT38Kd7ds2VKdHThwINy9+OKL\nq7MNGzaEu8uWLQvnU1NT1dnAwEC4e+LEiers8OHD4W7UfzUyMhLuRp9DKfHnHPVMtY7d6jHTQwUA\nsMAJVAAASQIVAECSQAUAkCRQAQAkCVQAAElqExaI6PHWm266KdxtPaYc2bRpU3XWekQZODtlHj+P\nfldaj8xPT0+H82effbY627FjR7i7Z8+e6mz16tXh7lve8pbq7JWvfGW427pe0bVuXY/e3t5wHonq\nGlq1CZnqgyVL4vs80fWKrlUppXR3d4fzLHeoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoA\nIEkP1QJx2223VWf33Xdfx8e94YYbwvmNN97Y8bGBs1PUBdTf3x/uRl1As7Oz4e6//vWvcL5ly5bq\nbOvWreHuwMBAdXbZZZeFu+94xzuqs9HR0XC31SUV9UENDw+Hu1EvU9R9WEr8Oba6osbHxzueR++3\nlPg9z3XPVIs7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAktqEBeLmm2+ek+N+61vfCud9fX1z\n8rrAy1f0uH3LzMxMOI8ex289yn/s2LFw/ve//70627dvX7i7du3a6uzSSy8Nd9esWVOdtWokjhw5\nEs6jWoXJyclwN6o3aFUfRJ/j4cOHw93W53T8+PHqrPXdW7p0aXXWqk2I6j5a1+NUuEMFAJAkUAEA\nJAlUAABJAhUAQJJABQCQJFABACQJVAAASXqoFrmjR4+G89PRvdGJqEuklLhPpNVzMzEx0dE5ldLu\nwbnllls6Pnak1Z8S9ZT19vae7tPhLDc7OxvOo66g1vcx0wV06NChcP78889XZ//617/C3YGBgY7P\n67nnnqvOWr9XrfcU/YYvX7483M2I+rH27t0b7ra+P5GVK1eG8+g9t35HBwcHOzqnU+UOFQBAkkAF\nAJAkUAEAJAlUAABJAhUAQJJABQCQpDZhkVu7du18n8J/9JnPfCacr1mzpjrbt29fuHvrrbd2dE4L\nWfQ5fupTnzqDZ8JiEVUfZHZbxx0fH6/OokqFUnKPxbeqD/79739XZ3fffXe4u2XLlurs8OHD4e7Q\n0FA4Hx0drc5a9TM9PfU/8a1KnZdeeqk6e/HFF8PdsbGxcP6mN72pOnvlK18Z7h47dqw6a13LqM6h\n9d06Fe5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIeqgXi+uuvr85+8IMfnMEzOTNu\nu+22eXndqJellFwXySc+8Ylw/ta3vrXjY19xxRUd78J/EnU+tfqgok6n6enpcDf6N9bqihoeHg7n\nr3/966uzjRs3hru7d++uzp5++ulw98SJEx3NSill2bJl4Ty6nuedd164OzIyUp21fgujPrFW31PU\nB1ZK3DXVek+veMUrqrPW96c1z3KHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJK6oq6J\nebQgT2q+/OhHPwrnk5OTc/K6jz32WDi/9dZb5+R1Synli1/8YnX22te+tuPjvv/97w/nq1at6vjY\nC1hcKsSidODAgTn7HY16qlpdbq2Oq0ir4+qZZ56pzh5//PFwN+qLOnr0aLh7/Pjx6mz79u3h7sGD\nB8N5dD3XrVsX7r761a+uzlr9V6tXr67OZmZmwt1WD1XUNTU2NhbuRh1YrX6sTA/V2NhY84vrDhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkKQ2ARY/tQlnobmsTYi0ahP6+vqqs9bj+EuXLg3nmQqZ\nqM6h9XdyYmKiOvv3v/8d7u7duzec9/T0VGeteoLR0dHqrFUh0N/fX521rnPrekWv3dvbG+5G1yNT\nydGiNgEA4AwQqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJD1UsPjpoToLzVcPVUvUQdTqsGr9\nvTp+/Hh1Nj4+3vF5rVixItyNOpta5zw7OxvOh4aGqrPp6elwN+psau1G86mpqY53S2lfk4VIDxUA\nwBkgUAEAJAlUAABJAhUAQJJABQCQJFABACTVn6kEgP9S65H4mZmZ6qyrK34yPdpt7UcVAq3dVuXC\nxMREx7sDAwPhPKogiKoeSonfU7bOodPXbc0zrzvf3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCAJD1UAJwxUf9Rq2equ7s7nEf9Rq3Opkirsynqimr1TLU6m6KOq1ZnU19fX8e7met1tnLF\nAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAktQkAnDatGoDWPNKqVYjqDXp64j93UY1A65yjY7d2\nW5UMGdGx1SKcfq4oAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJHXNZQcGAMDZwB0qAIAk\ngQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAk\ngQoAIEld5rK6AAAAYElEQVSgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaAC\nAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgKT/AzREwegG\nml2+AAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_one_at_a_time.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cache the frozen layer outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training phase #1\n", + "0 Train MSE: 0.00731381\n", + "1 Train MSE: 0.00788415\n", + "2 Train MSE: 0.00775154\n", + "3 Train MSE: 0.00779115\n", + "Training phase #2\n", + "0 Train MSE: 0.00237241\n", + "1 Train MSE: 0.00256232\n", + "2 Train MSE: 0.00269759\n", + "3 Train MSE: 0.00276068\n", + "Test MSE: 0.00299921\n" + ] + } + ], + "source": [ + "training_ops = [phase1_training_op, phase2_training_op, training_op]\n", + "mses = [phase1_mse, phase2_mse, mse]\n", + "n_epochs = [4, 4]\n", + "batch_sizes = [150, 150]\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for phase in range(2):\n", + " print(\"Training phase #{}\".format(phase + 1))\n", + " if phase == 1:\n", + " mnist_hidden1 = hidden1.eval(feed_dict={X: mnist.train.images})\n", + " for epoch in range(n_epochs[phase]):\n", + " n_batches = mnist.train.num_examples // batch_sizes[phase]\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " if phase == 1:\n", + " indices = rnd.permutation(len(mnist_hidden1))\n", + " hidden1_batch = mnist_hidden1[indices[:batch_sizes[phase]]]\n", + " feed_dict = {hidden1: hidden1_batch}\n", + " sess.run(training_ops[phase], feed_dict=feed_dict)\n", + " else:\n", + " X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n", + " feed_dict = {X: X_batch}\n", + " sess.run(training_ops[phase], feed_dict=feed_dict)\n", + " mse_train = mses[phase].eval(feed_dict=feed_dict)\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " saver.save(sess, \"my_model_cache_frozen.ckpt\")\n", + " mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n", + " print(\"Test MSE:\", mse_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3FuMXmUZNuA1+01bZjoNLaUEpBRsUfa7okbBIAnBgGIw\nAQ5UNJGYaCKJGkhM1BMPlROCBgOKxpiYIDFGUBGwaFuRTRVqSUtbi6QN1G6mnf1M+x/5J39+3uet\n39N2pu11nT551re+TdfcXcm62w4fPtwAANC69tk+AQCAE51ABQCQJFABACQJVAAASQIVAECSQAUA\nkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQFLnbJ9AweHZPgE4ibTN9glw/O3fv991\nFI6SgYGB6nXUHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIGmu1iYAwAnh8OFj11DR1lZ+Wr/2\nusdql3fnDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkKQ2AYA5YbbqB2rzQ4cOtfy6tffU0dHR\n8n7mPWVet/Z5ZL7HE7muwR0qAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASNJDBcCc0N4e\n/x8/029UO/bMzEzLrzs9PV2cdXbGf2Zrx47mk5OT4W5PT0/Lrxt1TUWf1ZGIvovaec3lnip3qAAA\nkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJLUJABw1mWqDzG7T1KsRIlHFQFQh0DRN09XVVZzVHvOv\nHTuqZKgde2pqquXXnZiYKM6i99s0TdPd3R3Oo++pdl5zmTtUAABJAhUAQJJABQCQJFABACQJVAAA\nSQIVAECSQAUAkKSHCoA5IdtBFPUbRZ1MTRP3PWXOq7e3N5zPzMyE85GRkeJseHg43F2wYEFxdtpp\np4W7fX19xVn2e4p6rGqfR0dHR3FW+46zPWc17lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkC\nFQBAkh4qAI6bqGdoYmIi3B0fHw/nUZdUbXfevHnFWdTn1DRx11Sthyp63dprL1myJNyNerna2trC\n3eh7qvVQZXuqIrWeqogeKgCAOU6gAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbcJSsW7cunD/wwAPh\nfNmyZcVZX19fuPuZz3ymOBsaGgp3a3Pg1JN5vLxWT7Bnz57ibNOmTeHuzp07w/nIyEhxVnuU//TT\nTy/OzjrrrHA3ukbXrrG12oSodiF6v7V5V1dXuBtVLtSqC+bPnx/Oo8+kduyo7qG2m6mROBLuUAEA\nJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAEBSW6Zv5BiakycVee973xvON2/efJzO5P81MDAQ\nzlevXn2czmRueM973hPO77vvvnB+9tlnH8WzOW7yBSuccPbv3x9eR6Penenp6fDYk5OTxVmtG2nL\nli3F2euvvx7u7t+/P5zv3bu3ODt48GC429PTU5wNDw+Hu1H/Ua2XK+qZapqm6ews10XWPo/+/v6W\nZk0Tf8e16+CNN94YzleuXFmcdXd3h7vRZ93R0RHuZgwMDFSvo+5QAQAkCVQAAEkCFQBAkkAFAJAk\nUAEAJAlUAABJ5ecx+Z/86le/CuevvPJKOH/f+95XnL322mvh7vr164uzJ554Itx96qmnirNzzz03\n3N22bVs4z4geFV66dGm4++abb7b8urVahW984xstHxvmkqg2ofb4eVS3U6vi6evrK85qj/K3t8f3\nALq6uoqz2nVjdHS0OIuuR03TNO+8805xVqtNiOoamqZptm7d2vLu0NBQOI/s2LGjOKt9Hm+//XY4\nX7FiRXFWe0/Ra892DZQ7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJDUNtu9DQVz8qRO\nRLUOlO3btxdntR6qqB8lq7u7uzir9clE5x31xTRN0zz++OPh/NZbbw3nc1S5cIiT1v79+1u+jkYd\nVU3TNNPT08XZgQMHwt1oPjMzE59YRXS9q3UnjY2NFWeTk5Ph7vDwcHE2ODgY7u7bty+c79q1qzg7\n//zzw91FixYVZ3/605/C3d/97nfFWe0afMcdd4TzK664ojiLusSaJv7tHTp0KNyt/a4jAwMD1WV3\nqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACApLubghNfb2xvOV65c2fKxV61a1fJuxvr1\n68P57t27i7Nrrrkm3L3xxhtbOic4mdT6CaNOp1rvUk9PT3FW6yCq9QhF+7WOoug91/qxomN3dHSE\nuwcPHgzn/f39xdnAwEC4OzExUZw988wz4W7UF7Z8+fJwt/YbaG8v38upfU/R5znbvZruUAEAJAlU\nAABJAhUAQJJABQCQJFABACQJVAAASWoTmJNGRkaKs09+8pPhbvTY7fe///1wt6+vLz4xOAXU6glq\nj7ZHon9j3d3d4W7tsfjocfzx8fFwd3JysuXzqs0j8+fPD+dR9U3te3jjjTeKs1dffTXcjd7TxRdf\nHO6efvrp4Tz6nmoVFbUaikj0+6n95o+EO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQ\npIeKOenRRx8tznbt2hXuLlq0qDg755xzWj0lOGXU+p6izp7Mbq0LaGJiouVj13Zr5x2JOpvGxsbC\n3VrvUjTfvXt3uPvEE08UZxs2bAh3L7nkkuLssssuC3cHBwfDeeb3k+lAO9bcoQIASBKoAACSBCoA\ngCSBCgAgSaACAEgSqAAAkgQqAIAkPVTMijfeeCOc33vvvS0fe+3atcXZGWec0fJxgbyoZ2h8fDzc\nrXU6dXa2/ietp6en5d2pqanibHJyMtytnXP0eb366qvh7rp168J55GMf+1hxtmLFinC3o6MjnEdd\nUrWeqWhe67CqnVeWO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJLaBGbFr3/963AePYZ8++23\nh7vLly9v6ZyAIzM9PV2ctbW1tXzcmZmZcJ557L1WixDVF9Qex4+uV93d3eFu7T1FVRIvvPBCuLt3\n797i7JJLLgl3L7/88uJs/vz54W70+2iaejVCq7u147a3l+8hZX63//f46SMAAJziBCoAgCSBCgAg\nSaACAEgSqAAAkgQqAIAkgQoAIEkPFcdM1M3y+OOPh7tRZ8x3v/vdcDfTVQPURf/GMh1EtS6g3t7e\n+MQSomtOdC2rzaN+q6Zpmq6urnC+cePG4mzLli3hbtQXddNNN4W7ixcvLs5qfU+1PrHo86rt1j6v\nSK1PLMsdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSW0Cx8yPfvSj4mzNmjXh7p133lmcLV++\nvOVzAo6tWvVB9Mh9rRahvT2+BxBVNtQe9Z+YmCjOxsbGwt2+vr7irFbj8tZbb4XzJ598sjjbunVr\nuHv99dcXZ1dddVW4G30XmRqJpok/k9rvJ5rXPutapUeWO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJ\nAhUAQJJABQCQpIeKlr3yyivh/Mtf/nJxNjg4GO5+5zvfaemcgGMv6gKq9T1FXUG1HqFMl1St36i7\nu7vl84r6sSYnJ8Pd9evXh/N169YVZ/PmzQt3P/KRjxRnixYtCncz76km+i4OHz4c7tZ+A5Ho2LXf\nx5FwhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS9FBRNDY2Fs7vuOOOcD4zM1Oc3XXX\nXeHu8uXLwzlw7NQ6eaI+n+jffdPEfU+1DqLMvKurK9yN5p2d8Z/KqBtp48aN4e5TTz0Vznfs2FGc\n3XbbbeHuBRdcEM4jma6p2u8n6rjK9EHVfh9Ho2sq4g4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJCkNuEUFz3ue/PNN4e7r7/+ejhftWpVcfbtb387PjFg1kTXhaaJHz+vPboe1SrUHmuvnVdHR0dL\ns6aJawJq5/Xmm28WZz//+c/D3eeeey6cr169ujj7+Mc/Hu6efvrpxVntsxwfHw/nkVrNxLHarb2n\nSO13eyTcoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkPVSnuD179hRnzz77bOrYjz32\nWHE2NDSUOjYwe45GZ8+7iTqqmqZpurq6wnmmHytSO6+1a9cWZ+vXrw93zznnnHD+6U9/ujh7//vf\nH+5Gn0ftPUW9XbVervb2+F7Nsfqepqenw3mtiyzLHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoA\nIEltwklu//794Xz16tUtH/unP/1pOL/ssstaPjYwe2qPxR86dKg46+yM/6xEuz09PeFu7VH/2n6r\nNm3aFM7/9re/FWf79u0Ld2+55ZZwfvHFFxdn3d3d4e74+HhxVqsniH4Dtd9HTebY0Xkf61qEGneo\nAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEkP1UnukUceCedbt25t+dgf+tCHwnm2qwSY\nm6J/2+3t8f/Tox6h2jWjduypqanirNaPFfVFPf/88+HuX/7yl+JscHAw3L366qvD+RlnnFGcRZ1e\nTRN/nrXOpujYtQ6rWl9Y5vczl524Zw4AMEcIVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkh6q\nk8DmzZuLs29961vH70SAU16t3yjqg8p2EEX9SCMjI+HuSy+9VJz98Y9/DHc3bNhQnN10003h7pIl\nS8J59HnV+qCizzPTJZXpv6qpnddc7jd0hwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbcBJY\ns2ZNcTY8PNzycVetWhXO+/r6Wj42cOLKPLpeq1WI1B7XHx0dLc7Gx8fD3ahWYenSpeHuzTffXJzd\ncMMN4e5ZZ50Vzqempoqz2vcQ1SZEx22a+LPO1lucrHwqAABJAhUAQJJABQCQJFABACQJVAAASQIV\nAECSQAUAkKSH6hT3gQ98oDj7/e9/H+7qoQL+VzMzM8VZrVepNo86rmrdSWeeeWZxdt1114W7hw8f\nLs5WrlwZ7vb394fz6enpcB6JuqRqnV6ZrrHaZ5059lzmDhUAQJJABQCQJFABACQJVAAASQIVAECS\nQAUAkNQWPe45i+bkScEJ6uR8RpnQ/v37T7nraPT3rPaofrRbqxiI5pnXbZqm6erqann3WP19z9Zb\nnIgGBgaqb8odKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEiaqz1UAAAnDHeoAACSBCoA\ngCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoA\ngCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoA\ngCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoA\ngCSBCgAgSaACAEgSqAAAkjpn+wQKDs/2CcBJpG22T4Djb8+ePa6jc0BbW/zP7/BhX9OJYGhoqHod\ndYcKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEiaq7UJAHDcRPUGtWqDzG5NrXah1d1Dhw61fFze\nnTtUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECS2gQATgiZ+oJa/UA0n56ebvm8Ojo6Wn7dponr\nDWZmZsLd6Lzb2+P7KdF5195T5ns6kblDBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEl6\nqE4CP/vZz4qzkZGRcPfFF18szn74wx+2fE5N0zTf/OY3i7OPfvSj4e51112Xem3gxJPpVarNJycn\nw93R0dHibNeuXeHu+Ph4cTY8PBzuLly4MJz39fUVZz09PeFuV1dXS7OmaZoFCxa0dE5N0zS9vb3h\n/GTlDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkNR2+PDh2T6HdzMnT2q2fOlLXwrnP/jBD47T\nmRw9F154YTh//vnni7OBgYGjfTonu7bZPgGOvz179pxw19GpqalwfuDAgXC+b9++4mz79u3h7nPP\nPVec/f3vfw93d+7cWZx1dHSEu21t8T/PpUuXFmeXXnppuHv99dcXZytXrgx358+fX5zV6ivmzZsX\nzjs7y41Nhw4dCnejzFLbzRgaGqpeR92hAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgKRy\nGQTHVdQ1dSx7pi677LLi7FOf+lS4u3nz5nD+4x//uDjbuHFjuPvLX/6yOPv85z8f7gKzp9arFHUF\nTUxMhLt79+4N5//85z+Ls6jbrmnia1LUM9U0TXPOOecUZ+eee264+5///Cecv/HGG8XZ66+/Hu5e\nccUVxVmtK6q7uzucR2r9lu3t5Xs5tS6p6PdVe93o2LW+sCPhDhUAQJJABQCQJFABACQJVAAASQIV\nAECSQAUAkKQ24TjZsWNHOH/44YdbPvZVV11VnD355JPhbn9/f3FWe2x2ZmYmnG/ZsqU4+/Of/xzu\n7t69O5wDc1Pm0fXh4eFwd8+ePeF8+/btxVntenXppZcWZ5/73OfC3RtuuKE4W7RoUbhbuxZGfxum\np6fD3eg9j4yMhLvR99jb2xvu1v52RL+B2nuK5kej+iDDHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAk\ngQoAIEmgAgBI0kN1nNR6laLOj6hnqmma5g9/+ENxNn/+/PjEEh599NFw/sILL7R87FtvvbXlXeDY\ninqEolnTNM34+HhxVutGam+P7wGsWLGiOFu1alW4OzQ0VJxdeeWV4W5fX19xVuvO2rp1azh/9dVX\ni7PFixeHu3v37i3Odu7cGe6+5z3vKc46O+PoUPueor93tS6pqONqamoq3I20tbW1vPtf7lABACQJ\nVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkh6q4+Tyyy8P51FPVdS70TRxB8qx9PDDD4fzycnJ\n43QmwP8q6gqqdUl1dXUVZ7UuoGj3tNNOC3fnzZsXzk8//fTirHadjHZr1+CZmZni7M033wx3165d\nG86jbq7BwcFwd9myZcXZeeedF+5mfh/T09PhPNqvdVhFfVG1Dqvoezoa3KECAEgSqAAAkgQqAIAk\ngQoAIEmgAgBIEqgAAJLUJswRAwMDs30K/5/HHnssnG/YsKHlY994443hvPZIL5ATPX5ee3Q9mke1\nCLV5Z2f8J2l8fDycR++pVrkQVTbUagCGh4eLs7/+9a/h7vbt28P56OhocVa7Tp599tnFWe07jqoi\nDhw40PJu08S/gVrdTnTsWp1D5PDhwy3v/pc7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJCkh+oU9/LLLxdnX/ziF8PdiYmJcL506dLi7IEHHgh3a102QE7U2RP1OTVN03R0dLT8utFurYeq\np6cnnEedTbVjR59HrRtp/fr1xdmaNWvC3R07doTzCy+8sDi75JJLwt1ly5YVZ/39/eHuzMxMcVb7\n/mu/n6mpqeKst7c33M2IzksPFQDAHCBQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlNOMWtXbu2OKvV\nItTcc889xdkFF1yQOjaQEz0m3t4e/18784h59Mh8rdpgfHw8nEeP3E9PT4e7IyMjxdnmzZvD3aef\nfro427hxY7i7atWqcB5VI1x55ZXhblQzEdUiNE38eWVrbaL6grGxsXC3Vvcwm9yhAgBIEqgAAJIE\nKgCAJIEKACBJoAIASBKoAACSBCoAgCQ9VCe5u+++O5z/4he/aPnYX/3qV8P517/+9ZaPDcyeWs9U\n1CUVdQzVjl3rvuvo6AjnkUOHDoXz/fv3F2dPPfVUuBv1UNXOefXq1eH8mmuuKc6WLFkS7kZq33HU\nYVXbHR0dDedRx1XUJdY09f6sSPQbqP1uj4Q7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAktqE\nk8DBgweLs9/+9rfh7vj4eHFWeyT3/vvvD+fd3d3hHJg9R+Mx8XdTe6Q+uubUrhmdnfGfrGje3h7f\nP3jllVeKsxdeeCHc3blzZ3H24Q9/ONy9+uqrw/n5559fnNXqCaJ6i9r1Pfp97N27N9ytfdYDAwPF\nWVSp0DTx76tWjVE7ryx3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJD9VJ4Pbbby/O\n3n777ZaP+5WvfCWcDw0NtXxsYHbV+qIimQ6rqGuq1hPU0dERzqMOo23btoW7a9asKc5q19HTTjut\nOLv00kvD3cHBwZaPPTIyEu5GvVyTk5PhbnTssbGxcHfhwoXhPPoea11S0bz2m45+t5l/D//lDhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk6aE6Abz44ovh/Nlnn2352Lfddltxdu+997Z8\nXODElenzqfUIRV1TtR6q2vzgwYPF2bp168Ldf//738VZre/phhtuKM6uvfbacPess84K5zMzM8VZ\n7bPu7e0tzqKOqqZpmvHx8eKsv7+/5ddtmriH6sCBAy3vZvrRjgZ3qAAAkgQqAIAkgQoAIEmgAgBI\nEqgAAJIEKgCAJLUJc8TY2Fhxdt9994W7k5OTLb/uFVdcUZx1d3e3fFzg5BU9rj89PR3uRo/U1x7H\njyoEmqZpdu/eXZz94x//CHc3bdpUnM2bNy/cXb58eXFWq0VYsGBBOI+qIPr6+sLdaF77nqKKilot\nQq12Y+/evcXZ8PBwuDswMFCc1aogolqF2jkfCXeoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAk\ngQoAIEkP1Rzx0EMPFWdPP/10y8e9++67w/m9997b8rGBU1PUQ1XrAor6jSYmJsLdWnfSa6+9Vpy9\n9NJL4W7UBXjnnXeGu5/4xCeKs6g3qWnq7znqR6p1EPb09LQ0a5qmGRoaKs5qHYW172lkZKQ4i7qi\nmib+vGrnFf1ua697JNyhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS1CbMEffff/8xOe73vve9\ncF57zBQ49UTVBk3TNB0dHcXZzMxMuBvNx8fHw9233nornK9fv7442759e7h70UUXFWfXXHNNuDs4\nOFic1d5TVCHQNLn6gqhyoVbXEH3HBw8eDHdr39OuXbuKs1qdw4oVK4qz2m8vek9HgztUAABJAhUA\nQJJABQCQJFABACQJVAAASQIVAECSQAUAkKSH6iRX6wup9c0cK7WukUzPTa1fJTI2NhbOH3jggZaP\nHan1o0Q9ZV1dXUf7dDjFZa4LUfdR0zRNW1tbcRb1EzVN07z88svhfNu2bS0fe/ny5cXZ3r17w92X\nXnqpOBsdHQ13Fy5cGM4PHDhQnNWudfv27Wv5daPvccuWLeHuv/71r3Ae/b6uvvrqcDe6Rtd+t/Pm\nzSvOar/bI+EOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpDbhJLds2bLZPoV3dc8994TzM888\nszirPf784IMPtnROc1n0PX7hC184jmfCySKqL8jI1CYMDg6GuwMDA+E8qhDp7+8Pd6NH/X/zm9+E\nu9PT08XZ+Ph4uNvX1xfOo/MeGRkJdxcsWFCc9fb2hrvvvPNOcVarkag577zzirNapU70eXZ3d7d8\nTkeDO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpIdqjrjrrruKs0ceeeQ4nsnx8dBD\nD83K63Z2xj/5jo6Olo/92c9+Npxfe+21LR/7gx/8YMu78L/KdEnVuoCif4O1HqqLLroonG/durU4\ne+utt8Ld0dHR4uyZZ54Jd4eHh4uzsbGxcLf2eUU9VFHPVM2SJUvCefSeFi5cGO6uWLEinF944YXF\nWa2XK+qpqnVYHWvuUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAEBSW61vZJbMyZOaLT/5\nyU/C+eTk5DF53Q0bNoTzBx988Ji8btM0zde+9rXirNZxErnlllvC+eLFi1s+9hxWLg3ipLVnz56W\nr6O1Prbo70Z7e/z/9Gi31sk0MTERzqMeqk2bNoW7e/fuLc7efvvtcHfbtm3FWa2HqvZZHzp0qDjr\n6uoKd4eGhoqzWudX9D3WeqjOO++8cB5dZ2vdWgMDA8VZb29vuBv1p9UMDQ1Vl92hAgBIEqgAAJIE\nKgCAJIEKACBJoAIASBKoAACS1CbAyU9twikoU5tQe7w8mtf+pkS7tRqA2rGjioL+/v5wd3p6ujir\nVUHs2bOnpeM2TdPs27cvnEe1CYsWLQp3e3p6irPh4eFwN6rjmT9/frhb+/1E51Wrzoi+i0wtQo3a\nBACA40CgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSOmf7BACYW2p9T9E800M1Ojoa7kbdSE3T\nNOPj48XZ1NRUuNvR0VGc1TqsFi9eXJz19fWFu1F3Vk3U59Q0TTMzM1OcDQ0NhbvR51X7jqPurCOZ\nn6jcoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAktQmAHDURLUITRM/yl9Te1y/s7P8J629Pb5/\nEM1rr9vV1VWcRVUOTdM0IyMj4TxSqx+onXck+p6ytQnR93QiVyq4QwUAkCRQAQAkCVQAAEkCFQBA\nkkAFAJAkUAEAJAlUAABJeqgAOG46OjqKs1pXVLRbU+tOiua13QMHDhRn09PT4W6td6mnp6flY09N\nTYXzSPSea99D7Xs8kbumIu5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahMAmBOyj9PXHteP\nRDUBMzMzLR+3ra0tnHd3d4fz6DPJ1hdwdPm0AQCSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoA\nIKkt6t4AAKDOHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJ\noAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJ\noAIASBKVlhdKAAAANUlEQVSoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCApP8DjeUaj0e8wRsAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_cache_frozen.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tying weights" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is common to tie the weights of the encoder and the decoder (`weights_decoder = tf.transpose(weights_encoder)`). Unfortunately this makes it impossible (or very tricky) to use the `fully_connected()` function, so we need to build the Autoencoder manually:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28 * 28\n", + "n_hidden1 = 300\n", + "n_hidden2 = 150 # codings\n", + "n_hidden3 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "l2_reg = 0.0005\n", + "\n", + "activation = tf.nn.elu\n", + "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "\n", + "weights1_init = initializer([n_inputs, n_hidden1])\n", + "weights2_init = initializer([n_hidden1, n_hidden2])\n", + "\n", + "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n", + "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n", + "weights3 = tf.transpose(weights2, name=\"weights3\") # tied weights\n", + "weights4 = tf.transpose(weights1, name=\"weights4\") # tied weights\n", + "\n", + "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n", + "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n", + "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n", + "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n", + "\n", + "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n", + "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", + "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n", + "outputs = tf.matmul(hidden3, weights4) + biases4\n", + "\n", + "mse = tf.reduce_mean(tf.square(outputs - X))\n", + "reg_loss = regularizer(weights1) + regularizer(weights2)\n", + "loss = mse + reg_loss\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train MSE: 0.0155238\n", + "1 Train MSE: 0.0169992\n", + "2 Train MSE: 0.016667\n", + "3 Train MSE: 0.0176304\n", + "4 Train MSE: 0.0163978\n" + ] + } + ], + "source": [ + "n_epochs = 5\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = mnist.train.num_examples // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch})\n", + " mse_train = mse.eval(feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " saver.save(sess, \"my_model_tying_weights.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3FuMnWX5N+BnOvtOO9MdLdsCw7YgshMR9ID8TUiMiUaN\nB8qBBk00JppIogYTE/VEz5QTgkYj7g5MSNRoFBUJsSAgIWyEViwUtFjaUgqddjr7mf/BF02+Lz73\n02/dnc4wva7TO/e73vWu1TW/vsn761pYWCgAAHRu1VKfAADAm51ABQCQJFABACQJVAAASQIVAECS\nQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQFLPUp9AxcJSnwCsIF1LfQKcfAcO\nHPA7CifI5s2bm7+j7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEnLtTYBAE6ahYV6y0RXV/zE\nfDSfn58Pd7u7uzs+r5bM7mKKrtdyPefj4Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkNgGA\nFaFVbxCJHtefmpoKdwcGBqqzvr6+cHd2drbjeU9P/Cc8sxtdy9Z1blVFrFTuUAEAJAlUAABJAhUA\nQJJABQCQJFABACQJVAAASQIVAECSHioAloWoC+p45r29vdVZq+8p6mWKjts6r+7u7nA309k0PT0d\nzletqt8zaXVJzc3NdXTcUtrvKbperWNnusYWmztUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECS\n2gQATprosfdWLULrcfypqanqLFN90HpUP5q3qg2iuoZSShkcHOz4vKKqiOhalRLXF7SqDTL1F63r\nNTAw0PHrZr57x8MdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjSQwXACdPqioo6jFod\nRJmeofHx8XA302/U39/f0Tm1Xrcl6pkqJf4sWv1Xkdb1aPVUdXd3V2d9fX0dv3brWs/NzXW8ezzc\noQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkPVQAnDBRx1AppczMzFRnrR6q1vzYsWPV\n2eTkZLgb9TKNjIws2m6r06nVNRXp7e2tzlqfU3Re0WfYet1S4j6o1nlFu61rFfVjtT6H4+EOFQBA\nkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpDbhBHnkkUfC+R133BHOzzrrrOpscHAw3P3Yxz5WnW3Y\nsCHcbc2BU09XV1fHu61H6g8ePNjx7iuvvNLx/PXXXw93165dW50NDw+Hu9Gj/lu2bAl3oxqAUkoZ\nGxvr6HVLKWXTpk3hPDI/P1+d9ff3h7tr1qwJ5+vWravO+vr6wt3oPUfn3No9EdyhAgBIEqgAAJIE\nKgCAJIEKACBJoAIASBKoAACSBCoAgKSuhYWFpT6H/2ZZnlTkkksuCee7du06SWfyfxsZGQnn73jH\nO07SmSwP5513Xji//fbbw/nWrVtP4NmcNJ2XCvGmdeDAgfB3NNM1FXUnHT58ONzdv39/dbZnz55w\n94UXXgjnhw4dqs4mJibC3eg9jY+Pd7zbEl2PUuLerlbH1YUXXlidTU9Ph7vRfHR0NNxt/V259NJL\nq7OhoaFwN8osq1bF94iiHqpWFtq8eXPzH4w7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAUs9S\nn8BK8Ytf/CKcP/nkk+H88ssvr86effbZcPfRRx+tzn75y1+Gu7/73e+qs/PPPz/cffHFF8N5Rk9P\n/at5xhlnhLutR68jrVqFL33pSx0fG1aKqCag9fj5/Px8dfb666+Hu0ePHg3nmSqI6LwnJyfD3amp\nqepsdnY23B0bGwvnURVA9DtZSlxh8fLLL4e7O3bsqM6uvvrqcHfjxo3h/OKLL67OWnUOAwMD1Vmr\nNiH63rZ2j4c7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkh+oE2bZtW2oeeetb3xrO\nP/KRj1Rn3/zmN8Pdl156qTpr9VDt3r07nGf09fVVZ60equi8X3311XD30ksvjU8MTgGtLqlo3t/f\nH+6OjIxUZ+ecc064u2bNmnAe9VCtXr063D1y5Eh1Nj4+3vHrtjqsol6uUuLz3rBhQ7i7f//+6uz+\n++8Pd6Pzjnq3Sml/B6Lf9+7u7nA30xcV7ba+88d1/PQRAABOcQIVAECSQAUAkCRQAQAkCVQAAEkC\nFQBAkkAFAJCkh2qFGxgYCOeZ3qVMt1bGo48+Gs4PHjxYnV1//fXh7s0339zROcFyE3UjtbS6fqIe\noZbBwcHq7IILLgh3W11Bc3Nz1Vmrhyp6z63epWi39Tm05sPDw9VZq7Np586d1dmf//zncDfq/Dr9\n9NPD3fPOOy+cZ3rMFkvm38u/uUMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTWBZGh8fr84+\n8IEPhLvz8/PV2be//e1wN3qkG1aS6NH1TD1B61H+qAYg+rfbet1S4kfuW/+2o8fmM4/Uz87OhvNW\ntU103tHvZCmlvPLKK9XZ0aNHw93169dXZ616i1atQvSeWpUcUYVF6/vTqgPJcocKACBJoAIASBKo\nAACSBCoAgCSBCgAgSaACAEgSqAAAkvRQsSzdfffd1dm+ffvC3Y0bN1Zn5557bqenBG8qrS6pzG6m\nwyrqClq9enW429vbG86jzqeenvjPXXTerS6pqA+q1avUul4TExPV2Z49e8LdJ554ojo7ePBguHvh\nhRdWZ1dddVW4u2HDhnAedW+1Or+i6zk5ORnuRtc60zX2b+5QAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJAlUAABJAhUAQJIeKpbECy+8EM5vu+22jo/98MMPV2enn356x8eFU0Wrk2fVqvr/xVu9SlGXVH9/\nf7jb3d0dzmdmZqqzVpdUND906FC4Ozc3V50NDQ2Fu633FHVc/f73vw93o9/CqampcPeaa66pzrZt\n2xbutj7HSNRT1pq3rmX0OemhAgBYBgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJLUJLIlf/epX4Tx6\n/PnDH/5wuDs6OtrROQHHJ/OIeU9P/c9OVMdQSrv6IKpsOHbsWLh75MiR6iz6PSqllOHh4epsZGQk\n3G1dy71791Znjz/+eLi7c+fO6uzaa68Nd6+88srqbMuWLeHu6tWrw3lUX9D6DkTz6LjHc+wsd6gA\nAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSQ8Viybqbvn5z38e7vb391dn3/jGN8Ld7u7u\n+MSAlKjPp9UVNT8/X51NT0+Hu5OTkx0fe2pqKtyN+qCinqlSStm0aVN1Fv2WlVLK4cOHw/lzzz1X\nnR04cCDcPf3006uz66+/Pty9/PLLq7Nst1b0HYk+w1Li3/fWbvS9jTrMjpc7VAAASQIVAECSQAUA\nkCRQAQAkCVQAAEkCFQBAktoEFs33v//96mz79u3h7kc/+tHqbHR0tONzAtpaj71ndqN5q9qgVYkS\nPTbf0xP/uYseqV+7dm3HuxMTE+HuM888E84feuih6uyNN94Id6+55prq7IYbbgh3t27dWp319fWF\nu636i6g2obe3t+Pd6HMoJf5+ZL7z/3n99BEAAE5xAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUA\nkKSHio49+eST4fyzn/1sdbZu3bpw9+tf/3pH5wTkLSwshPOoz6e1G/UytbqAWl1SkVZHUXTerfcU\n9Wft2rUr3L3nnnvC+V/+8pfq7OKLLw533/3ud1dnl112Wbgb9UG1eqZmZmbC+dzcXHUWfbdKWdyO\ntCx3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJDxVVUV9MKaV85CMfCedR18gtt9wS\n7o6OjoZzYPFkeqhafU/Hjh2rzvr7+8Pd7u7ucB795rSO3dfXV521rsfBgwers/vuuy/c/fWvf93x\neV177bXh7o033lidbdiwIdyNurVmZ2fD3bGxsY6PPTAwEO5G36+oO6u12/qMj4c7VAAASQIVAECS\nQAUAkCRQAQAkCVQAAEkCFQBAktqEU1z0+PN73/vecPe5554L59u2bavOvva1r8UnBiyZVvVB9Nh7\nT0/8ZyWqAejq6ur4dVuvPTg4GO5GlQzT09Ph7u7du6uz7du3d7xbSik33XRTdXbllVeGu6eddlo4\nj0TVCK1KnZmZmXAeVRRE1Rctre/tYnOHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJL0\nUJ3iDh06VJ098MADqWP/+Mc/rs42bNiQOjaQE3U+tfqg+vv7O37dqCuot7c33I26kUqJu6Ra/VjR\neY2NjYW7TzzxRHX29NNPh7urV68O51dffXV1dsUVV4S7AwMD1Vmrsynqg2r1ULWOHX1O0ayUuMes\nJeq/an3nj4c7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAktqEFe7w4cPh/B3veEfHx/7JT34S\nzqPHfYGlFT1CHs1KiR+pb1UqRMeOHvMvpZSpqalwHj2uPzMzE+5GVQBPPfVUuPvoo49WZ0eOHAl3\nr7rqqnD+nve8pzrbsmVLuBtVEExPT4e7UUVF6zNuVR9E35/Wdy9T95H5zh8Pd6gAAJIEKgCAJIEK\nACBJoAIASBKoAACSBCoAgCSBCgAgSQ/VCveDH/wgnO/evbvjY7/rXe8K561OEODNKfq3PT8/H+4O\nDg4uyuuWEncJHTx4MNzds2dPdRb1TJVSyo4dO6qz4eHhcPe6664L51u3bq3OWtc60ur0io7d19cX\n7rY6naK+sFbHVdSPdSK6pDLcoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkPVQrwK5d\nu6qzr371qyfvRIBTQtQj1OpGmpmZ6fh1W7sTExPV2eHDh8Pdp59+ujp78cUXw92RkZHq7Kqrrgp3\nb7jhhnAe9TJNT0+Hu5G5ublwHl3r1m70/WjNW/1YURfZUncfukMFAJAkUAEAJAlUAABJAhUAQJJA\nBQCQJFABACSpTVgBtm/fXp2NjY11fNxt27aF88HBwY6PDbx5ZR5dX1hYqM5mZ2fD3UwlQ6tiIKon\nOOecc8LdNWvWdLwbVS6UEl+T1nuKPovMtYyu1fHo7u7ueDf6/iw1d6gAAJIEKgCAJIEKACBJoAIA\nSBKoAACSBCoAgCSBCgAgSQ/VKe7GG2+szv7whz+Eu3qogP9Xpieo1WHV19cXzqM+qNax3/KWt1Rn\nGzduDHejXqX169eHu1u2bOn42OPj4+FuT0/9T/yqVfH9lKhrqvUZt3qmlnOXVIY7VAAASQIVAECS\nQAUAkCRQAQAkCVQAAEkCFQBAUtcyfXxxWZ4UvEnFz4uzIh04cGBZ/o7Oz88v2rFb1QiRmZmZ6mxi\nYiLcHRoaqs4Ws0Jgbm4unEfXo3VeGZnPYbnavHlz8025QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAk\nUAEAJAlUAABJy7WHCgDgTcMdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQq\nAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQq\nAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQq\nAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgKSepT6BioWlPgFYQbqW+gQ4\n+Xbv3u13FE6Q0dHR5u+oO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQNJyrU0A4BTT1RU/mb6w\nEDdBtPY7PfZiHbeUUrq7uzven5+f7+icltpiXs+l5A4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJCkNgGAZWExH4nPVDJMT0+Hu/39/dXZqlXxfYu5ublwHtUqtK5XNO/pWbw//7Ozs+G8dU0i0ee4\n1JUK7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkh6qFeCnP/1pdTY+Ph7uPv7449XZ\nd7/73Y7PqZRSvvKVr1Rn//M//xPu3nTTTanXBhZPpgsos9vqbIr2JyYmwt2pqanqbGZmJtw9cuRI\ndRZ1VJVSyurVq8P5/Px8x+cV9UGtXbs23I0+p4GBgXC3r6+v42NnOqqW2pv3zAEAlgmBCgAgSaAC\nAEgSqAAAkgQqAIAkgQoAIKmr9ZjqElmWJ7VUPvOZz4Tz73znOyfpTE6cyy67LJw/+OCD1dnIyMiJ\nPp2Vrv6MMivW7t27O/4djR5rLyWuJ2jtRvOoIqCUUsbGxsL5a6+9Vp3t3r073H3ppZeqs71794a7\nUW3CsWPHwt1WTcDg4GB1dvbZZ4e727Ztq84uvfTScHd4eLg6y1ZBRPUXrSqIViVDp1pZaHR0tPk7\n6g4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJPUs9Qnwf0RdU4vZM3X11VdXZx/60IfC\n3V27doXzH/7wh9XZjh07wt177rmnOvvEJz4R7gI5rU6eqEuq1UMVOXr0aDjft29fOH/mmWeqs7/+\n9a/h7ssvv1ydHThwINyN+rNavUpR31PW5s2bq7PW5xR1Sa1ZsybcnZiYCOfRa0e9W6WUMjs7W521\nvrfR59Td3R3uHg93qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJLUJJ8k///nPcP69732v42Nf\nd9111dm9994b7kaPxvb19YW7c3Nz4fz555+vzh566KFw9+DBg+EcWDqtx9MjUY1A9Fh7Ke1ahcOH\nD1dnrXPesmVLdXbjjTeGu+edd1511vqdbNXPPPnkk9VZq55gfHy8Omtd66GhoepsYGAg3F21Kr5X\n8/rrr1dnY2NjHR+7dV6ZSo/j4Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJOmhOkla\nvUpRR0rUM1VKKffdd191tmbNmvjEEu6+++5w/thjj3V87Pe///0d7wI5rb6e6Peq1bsUzaemplLn\ntXHjxo5mpZSybdu26uzyyy8Pd6NupH379oW7e/bsCefHjh2rzv71r3+Fu+ecc0519sorr4S769ev\nr842bdoU7rb+3kVdZNPT0+FuT089tkSzUkrp7u7uaHa83KECAEgSqAAAkgQqAIAkgQoAIEmgAgBI\nEqgAAJIEKgCAJD1UJ8k111wTzqPejr6+vnB3cHCwo3PK+t73vhfOW30iwOLJdEm1djs9bkurC2hk\nZCScDw8PV2fr1q0Ldy+++OKOX/fw4cPVWatnaseOHeH8ueeeq86i91tKKatXr67Oent7w935+fnq\nLNsXNjExUZ1FnV6t82rtRt+vzHf+P6+fPgIAwClOoAIASBKoAACSBCoAgCSBCgAgSaACAEhSm7BM\ntB7LXQo//vGPw/lTTz3V8bFvvvnmcH7BBRd0fGwgV1+wmNUH0Xx2djbcXbt2bTiPHpvfvHlzuDs0\nNFSdtWoComqDe++9N9zdtWtXOI/Ou1XH8/a3v70627ZtW7jb01OPB1HtQSnt+oJMRUF/f391Fp1z\nKe3zynKHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJL0UJ3innjiiersU5/6VLjb6mY5\n44wzqrM77rgj3O3t7Q3nQE6mCyjabXVYRV1ArQ6rwcHBcL569erqbP369eHu/Px8dbZv375w9/77\n76/Onn766XC39Tt65ZVXVmfXX399uHvFFVdUZxs2bAh3jxw5Up21+sL6+vrC+bp166qz1ncgkumZ\nynSv/ef100cAADjFCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahNOcQ8//HB11nqct+XTn/50dXbx\nxRenjg3kRI+JtyoV5ubmqrOenvjPSrTbemS+v78/nA8NDVVnUS1CKaW88cYb1dljjz0W7j755JMd\nHbeUUoaHh8P5RRddVJ1dd9114W5UjRB9DqWUMjY2Vp216gkGBgbCefT9alUyROfdOq/WdyDLHSoA\ngCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBI0kO1wt16663h/Gc/+1nHx/785z8fzr/4xS92\nfGxg6UQdVaXEPUKt3agrqLe3N9xt9VS1eogie/furc7+9Kc/hbt79uypzlrX45JLLgnno6Oj1dm6\ndevC3cnJyers0KFD4e7ExER1NjIyEu5GfWClxH1QUf9VKaUcO3asOmt9/q3vT5Y7VAAASQIVAECS\nQAUAkCRQAQAkCVQAAEkCFQBAktqEFeDo0aPV2W9/+9twN3qsdsuWLeHul7/85XDe19cXzoHlKapF\nKCV+PL316Hr0yPzAwEC423ocf2ZmpjprPY7/yCOPVGf79+8Pd/v7+6uzc889N9w955xzwvmmTZvC\neeSNN96ozsbHx8PdtWvXdjQrpV1/EVUytGomenrqsaX1vY2O3do9Hu5QAQAkCVQAAEkCFQBAkkAF\nAJAkUAEAJAlUAABJAhUAQJIeqhXgwx/+cHV24MCBjo/7uc99Lpxv2LCh42MDSyvq3Wl1AUVdUpnX\nbfUbdXd3h/MjR45UZzt37gx3n3322Y6OW0rcQ7V169Zw9/zzzw/no6Oj1VnUyVRK3PfUEh07er/H\nI+oLa51z9P1pXY+5ubnqrPWdPx7uUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSHqo3\ngccffzycP/DAAx0f+4Mf/GB1dtttt3V8XGDlijp7Wl1Aq1evrs4GBgbC3ahHqJRSXn311erswQcf\nDHd/85vfVGfROZdSyjXXXFOdXXvtteHu2972tnB+xhlnVGdjY2Phbl9fX3XWek/RvNUHdvjw4XA+\nPj5enbX6oKJ56/sRnXdr93i4QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlNWCYmJiaqs9tv\nvz3cnZ6e7vh1o0d6o0dugZWrq6srnK9aVf+/+Pz8fLgb1Sq0dqPfyVJK2bNnT3X2xBNPhLuHDh2q\nztauXRvunnfeedXZRRddFO5u2rQpnM/MzFRnk5OT4e6aNWuqs1ZtQlQxENUelBJfy1JKefnll6uz\n0047LdwdGhqqzlrVB63vdZY7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkh2qZuOuu\nu6qzP/7xjx0f99Zbbw3nt912W8fHBt68FhYWqrNMX09vb284j7qCZmdnw90DBw6E88cff7w627t3\nb7g7PDxcnV122WXh7iWXXFKdnXXWWeFu61rv37+/Ojt27Fi4OzIyUp21rnWkdS1feOGFcH706NHq\nLPocSol7qKLv9PHMs9yhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS1CYsE1/+8pcX5bjf+ta3\nwnlfX9+ivC6wckXVB/Pz8+Hu9PR0dXbo0KFw9+9//3s4/9vf/tbxsdesWVOdbd26Ndw999xzq7NW\nLcI//vGPcL5v377qbNWq+J5Id3d3dRZ9DqXE1+vBBx8Md6NahFJK2bJlS3X22muvhbvr16+vzlrf\nvUimKuTf3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJD1UK1yrD6TVY7JY+vv7w3nU\nnxJ14JRSytTUVEfnVEopExMT4fyOO+7o+NiR6P2WEveU9fb2nujTgdDs7GxHs1JKmZmZqc5effXV\ncHfnzp3h/Omnn67ODhw4EO5GXVKt34V//vOfHc1KKWX//v3hfHJysjpr9QhGXVLHjh0Ld59//vnq\nbNeuXeHuwMBAx/ONGzeGu5HW72jUU3Ui/ha6QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlN\nWOHOOuuspT6F/+rTn/50OD/zzDOrs3379oW7d955Z0fntJxFn+MnP/nJk3gmnAoWFhbCeVRd0nr8\nPDp2V1dXuNt61L+np/4nbXp6OtyN6gu2b98e7kbVCK3KhZaoYmZ4eDjcjWoC9u7dG+4ePHiwOmvV\nE7z1rW8N59HvWfQZZrW+X1nuUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSHqpl4pZb\nbqnOfvCDH5zEMzk57rrrriV53VbHSatfJfLxj388nN9www0dH/ud73xnx7vw/6u3tzecR/1Gra6f\n6N9Yq1dp69at4fzGG2+szqJzLqWU1157rTr7xz/+Ee4+9dRTHb9u6zdnZGSkOtuyZUu4G30WrWu9\nfv366mzDhg3h7hVXXBHOL7jgguqsr68v3G31nC3W7nEdf1GPDgBwChCoAACSBCoAgCSBCgAgSaAC\nAEgSqAAAkgQqAICkroWFhaU+h/9mWZ7UUvnRj34UzqenpxfldaNulVJKufPOOxfldUsp5Qtf+EJ1\nduGFF3Z83Pe9733hfPPmzR0fexmLi4FYkXbv3r1ov6PR343BwcFwN+qCa3U2HT58OJz/61//qs5a\nv2evvPJKdbZz585wd8eOHeE8Mjc3F85HR0ers7PPPjvcja7npk2bwt2op+rMM88Mdy+99NJwvm7d\nunAeGRoaqs5aPVOtjrTI6Ohoc9kdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSW0CrHxqE05B\ni1mbEIlqEUqJH8dv/T1qzXt7e6uzVj1BVMkwMTER7u7Zs6ejWSnt2puoCqBVURFVI0SfQymlrF27\ntqNzKiX3OXV3d4e7re9Xp1rnrDYBAOAkEKgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgKTFKXQA\n4JQ0Pz8fzo8ePVqdRf1ExyPqKFqzZk24OzAwUJ21Ooq2bt1anc3MzIS7rX6sycnJ6mx2djbcja5H\n63OKzqu1u5j9ltGxu7riqqjF7t10hwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbAMAJ03qk\nfmpqqjpr1QCsWhXfA4ge9T948GC4Ozg4WJ213lP0uP7Q0FC4OzExEc67u7urs/7+/nA3uh6tioHW\ne460jh35j9IkAAABf0lEQVTNW9UG0XyxaxFa3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIE\nKgCAJD1UAJw0rS6pSNSrVEopMzMzHe8eOXKkOmv1KkVdUZOTkx3vlhL3dvX29oa7Ua9X6z1FnU6t\nz7DVB5XpuFrO3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJLUJgBwwrQex4+0HrdvzTOVDNHu\n9PR0uBtVH2SuR0u2kmGxtD6n6Jq0dpczd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAg\nqevN3PkAALAcuEMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk\nCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk\nCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkPS/d0Vq\n3wwaqEsAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_tying_weights.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Unsupervised pretraining" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28 * 28\n", + "n_hidden1 = 300\n", + "n_hidden2 = 150\n", + "n_outputs = 10\n", + "\n", + "learning_rate = 0.01\n", + "l2_reg = 0.0005\n", + "\n", + "activation = tf.nn.elu\n", + "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "y = tf.placeholder(tf.int32, shape=[None])\n", + "\n", + "weights1_init = initializer([n_inputs, n_hidden1])\n", + "weights2_init = initializer([n_hidden1, n_hidden2])\n", + "weights3_init = initializer([n_hidden2, n_hidden3])\n", + "\n", + "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n", + "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n", + "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n", + "\n", + "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n", + "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n", + "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n", + "\n", + "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n", + "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", + "logits = tf.matmul(hidden2, weights3) + biases3\n", + "\n", + "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y)\n", + "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n", + "loss = cross_entropy + reg_loss\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "correct = tf.nn.in_top_k(logits, y, 1)\n", + "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n", + "\n", + "init = tf.initialize_all_variables()\n", + "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Regular training (without pretraining):" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train accuracy: 0.94 Test accuracy: 0.9256\n", + "1 Train accuracy: 0.966667 Test accuracy: 0.9145\n", + "2 Train accuracy: 0.926667 Test accuracy: 0.9375\n", + "3 Train accuracy: 0.98 Test accuracy: 0.9487\n" + ] + } + ], + "source": [ + "n_epochs = 4\n", + "batch_size = 150\n", + "n_labeled_instances = 20000\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = n_labeled_instances // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " indices = rnd.permutation(n_labeled_instances)[:batch_size]\n", + " X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n", + " saver.save(sess, \"my_model_supervised.ckpt\")\n", + " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", + " print(\"Test accuracy:\", accuracy_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now reusing the first two layers of the autoencoder we pretrained:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train accuracy: 0.94\tTest accuracy: 0.9176\n", + "1 Train accuracy: 0.966667\tTest accuracy: 0.9457\n", + "2 Train accuracy: 0.98\tTest accuracy: 0.9456\n", + "3 Train accuracy: 0.98\tTest accuracy: 0.9416\n" + ] + } + ], + "source": [ + "n_epochs = 4\n", + "batch_size = 150\n", + "n_labeled_instances = 20000\n", + "\n", + "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3]) # Freeze layers 1 and 2 (optional)\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " pretrain_saver.restore(sess, \"my_model_cache_frozen.ckpt\")\n", + " for epoch in range(n_epochs):\n", + " n_batches = n_labeled_instances // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " indices = rnd.permutation(n_labeled_instances)[:batch_size]\n", + " X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n", + " sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n", + " accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n", + " saver.save(sess, \"my_model_supervised_pretrained.ckpt\")\n", + " accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n", + " print(\"Test accuracy:\", accuracy_val)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Stacked denoising Autoencoder" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "from tensorflow.contrib.layers import dropout\n", + "\n", + "n_inputs = 28 * 28\n", + "n_hidden1 = 300\n", + "n_hidden2 = 150 # codings\n", + "n_hidden3 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "l2_reg = 0.00001\n", + "keep_prob = 0.7\n", + "\n", + "activation = tf.nn.elu\n", + "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "is_training = tf.placeholder_with_default(False, shape=(), name='is_training')\n", + "\n", + "X_drop = dropout(X, keep_prob, is_training=is_training)\n", + "\n", + "weights1_init = initializer([n_inputs, n_hidden1])\n", + "weights2_init = initializer([n_hidden1, n_hidden2])\n", + "\n", + "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n", + "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n", + "weights3 = tf.transpose(weights2, name=\"weights3\") # tied weights\n", + "weights4 = tf.transpose(weights1, name=\"weights4\") # tied weights\n", + "\n", + "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n", + "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n", + "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n", + "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n", + "\n", + "hidden1 = activation(tf.matmul(X_drop, weights1) + biases1)\n", + "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", + "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n", + "outputs = tf.matmul(hidden3, weights4) + biases4\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "mse = tf.reduce_mean(tf.square(outputs - X))\n", + "reg_loss = regularizer(weights1) + regularizer(weights2)\n", + "loss = mse + reg_loss\n", + "training_op = optimizer.minimize(loss)\n", + " \n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train MSE: 0.0202949\n", + "1 Train MSE: 0.0140252\n", + "2 Train MSE: 0.0110557\n", + "3 Train MSE: 0.0100771\n", + "4 Train MSE: 0.00952982\n", + "5 Train MSE: 0.00892205\n", + "6 Train MSE: 0.00884232\n", + "7 Train MSE: 0.00900632\n", + "8 Train MSE: 0.00832438\n", + "9 Train MSE: 0.00875768\n" + ] + } + ], + "source": [ + "n_epochs = 10\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = mnist.train.num_examples // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch, is_training: True})\n", + " mse_train = mse.eval(feed_dict={X: X_batch, is_training: False})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n", + " saver.save(sess, \"my_model_stacked_denoising.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3FusXWXZNuB3dW26Nt23UJBQoAiFatgGAQUEVBANGiXE\nIAcaMJGYaCKJGkhM1BM9U04IGgwgEqPRaMMBEhHRSgChSiMgUqkNFGyAtpTStd/8B9//HXzG93nr\nfLq6VtvrOn3yjDnmmLNz3R3JuLtmZmYKAACdWzDXJwAAcKgTqAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjqmesTqJiZ6xOAw0jXXJ8AB9+uXbv8\njsIBsmLFiubvqDtUAABJAhUAQJJABQCQJFABACQJVAAASQIVAEDSfK1NAIB5YcGC+N7DzEy9oSKa\nzWfRe269p0P1PWe5QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlNAOCw16o+mK1jd3V1pY49\nNjbW8W50XtPT0+Hu1NRUR8ctpZTu7u74xAKt85rP3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCAJD1UAMwLrc6mmZmZ1Hy2dqPepdZ7avUuRcceHx8Pd6MuqYULF4a7md6u1rWMrknrdaNj\nZz7DA8EdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSW0CAAdN9Fh85pH5UkqZnJyszlr1BdE8\nU33QqkWIqg1KiWsT+vv7w93R0dHqbGJiItyNahWicyqlXecQfc6Z69Ey27UK7lABACQJVAAASQIV\nAECSQAUAkCRQAQAkCVQAAEkCFQBAkh4qAA6Y2ex7inqmSok7inp64j93Uf9R67wy3VqtefSeW7vR\nebf6saJ5q88p09s1NjYW7g4ODobzueQOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACTp\noQLg/2j1CEVaHUWRffv2hfPh4eFwHnVJLV68ONzt7++vzlodVn19fdVZpu+plFJ6e3urs+icSyll\nfHy8OouuVSmlDAwMVGet/qvW5zgxMVGdzWZf2GxzhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIA\nSFKbcIA8/vjj4fy2224L58cdd1x1Fj2+Wkopn/nMZ6qzFStWhLutOcC/ix5db9UmvPHGG9XZ3/72\nt3D3z3/+cziPKgiOPfbYcDeqJzj66KPD3WjeepQ/et3WflSLUEr8OU1OToa7ixYtqs5WrVoV7rbe\n0+DgYHXWul7RvFW5EH0/WvUV+8MdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjqanWG\nzJF5eVKRdevWhfMtW7YcpDP5v5YuXRrOL7jggoN0JvPDiSeeGM5vueWWcL5mzZoDeDYHTVzswmFp\n165d4e9o1OfT6gKKDA8Ph/OtW7dWZw8//HC4u3nz5nC+Y8eO6mz58uXhbnTerX6j7u7u6mxsbCzc\nXbhwYTiPuqZGR0fD3ej3v6+vL9xduXJldbZkyZJw9/LLLw/n5513XnUWXctSSlmwoH4fqPU5RXmn\n1UO1YsWK5j8Kd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgKT4GUP2269+9atw/vTTT4fzd73r\nXdXZs88+G+4+8cQT1dmGDRvC3QcffLA6O+mkk8Ldf/7zn+E8I3r89dhjjw13X3755Y5ft1Wr8LWv\nfa3jY8N8Ej1CPjU1Fe729vZWZ61H14866qjq7D3veU+426pzeOutt6qzVj3BSy+9VJ1FdQwtrbqG\nXbt2hfPdu3dXZ61rHb12q97imWeeqc5adTytGqHocx4YGAh3oxqJltmuiXKHCgAgSaACAEgSqAAA\nkgQqAIAkgQoAIEmgAgBIEqgAAJL0UB0gp59+emoeOeOMM8L5ddddV5195zvfCXe3bdtWnbV6qLZu\n3RrOM/r6+qqzVg9VdN6vv/56uHvaaafFJwaHiFbnTjRv7XZ3d1dng4OD4W7UM9TqNzrllFPC+eTk\nZHUW9TmVUsqePXuqs+j9ltLu7YqMjIyE84mJieqs1a21ZMmS6uz3v/99uPvCCy9UZ63PuPUb3eqa\nmi1Rj9mB6KhyhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS9FAd5vr7+8N5pncp062V\n8cQTT4TzN954ozo7//zzw90rrriio3OC+WbBgvj/y5kequnp6Y53o/Nq9VAtWrQonEevvXr16nC3\n1TUVyfQbDQ8Ph/Ooh2rZsmXh7s6dO6uzDRs2hLujo6PV2fHHHx/urlmzJpxnOp+i6zHX3KECAEgS\nqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJLUJjAv7du3rzr7xCc+Ee5Gj3R/73vfC3cHBgbiE4NDRObR\n9KgGoJT431hrd2pqqqPj7o/e3t7qrFWLENU5ROdcSnytW7ut6xVVI/T19YW7L730UnX2zDPPhLvR\n9Vq/fn2426pzaL3nSPQ5tb7zmdfdH+5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIe\nKualu+++uzrbsWNHuLty5crq7IQTTuj0lID/L+pWmpiYCHejrqlWV9Tg4GA4j3qIWseOzrvVbxRd\nj8nJyXC3pyf+Mxx1a+3Zsyfcfeihh6qzLVu2hLtnnnlmdXbppZeGu6tWrQrnmS6pqHur1WOW6Wbb\nH+5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIeKubEiy++GM5vvvnmjo/92GOPVWfH\nHHNMx8cF/kfU99PqAor6njKdTK1jt86rq6urOmv1F0XHjo5bSq5b66mnngp3N27cWJ1FfU6llHLV\nVVdVZ+vWrQt3W59jq6usU61rrYcKAGCeE6gAAJIEKgCAJIEKACBJoAIASBKoAACS1CYwJ+6///5w\nHj1We+2114a7a9eu7eicgP/Rerx8amqqOluwIP5/evRIfetx+1b1QWse6e/vr86i91tKfL26u7s7\n3i2llN27d1dnjz76aLj7/PPPV2cf/OAHw90PfOAD1dnixYvD3ZGRkXA+OTlZnbWuV6T1ObVqFbLc\noQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkPVTMmqhL6pe//GW4u3Dhwurs29/+drib\n6TEB2n090bzVBRT1PfX29oa7UX9Raz40NBTuRn1Qra6o6Leur68v3G1drz/96U/V2UMPPRTurl69\nujr70Ic+FO4eddRR1dn4+Hi425pH35/WbtRV1vreRj1lrf60/eEOFQBAkkAFAJAkUAEAJAlUAABJ\nAhUAQJJABQCQpDaBWfPDH/6wOtu4cWO4++lPf7o6W7t2bcfnBLRlahNa1QfRY+/RrJR2bUL06PvY\n2Fi4G53322+/3fHrtq7H9u3bw/mGDRuqs5deeincveaaa6qzCy+8MNyN3lPrc2iJvj9RZU5rt1VB\nMdvcoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAkPVR07Omnnw7nX/ziF6uzZcuWhbvf\n+ta3OjonYPZNT0/PynFbPUKteXRerW6t0dHR6izqZCqllMWLF1dnIyMj4W7UM1VKKY888kh1dtJJ\nJ4W7n/rUp6qzVatWhbuR7u7ucN7qE4s+i9a1jvrEWt/L6HVnZmbC3f3hDhUAQJJABQCQJFABACQJ\nVAAASQIVAECSQAUAkCRQAQAk6aGiqtWfct1114XzqDPm+uuvD3fXrl0bzoHZ0+p7avUQRWazCyjq\nKOrt7Q13o/c0NDQU7ka9S5s2bQp3f/GLX4Tzt99+uzr78Ic/HO5GPVWtXq7os8h+TtFrT0xMhLut\n72bkQHRNRdyhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS1CYc4aanp6uzj370o+Hu3//+93B+\n+umnV2ff/OY34xMD5syCBfH/taPHz1u7kVYdQ/R71Xrt1iPz0eP4rUf1X3vttersnnvuCXefeuqp\ncH7ZZZdVZ1dffXW4G9U9ZOoJopqIUtqfY/Q5jY+Pd7w7m3Uf+8MdKgCAJIEKACBJoAIASBKoAACS\nBCoAgCSBCgAgSaACAEjSQ3WE27VrV3X2yCOPpI597733VmcrVqxIHRuYO1EfVFdX16zttuZRR1Fr\nNzI2NhbOf/azn1Vn999/f7i7atWqcH7jjTdWZ+vWrQt3o66pVt9T1GHV6gNr9UFF+62+sEimA+1A\ncIcKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEhSm3CY27NnTzi/4IILOj72j3/843B+9tlnd3xs\nYP7q7u6eleO2Hplvzfv7+6uznp74z11UMbB58+Zw98EHH6zOWpUL1157bTi/8sorq7Po/ZYS1xcs\nWrQo3I0qCFq1CK2Kimi/VecQvedMNcaB4A4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJOmhOszddddd4Xzr1q0dH/uiiy4K53PdCQLMjujfdtRfVEopk5OT1dnIyEi42+qSypzX8PBwdfbY\nY4+Fu88//3x1tnz58nA36pkqpZQlS5ZUZ63f2KjTqa+vL9yNjt3qA4s+49axW91a0efYOq/Z5g4V\nAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJOmhOgxs2bKlOvvGN75x8E4EOCJEfT+tLqDu\n7u7qbGxsLNydnp4O5729vdXZ6OhouPvss89WZ5s3bw53t23bVp1ddtll4e4JJ5wQzqNOp9Z7ikQd\nVaXEPVWtTq/ocyillKmpqeqs1a01111TEXeoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAktQmH\ngY0bN1Znb731VsfHPf3008P5wMBAx8cGDk/RI/EtrUfmo8qFUuJahR07doS7DzzwQHX2+OOPh7s9\nPfU/pWeeeWa4u2rVqnAevafW9YqqERYuXBjuzma1Qat24VB1eL4rAICDSKACAEgSqAAAkgQqAIAk\ngQoAIEmgAgBIEqgAAJL0UB3h3vve91Znv/nNb8JdPVTAv2t1FEWiPqdSSpmYmOj42L29veE86t1r\ndWstXry4OrvkkktS5zU6OlqdRR1VpZTS19cXziOtY0cy34FDmTtUAABJAhUAQJJABQCQJFABACQJ\nVAAASQIVAEBS18zMzFyfw38yL08KDlFH5jPMR7hdu3b5Hf0vRPUErbqG/v7+6qxVi9D6G7xgwdzc\n95in2WDOrFixovk76g4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJM3XHioAgEOGO1QA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJAlUAABJPXN9AhUzc30CcBjpmusT4ODbuXOn39F5oKsr/uc3M+NjOhSs\nXLmy+TvqDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkDRfaxMA4KCJ6gta1QfRvFWLMJu1CdPT\n09VZT0/853/Bgs7vt0xNTXW8eyhzhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbAMBhIaog\naNUTTExMVGeTk5PhblRPEB13f85rYGCgOhsaGgp3x8fHq7PWexocHKzOuru7w91WzURUq9CqXIiO\n3Xrd2eYOFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACTpoToM3HfffdXZvn37wt1NmzZV\nZz/4wQ86PqdSSvn6179enV1++eXh7qWXXpp6bWB+irqCWp1MLaOjo9XZ7t27w93t27dXZ88991y4\nu3fv3urs1VdfDXdHRkbC+apVq6qzc845J9w9+eSTq7Pjjjsu3I0+p1ZXVNTLVUopY2Nj1dmCBfF9\nnlYH1lxyhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASOrKPqY6S+blSc2VL3zhC+H8+9///kE6\nkwNn/fr14fyPf/xjdbZ06dIDfTqHu/rzzxy2du7cOSe/o9Hj9qXEj8W3HscfHh4O5y+88EJ1FlXE\nlFLK7373u46OW0pc99CqEHjzzTfD+fLly6uz888/P9x9//vfX5195CMfCXePOuqo6iz7niYmJqqz\nwcHBcDe61j09cRNU67sZWblyZXPZHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIiksb\nOGiirqnZ7Jk6++yzq7Nrrrkm3N2yZUs4v+eee6qz5557Ltz9+c9/Xp3deOON4S4wu6I+n0y34fj4\neDjftWtXOI+6pO67776OzqmUUtauXRvOV69eXZ21rkfUyVRKKXv27KnOWn1QUZdUq+8pOq/W5zQ0\nNBTOJycnq7OxsbFwt6+vrzpr9UzN1vf2f7lDBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkqU04\nSF566aVwfuedd3Z87PPOO686+/Wvfx3uRo/ORo+nllLK1NRUOP/HP/5RnT366KPh7htvvBHOgdnT\nevw8s7tgQf3/8a0agNdffz2cv/DCC9XZiSeeGO6++93vrs4uvfTScPfUU0+tzlq/ky+++GI4f/jh\nh6uzqH6glFJ6e3urs56e+M//yMhIddb6nLq7uzuet+ocou9P67vX+iyy3KECAEgSqAAAkgQqAIAk\ngQoAIEmgAgBIEqgAAJIEKgCAJD1UB0mrV2lmZqY6i3qmSinloYceqs4WLVoUn1jC3XffHc6ffPLJ\njo/98Y9/vONdICf6PSol7hFqdRBFPUITExPhbuv3bP369dXZwMBAuHvllVdWZ6eddlq429/fX53t\n3bs33G11a+3atas6a/1defPNN6uzqGeqlFwXWfQZt46ded3W9zbqoWqd8/5whwoAIEmgAgBIEqgA\nAJIEKgCAJIEKACBJoAIASBKoAACS9FAdJOecc044j/pE+vr6wt1Wv8psufPOO8P5+Pj4QToT4L8V\ndfa0OnmiPp9WF1DUM9Ta7e3tDeeXXHJJdbZ27dpwd+XKldVZqxtpdHS0OtuxY0e4u3nz5nD+/PPP\nV2fHHntsuLt06dLqrNX5Fb3noaGhcLdlenq6Omv1Yy1cuLA6y3SgHQjuUAEAJAlUAABJAhUAQJJA\nBQCQJFABACQJVAAASWoT5ono8da5cu+994bz1uO+kSuuuCKcn3zyyR0fG8iZnJzseB5VKpRSSk9P\n/c9OqzZh+fLl4XzJkiUd70aiWoRSStm3b191tnXr1nB306ZN4TyqzTn//PPD3ZNOOqk6W7ZsWbgb\n1Re06ita34FIq9ogqlxoVUFEtQqtaoz94Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJOmhOsL95S9/qc4+//nPh7tjY2Ph/Nhjj63ObrvttnC31XMCzJ5WJ0/077O/v3/WXrcl6riKOoha\nu+Pj4+Huyy+/XJ09+OCD4e4rr7wSzs8666zqbP369eHuqlWrqrNW31P0WQwPD4e70bUsJe7Waok6\n0Fqf8WxzhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbcIR77LHHqrNWLULLTTfdVJ2deuqp\nqWMDOdFj8a1H6qenpzvejeZTU1Ph7tDQUDiP3lPrkfqJiYnq7MUXXwx3f/7zn1dn0W9sKaUMDAyE\n82OOOaY6i6ppSokrLGZmZsLdaD46OhruLlq0KJxH1Qctme9tNG9dj/3hDhUAQJJABQCQJFABACQJ\nVAAASQIVAECSQAUAkCRQAQAk6aE6zN1www3h/Kc//WnHx/7yl78czr/61a92fGxg7kQ9Uy1RT1BL\nq0eo1X/U01P/k9bX1xfu7ty5szp78MEHw90NGzZUZ61recUVV4TzE044oTpbs2ZNuDs4OFidtbqg\nok6w1ufUOnZvb291lukLa3VJRZ9F5nv7v9yhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS1CYc\nBt5+++3q7IEHHgh3o8eQV69eHe7eeuut4bz1mDIwP7UeIY/mrZqA6HH8Vi3C+Ph4OF++fHl1Njw8\nHO5u2bKlOnv44YfD3VdeeaU6u+qqq8Ldc889N5xffPHF1dkxxxwT7kb1BXv37g13I4sXLw7nIyMj\n4XxsbKw6a9Um9Pf3V2dqEwAADnECFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpIfqMHDttddW\nZ6+99lrHx/3Sl74UzlesWNHxsYFDV9QltWBB/P/0Vk9VpKcn/pO1e/fu6uytt94Kd//whz9UZ089\n9VR8YoETTzwxnK9duzacR32AUZ9TKXGvV+tzGhwcrM5aXVFR/1UpcZ/YwMBAuDsxMdHxeUVdU60O\nq/3hDhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk6aE6BGzatCmcP/LIIx0f+5Of/GR1\ndvPNN3d8XODQlenkaXUBRd1JCxcuDHejHqFSStm2bVt1tmXLlnD3ySefrM76+vrC3SuuuKI6u/ji\ni8PddevWhfOo06nVJRX1ULWuZW9vb3XW6hIbHh4O51GfWOtaR9+RkZGRcHe2uUMFAJAkUAEAJAlU\nAABJAhUAQJJABQCQJFABACSpTZgnosc9b7nllnB3fHy849c999xzq7PW46vA4an1SH00bz3KHz32\nPjQ0FO5GNQCllLJnz57q7NFHHw13n3766ers1FNPDXevvvrq6uycc84Jd5cvXx7OJyYmqrOoUqGU\n+Fq3PqfM67ZEtQnR65YSn3erziGat77z+8MdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaAC\nAEjSQzVP3HHHHdXZb3/7246Pe8MNN4Tzm2++ueNjA4enVidPd3d3dTYzMzNru62Ook2bNlVnrd/R\nwcHB6uziiy8Ody+66KLq7Kijjgp3WzL9SFEPVUvUFdXb2xvu7tu3L5xHfWKt/sOpqanqrNVDFV1L\nPVQAAPOAQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkNmGeuPXWW2fluN/97nfDeesRVYB/Fz1inqlN\naD3m/69//Sucv/LKK9XZG2+8Ee6ed955Hc1KKWXZsmXhPDI2NhbO33777eqsda0XLVpUnUWfQylx\nPUHrdUdGRsL59u3bq7Ph4eFwd/HixdXZmjVrwt0lS5aE8yx3qAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCAJIEKACBJD9VhLuowKaWUBQvmJlO3+maijpSoH6WUdq9LpNWfctttt3V87EirEybqKevt\n7T3Qp8MRLuqZKqWUiYmJjnd7eup/dqanp8PdVkfR1q1bq7N9+/aFu1FnU+u8/vrXv1Znrd+j1nzv\n3r3V2dDQULj7jne8ozrr7+8Pd/fs2VOd7dy5M9yNrkcppbz66qvVWetan3LKKdXZ0UcfHe4ODg5W\nZ9H3cn+5QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlNOMwdd9xxc30K/9FNN90UzqPHfXfs\n2BHu3n777R2d03wWfY6f+9znDuKZcCRoVZNENR+tKpZo3qpcaJ1XX19fdbZs2bJwN6pc+MlPfhLu\nRtUGu3fvDndb1QdRVcTq1avD3eOPP746e+utt8LdV155pTp7/fXXw93M92fNmjXh7plnnlmdTU5O\nhrut71eWO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQpIdqnrj++uurs7vuuusgnsnB\ncccdd8zJ6/b0xF/5qB+l5bOf/Ww4v/DCCzs+9vve976Od+FAGxsbq84WLlwY7k5PT1dn4+Pj4e6q\nVavC+VlnnVWdRV1RpZSybdu26mzjxo3hbtTpNDIyEu5G3VmlxNer1a21aNGi6qzVQzUwMFCdHXPM\nMeHu2rVrw/ny5curs3e+850dH3twcDDcbXWkZblDBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAk\nCVQAAEldMzMzc30O/8m8PKm58qMf/Sict7pbOrV58+Zwfvvtt8/K65ZSyle+8pXqrNVTEvnYxz4W\nzo8++uiOjz2Pdc31CXDw7dy5s+Pf0a6uzr8yra6f3t7e6qzVAzcxMRHOX3755ers2WefDXe3b99e\nne3cuTPcff3116uz1vVovaeoL6rVQzU0NFSdta710qVLq7Mzzjgj3F25cmU4j74DresVHbu/vz/c\njTq/Wt/5lStXNv9RuEMFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTYDDn9qEI1CmNqElesS8\n9dh7NG/tth5tjypkWjUBCxcu7Ph1W9UHkcnJyXC+b9++jo8d1QiMjo52fNyBgYGOd0tpv+dOtT6n\nTB2I2gQAgINAoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAknrm+gQAOLRMT0/Pym6rFzHqmSql\nlKmpqeqs1XEVdSMNDg6Gu1GHVav/qvWeh4aGOt6Nrkerhyrq1mr1ULWudW9vb3UWnXPLXPdqukMF\nAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACSpTQDggMlUKrQqBqLH7Usppaurq+PXjrz55pvhPDrv\nVoXA2NhYOI/qC6K6hpbouK1jt3b7+vrCeaY6Y7Y+4wPBHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAk\ngQoAIEmgAgBI0kMFwH8l6gJq9QhFWh1WrR6qnp76n7TWsfft2xfOO9XqmWrp7++vzlodV5HWtYyu\nV6svLNNFlumZmuuOKneoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAktQkAHDCtR9czlQuTk5Md\nnVMp7Uf5o3qClug9tSoGWu95to7d19fX8eu2ZKozDmXuUAEAJAlUAABJAhUAQJJABQCQJFABACQJ\nVAAASQKgdOMUAAAAsElEQVQVAEBS15HaFwEAcKC4QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJAlUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEA\nJAlUAABJAhUAQJJABQCQ9P8AJ2j6Mj44+mcAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_stacked_denoising.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Visualizing the extracted features" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_stacked_denoising.ckpt\")\n", + " weights1_val = weights1.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure extracted_features_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAB7CAYAAACCcQ/CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnVmvJlX1xh9EZVAUFUQQRSablqEZBBPpIIgECBD8GtzI\nF+DOj6CfgHBhoiGRG0wkJAxC04jN1NgMggLSzMqgTCpe/PNU/eqcVW/t7j60p+r//G5Opd5Tw7vX\n3vvd+1lrr33Ixx9/rBBCCCGEMM2n/tcvEEIIIYQwFzJwCiGEEEJoJAOnEEIIIYRGMnAKIYQQQmgk\nA6cQQgghhEYycAohhBBCaCQDpxBCCCGERjJwCiGEEEJoJAOnEEIIIYRGMnAKIYQQQmgkA6cQQggh\nhEYycAohhBBCaCQDpxBCCCGERjJwCiGEEEJoJAOnEEIIIYRGMnAKIYQQQmjk0//rF6i46aabPl71\n+ccf/9/HhxxyyCf6Hr6/n7eR8J6t34P/99Of/vST/fIbzM9+9rPuC++P/VptsFF1Yux5VZ2ontlq\nX/7fT37yk1nZ9LzzzjughrGqLPelzU1dM1Un9qdu+ZopO+/atWtWNr3ttttWFsYn2SdWzyEHWk/8\nv5/6VK0XrLoX3+e6666blU2ffPLJT9ZYjRysurM/bNmyZZ9sGsUphBBCCKGRTak4TeGRazUDnGLq\n/8ZmI6v+l+/xr3/9a93/VbOn/VFGNuNI/UA4UNVt6p7/+c9/JEmf/nRfzT/66KOme7Ie+D6SdOih\nh667plVBW5r9KvanTZLqGpc57//vf/+7O2f7jLXd6nM+p7Jf6+x47J5LpfqOUzav+muWW2VT96Nj\nNvU1rBu8vno3f74vyuT/B5tO0aoMT/12uh2OXVOVP/vefX1fstF2jOIUQgghhNDILBUnMzbT8TFH\nq57BUHHg54cffrgk6aijjurOffnLX173zFdffbU7/vvf/77uPb7yla90x0cccYSk4UzIz+ezP/zw\nw+rrLYp9iQmq/q+alVYzSCp+Vpp4jf+X5zjr+dznPidJ+vznP19+jw8++EDS0GY+95nPfKZ89+rc\nEmayUzE/U0rcKpuzLA877LDu2GXtv7wP2zaVCNeD6t0k6Z///Oe6a1wnPvvZz6583/1RTTczrerR\nvtzHuI9dy/vvvy9p2Ce+9957g7+S9PWvf707PvHEE9c95/XXX++O//rXv0oa2s/35zWsZ6vq6xLa\n6xRTv6fsJ91/TqlMvI/bV2UTqe9TXR/4HKm3FT0IUyrVqvp6IO01ilMIIYQQQiMZOIUQQgghNDIb\nV10liY9Jcquk1jHJ1S66r33ta905S8Nvv/12d+7RRx/tjnfv3i1JOv7447tz3/jGN7rjLVu2SJL+\n8Y9/dOf+9re/rbsnv0flSlr72dzZn2BqlxHdMS6jN998szvHzysZ2TIv5WJKv5aJ33333XXPkaS3\n3npL0tB9ZJcs4bMtUS/Ffq1UQcGkapO2xZFHHrnuPlLt1rbbjv9H+7j8aRO6+nyd3bQ8Vy324OeV\nTfcnoHWzMFVH9yd1g21B9/cXvvCFdf/LfvKNN96QNLSZ+1NJOuGEEyQN3Tpum5K0d+9eSUP7ffWr\nX133HlMBz1N1eKnw+1b12e2U/8e+13Zh3+ly5/+xrG0/214a9tNHH320pGF/W4VwTLnUV51rJYpT\nCCGEEEIjs1GcpoJtOcPksXEgN0eZxx13XHd8zDHHSJK+9a1vdec8Mn788ce7c08//XR37BnSfffd\n153785//3B1feOGFkoYqlq/hO3JW5RE2Z1JVUOPcmHr3qeWt1VJWY9tJQ6XilFNOkTQMAPWshoGq\nVDEeeOABSUNFkEqEFUPOhDz75bOrWWs105qzTVtnbJyx0qaVeuTP+RmvpyphrF5UaQsIr+X9PZOt\nFpPw/1iPPMOlWrkEm7YyFUTONuMFMy7ntde/8sorkqS77rqrO/fggw9K6oPAJelLX/pSd3zsscdK\nGrY59rN+Ju1XKcMMPnefTJtOpbpYEmN9r4+p0rp9UJlnWb7zzjuShoutHLzPvpN9t/+X17AeWbHi\ne7722muD91nLgaSPWcXya0MIIYQQwgaRgVMIIYQQQiOzcdURS2x0tzB/kiVh5vWwq85uFWkoA1um\nvf/++7tz9957r6TePSNJ27Zt644tM1auHkm67bbbJA3lTEu/dAmeddZZ3bHfj3IxXRBzpTXfzVh2\nYcu7PGcXDesBgwSfeuopSUO32/PPPy9p6KZlnfC9GKB45plnrvv8ueee68657th9IPX1jcd0Bfma\nOQeMt8rfrL9VFnZi+1L2p4vA7jYGF7ssHRDMc1Lv9h7LI+R34nN8POb+8/3pIliCi641hxHLpXKr\n0n4+x+Btu3Kkvk3SlX3SSSdJki666KLuHN06du/RfUdX4HXXXSdp2A69iIThFnv27Fl3f7r83O8s\nwbZkajHVVN/r3zS2s9NOO607ti3Y5pwvjWEoxOdZD3j88ssvr3tf35PPqRbmbPSCjShOIYQQQgiN\nZOAUQgghhNDIbFx1Ver2L37xi905puO3bPeXv/ylO2eZltIupWOvjKMM6OfQrUbJz+5BbsNywQUX\nrHsP5hnyOcrO3h5A6l10XFngY7rv5syqTVWrrTKkoRvFvPjii5KGMi1dAJaYuZrK7rSxVTJ+Plfr\n0NXqlVVcGbJ161ZJQ7cN5Wgfc1WW69ES3LDSahfP2Aosf3fax67WaksVqS93uggs1/OaKn/M2Pv6\nXnQL+BzdP3T5rtq6ZylunVVbrrB+00Vqdwrbx0svvSSpX+UqDW3uzxkS4fvv2LGjO7dz587uuNqS\n5frrr++OL7744nXP9O+F64s0XAVt9x/bpF36bLtzo2p/Uy5z9qlejUib2j3L3ymuWqzcs76mylEo\n9basQhp4Heubn88wCdbbaousaqP3fSWKUwghhBBCI5tSwpjaXNKj/7FMtA4K56zGWWM5un7ssce6\nY89GqAR9+9vfljRUPjhatlLEQGIGFm7fvn3wbEk6//zzJQ3VsFtuuaU79oytyo66FFYFEDNvBxUE\nq3YuH6kvVyp2nK1YhXSgqdTPVjhj5iKBs88+W5L00EMPdecYSO7rWU88q+I9WSdcZ8YCGOdKFfBf\nbQRabZ4r9fatNm5mWbFc/Ry2SZc/34czVfcXVCc4e7aSxOdUs9Jqw1GyhCzTU5m0K9Wb5ULF11ix\nZ3/NY2+m/uSTT3bn3P9R0aAK7B0cnn322e7cnXfeue5/uZNDZRfa0Tbnd/C7zVlxmrJptfE824fh\nb5LrAdV4K3ZS35ZcflLtNeA535PvQW+P60K1ATHfjcq/71/lkkvm8BBCCCGEg0AGTiGEEEIIjWxK\nVx0lVctuPGd50MHBUp96XeoDwBk8bncBA7XpFvI9Kf3axUMJn24h3585pOgO8Ltfeuml3TnnJWEw\n++23394dP/zww5KG7qMlBBJPBYJbaqWLhcG4f/rTnyQNpXvbjO5R5m1xGTLA1/ennSgn+395T76H\nJWy6LFz3+GwGun7zm98cXEvGAiXnCu1sdxvdc2N5uky1LQOP7TLhggzXI+bjqrZgoDuf5W5XBeue\nJX7alPXVria6j/zMpdjUrgy6qdzn8Xuz/3MZsJ90YC7Lj2EUvobulqpv/cUvftEd2z52rUvDdurf\nBgaCu23Tzc4wCudm43eb84bNFa7r/E1zWbGuM9j69NNPlzRsr3Zn0qXNQHD/L13utindc+yHfS/2\nEdWCDFK54RkI7t8Itslqg+J9JYpTCCGEEEIjm1JxIh5lMgjNI1vO6jlb9Ax/y5Yt3TkHYzPwj4qU\nlSiOQquNZTnC9miao3Nmpd21a5ck6dRTT+3OeZks1a4qkJWznqWkIVgLZ7KewVMpYLn78+9+97vd\nOZfr7t27u3O0n+sHZ53OUsy0FJzV+p1YtziDqTYC9eyKM7apbLyeAc05kJjYVpX9aOcqyzs/t/pH\nlYntowo+9j05O6UC7eewHZJqU3C3P96H9WQsCzmftxQq+7EsqNhanXCKAalf0MF0AtXSdvbHtiXt\nTUXD78Tgb76nr2e6gXPOOUfSUNli8LH7a/a9c1b5K9wm2Va8if0f//jH7pxTrEh9GXCRjH8bWeb8\nHbQtec59w1iqEffJ7CepTFbqkT0D7HeqzdT5ud85weEhhBBCCAeBDJxCCCGEEBqZjQ+I8p6l3yee\neKI7R3fMD37wA0lD957lOwYGUmY8/vjjJfVyrtTL0Y888kh3jnKy3W6U+pkb6p577pE0lLKvuOIK\nSUOplAGIlp4pF29E3onNCL+P5Xq6xSiv2u1Kad3SPaV3utgcQEy3miVou3OlYfC4r6FETIna78dA\nZAeYMuDSLkGpr5uUmCv30Nyo8sNUru6xTL12+1QLKlj/eWyXOuuGj5k7i7awffm+7C9sCwYK20XE\nxQh0r9uWtOnS2mflRq5y/rB/swuO/aSP+X9c/GJb0f3nPo+hD7QPbWHYjl944QVJ9Ubv7uul4W+E\nwwH4nktw1VV5j6qM9yy/apcDllUVaM864d83LuCq2gzvSfsb9g3uB+jK83uyP+VviD+v6nKCw0MI\nIYQQDgIZOIUQQgghNLIpXXVTKeItqY5tS2LXzFNPPdWds+ROaZE5nZjjxTzzzDOShhLkueee2x1/\n5zvfkTR0GVZuiSpnBvOcUBa11L2ELTnGsC3pfrW7hdJs5S7jKievlGROrJNPPrk7fvDBByUN64bl\nem75QDn+/vvvX/cc3t/SMt/d2+jQJUiXYeV+qjbAnRtVHa3cApTjWUZuKzzna7iairZwm6Wrxu2L\nrhy2WbsGaBOuiqO0b3z/KncTz9Pl52fOue1WG8KyT3ObHFtB6nZKd5jLjasaWU9cXrSz+0e6vNlf\n+53uuOOO7hxdPbY5XXXuz1nf2Me4nbOdVrnX5oy/O+vytm3bJA23pqIr1avd2De7TbNucKWy3eZs\ncy5ftg+62Hyebnjma3Odovvc17CdVm276mOyqi6EEEII4SCwKRUn4tkMFSHP7DhCpirgkSlnPZ7B\nVkGlPL7rrru6c55NerNfSTrrrLO6Y890mf+CM5Qf//jHkqTvfe973TmP1BkczlkrZztmaUGnpsrK\ny1lN9b25OfKqnD5SP9vgTNR5XahG8hrPdMds/utf/1rSMOjRKiSDSqmIeFZbBU7POfh0ql561k87\nV8GeVRZv2pbqnsuLZe22O6Yk+JmcERPPWhlc7mdydsw26xn7lOq2BKgquKz5vamoWz2nKmAliR4C\nBiK73NmP+p5jGzNfdtllkoYbA7PN+ffAOzHwPVgf2fadc4jfbQnKMPH3oH2s+lCtZ3C/2yfblDf0\nZTvkPd3nsk36f9meeew2z99D2tT2Z53w92HdqLw+G90moziFEEIIITSSgVMIIYQQQiOb0lVXbQTK\nIDJLggwGZdA3A7iNtwDwZrGSdNppp3XHlpid/0PqZULem3lFHPhG6ZdBd5YP6X7bu3evpKHcSCxR\nU1qsXBFzo5JKWW7Vxr+V5EoZvdqQl3K/60y1ZQRdBQxgtFt1+/bt3Tm6iXfs2CFpmK/LsjXrE3M/\nefEAXcNztqWp3BdTkjgldduNbk/bgu4funhsF9rPdYL/5xxqUp/3jX0IXRFVnhrfi4HjfPfK/boE\nqoU5xOVBtybdNS5Xlps/Z6A226SfyTbh/pHtiO/jvvnyyy/vztEl73vS1XPfffdJGi7moUvei33Y\nHyyBaoN1nnOfS7clf7PsOmP78e8Y89mxfdgFx3vapmPB4a4n/A1gnfDvPute9X2I+9yqDA6EKE4h\nhBBCCI1k4BRCCCGE0Mim9xdULhxLgnSLEbvwKANa3uPKGm674bwk3NLDz6ZbzXKv1LsTuKKPKw+8\nGoGrcZyLhKsJ6LLy96xW68zZLVC9O7+jP6fN6K6xXEybn3LKKeuu4W7oLn/K/V65wfwgzmMiDd1+\nhhL0NddcI6l3/0h9rpnK/SPVeUOWkPNnyq1TrXhhm3MbqK6lK+faa6/tjm+88UZJw3bq/33ggQe6\nc7///e+7Y7cpuo8qVynPuZ7ZDbv2PV036YqwLblKcM5U9dZlRHcJXZy2C12pLjdew1Wt7pNZN+wq\n5Qqr73//+92x69RYni2/p/sIqbflo48+2p3jOxm65h0uMLZt0BKwfdgX8XfM7i6Wy6WXXjq4VqrD\nS1hu1RZLdPP6evad/J20C7Vqp+wvqjCY6j2z5UoIIYQQwkFg0ytOphodMtiW6sSePXskDYO6H3ro\nIUnDAEIG+HqUyhmM8/dQpaK64ZxCHOEy0LVSLzwbZSArlYpVzDk/zJQ6UQUoVvZlrhHPbpldmDMP\nb5jM/CIM/jdcMPDb3/5W0jCQ9YYbbuiOrS4yqNSBkq53a59jlYv1yPZfmopYZQ5n+dO+Vl+9WbPU\nz3ptO6nfTFvqMxqzLH09VSZm5ndZc8bMGbWVqCp3DdUjzn6rjMRzzsm1r1QZ06VeFWA/a3WIKtS9\n997bHbuMqQi6T2X5c2GO7X/11Vd356gi+3q+mwOZ2S9TpbL9qaD5evZFS8PfsVJ3pL5PZblVbZ+2\nqvJ52b5sh+wvKhWXz7F92Tf4mfz95zX+vabyleDwEEIIIYSDSAZOIYQQQgiNzMZVVwXbUi6m283B\n2JWE7Fwd0lDys7uGcuTpp58uaSgBVxI/N6fkRpYOfKMc6W1ixjYo9nei7Lm0jSar4Dy7OVguDPT2\n/1JG9328eac0DNq2W6/a6oE28VYLUu+q+81vftOdowvnvPPOW/d9bB+69/gelq1ZH+ccFG4q13Fl\nU7pDqi0W7OqUeluxTf3yl7/sjnfu3ClpGFTqNsOgX7psbT/K9XQP8n+N2zZdgrzeroGlbc9RBdHS\nzu6X2Pfy8+eee05Svf0QNwZmndi9e7ek4SKOqe057Cof60edx+vMM8/szlVuYLqFKptW20LNmaqO\n+rjapkjqbc3cTtWWUVWfRle2y5puNeL+gDali7Taosn/y/+jzfz8asHGgdh2/r13CCGEEMJBYjaK\nE/HMgxuBcuTrkSSVAm/4+8Mf/rA7x1mTN4vk7NcBvlySTFXBSsVYigOPrD0Lk/oRMIOLOevxd+II\neQkzWVJle63OcSZrBYLBhg4uZroHzjy85JlBp64HzHjLjN+uO9zYl8+84447JA1VqquuukrScEbM\nQFgHQ3LG5vo654D/KTwDpVLDWanrOhdcWFFkcKrtLEmPPPKIpGHbdvmOpQFwO+RMl/2FFxnQPn7+\nmPpgFWWjlzn/r5mqj7blmOLk/pHL/K3yUzmnmmwlioqTbcl0AlQUfQ1VLG746zZfqSBbt25d975S\nbfMq8/QSqFIHEP4muf9j+6qW/NN+voYqlsuQ7Yyfux2zv6aHwYu9qEhVKQ6qTakrko4ghBBCCOEg\nkIFTCCGEEEIjm95Vt2pzWEq/zBviAG9m9HYAKHOBOA+T1MvAzMPk4HDKltw0kgGQhhK1XQOUtR3I\nzIBUBqpW0nAViDc3Wl0atCldOJbk6Tb1MTd7pgvUkjtt5nrCusFytc39VxrmBPrVr34labiIwDKx\n3YDS0L5VAKOP5+yqq+w3tYiDMrwle+Z1cdAwJXy6Sn0vSvgua7pY6Mpz+6o2FJX6OkG3jd+d96RL\nfu37SL0rY842nQqMthuE5cfFMw51YD/oNu2FMVLtyiZun2yndN8+/PDDkoaLgugq8jHrqF36v/vd\n77pzDKNwm6abni75JVHlWyN0h7kM+Dv40ksvSao34ZX6sqZ7r1osws/dx9NmdMG5TtF9W+0AwDrs\n+lp9xwNpp1GcQgghhBAaycAphBBCCKGRTe+qs2xHSc8SfyWdS71MT7nfcrFX5UhDt4GvYZ4eP4fS\nH6VDS9RcrcOI/irdu10MljqlOvKfrp45u+hWwe9drTzjajevcKSLxq46usjoXvCqLa7gsdxP9x3v\n6ZV4XB1JV8Rll10maehSrPKc0GXo66dWssyNKam7yg/DtuR2Q7eN3SWU4Nm+3KbYH9imbGd0NfiZ\nbEeVG5g28QpJtkO+e7Vaa84uOsMy8Hes6irdLSwLlzFt6nZBVxvLyqulqvLjc5h/yaEVbGfcBNh1\njvnA7N679dZbu3PPPvtsd+zVsVxxa5dk5RKaC1WYROVS5+pkumLdVqa2Cau2OmGbcz3gs9mObWu2\nd76TP2c/a7iimddXoS8bYcsoTiGEEEIIjWzKYXS1ISxHiVYLGKxGVeHuu++WJJ199tndOSsEHHlS\nsXLAGUfIHg3z3lWgKke7nMl61sXNRw0DIqmY+Jl8D8+k56w8TdnUMwMGldI+l1xyiaShUudAf6qA\n3ATY92J+F9cD2ozqk2fFnGXTps75w0UEXmTArPGc9fiYCucS1IlqJkv1wWrNmOJU5V1yWVWZiwkV\nDc9EWb6cqboe8Xk8dvtinVj1boT/5/KYc7bpqc3UbUsqDmxztguVfSu6DN5mnfDnVHYrFZc2d99P\nRYrl7rrAzbZ9zOdUatoSsvpPQTu7rLhIidjW/P1xWxnLAu7fSarAtk+1uwD/l78LtL/7E37ufoLv\nwbxvvr7aHeBAWH4NCSGEEELYIDJwCiGEEEJoZFO66iq5mK4Ny7TMpUOZ9vHHH5ckPfHEE925H/3o\nR5KGAW5Vrh1uGOscH3Sr8Zl2+9BFwGBFu/ief/757lwVbEg50ucZsOxzc3bvTOX8sfTLAFK6RZ1b\nhXL9rl27JA3z7zCfl10IlG59f7r8KM07UJiuBN7TUNZ2Hil+R76Tv1sVeLvkrRwso08FWDPQ3vWe\n5U/7VVs5uE2xzbBN2aZ0C9CV5+ezTbrN02bsO8a2d5HmbdPq3Sv70c4MJPaiCoZReONstu1zzjmn\nO7ar/Pbbb1/3HNqUdcf9LAO52Z/Y5tV2P9u3by/v6f5iLPB9rkxtxl0tnmD/6PZBd5j7NIYscENl\n34vub/eZPMdn+pg2Y+iG22S1yKZalMXvudFhLvOvFSGEEEIIB4lNqTgRjy6rESNHlhdddFF3bIXH\nypPUL0t1cK80DAp2wCDvecYZZ0gaBjUyS7VH4Lt37+7O7dy5szt2UDKzXZ944omShrPbKkBu1Yx2\njlTB4aQKiudsw7NaLkl2sOeePXu6c1xe7MUBXCTgGRLvTfVh1XJqqbclA9c9Y6MKxVlVldnazFlF\nJLYpv0/VZnnOM0gGf7tdsB0ye7Ovr1ILTJUllWHe38fVOd6zCj6umLPiVFFlh2ZZVPZjO73gggsk\nSXfeeWd3jkHbXlSxbdu27pzbD/tolmulEjL1gH8D2I9a5WLmadYjK4ps+0uwZfUd2Be5rlNZZ7k5\nrQvLxTanGkl10AHlvlbqFWiq8WxztgvVZL5HtROH6wG9Qmybn9TvaBSnEEIIIYRGMnAKIYQQQmhk\n07vqKizVUQZkXpHzzz9f0jAY0XlF6EKhvGdJnjmVLCMyKI5y844dOyQNc85s3bq1O7YETbeO3X6U\nmKs8NUtx4Zip7LU+psxK+znQn5t+OhCVcixtYRme+ZXsQmCZsx5VGwfT/WdXKoNffczvUwUSTwVp\nzo3K/VptTj2Wu6kqD39OtwCvcftl+XvBBoPIaVO7e+hWoP3dzmk/B50yOJwBqNVG1Utrs4bfka4z\nwzZrFzgDxt3m2A6Jr6HNXJZjG8K67dOtw3bqfoDv4U2G6VJiPfAzuVhhzu3TVO2U5/x9+TtV9V8s\nF2d75/+x73XYAn873Wb5G0v7+ph5trghs387uSDA92I4RfW7stHtNIpTCCGEEEIjGTiFEEIIITSy\n6V111TYGlsy5MooyriXBa6+9tjtnyZArnyjDO+/HH/7wh+6cN4WkBEmZ0KsAuGrr4osv7o7tanrs\nscfWvTulxcp9MbUKbc74+9CFY/tylQtt7pUYe/fu7c55tVW1fYDU5+SiTV3+lIvpFrIc7W1U+Gyp\nd/fQfr7nlJ2m8pPNjUr+niqD6vtyVaPLl/9Xbc3Dayztj62gqVy6dNn7uNoKonI9kur7zrm9Trk0\n3GbZdtnmbAv3p1LfVpi7iX234cbadqHRZnQLGW7twnv6Xlwp5u9Gly1dvrbv0lzqZJWLmTal287t\nj+Vvly1tz99W24KuULtVx9zwXvH885//vDt38803d8fXX3+9JOnKK6/sztkVO7ZpepV3zByITaM4\nhRBCCCE0sukVJ8MRo485a6Qq4M85s6hytXAE7RE2z1mVqP5P6nOaMGcJgxE9CmYGbAdAcrQ7Natd\nAlMKWjUz4AzICgBnOA4GZtAu1QmrEgxQnFINPAPiPakuOYCRM9VqE+bWYMSlzWSJ7ccy5/f1DJR2\ndptmAC+DUn3MNmmbsz9gPfDst+pDCK+Zyn1TKcOr7j0XpgKJq4UO1SbADNr252Mb9hoqRl4MwgUi\nzKfnPpfBxezvrTRVaj7rIz933WLbX3vtUqi+z5hqs6rP5O8pbWH7M4eh1Uju6MH3cH/ORR7OpSj1\nmd3ZNq1ojdl01e/pgdg0ilMIIYQQQiMZOIUQQgghNDIbV91UYG0lyVF69OeU+KscIXS3WFqu3D88\npluBcrPdSnQvVdtuLG17lSlWBeqNbcZo+1Eatl3oyqHM7mvoVnP5061Dm1nuZ93gM319ZbMpV1DF\nUlx1q7aTqTYUlXq7TLVj1gnbhXm2fE8GJNM9W9mq2mx4XwLbV/3vUtw6q9yvU3W9cuEwjIGfu80x\nR5QX5DAMgscOmWDfS/eg7VP1B9Vm22v/dy1LaaerqFzRUl8uLDf/po1tHebyqoLMnXdt7TN9f26J\nw37aC7P4G852Xr372vcZ+3xfieIUQgghhNDIbBSniqlRZDWT5fJIHlfL4T0D4iy4eg5nKgyGq2bc\nHKGvYskznCrodGoWUKkCLsupgPKp+1RMKV/7c88wtI+VQiqGZsymhkqDobLEQOEqxUHrDHTss6Wo\nSmvZn9l6tdCF5e9zDPSeerZVBdqe6oIDjStlhOc3SmmYs703qq5Xv6dUhKpFNlSH7MGpPEE85mIq\n/l5WdWLVIo3qO2wUUZxCCCGEEBrJwCmEEEIIoZFD5ixBhhBCCCEcTKI4hRBCCCE0koFTCCGEEEIj\nGTiFEELu3WacAAABCElEQVQIITSSgVMIIYQQQiMZOIUQQgghNJKBUwghhBBCIxk4hRBCCCE0koFT\nCCGEEEIjGTiFEEIIITSSgVMIIYQQQiMZOIUQQgghNJKBUwghhBBCIxk4hRBCCCE0koFTCCGEEEIj\nGTiFEEIIITSSgVMIIYQQQiMZOIUQQgghNJKBUwghhBBCIxk4hRBCCCE0koFTCCGEEEIjGTiFEEII\nITSSgVMIIYQQQiMZOIUQQgghNJKBUwghhBBCIxk4hRBCCCE0koFTCCGEEEIjGTiFEEIIITSSgVMI\nIYQQQiMZOIUQQgghNJKBUwghhBBCIxk4hRBCCCE0koFTCCGEEEIjGTiFEEIIITTyX9W04toGCZ1p\nAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for i in range(5):\n", + " plt.subplot(1, 5, i + 1)\n", + " plot_image(weights1_val.T[i])\n", + "\n", + "save_fig(\"extracted_features_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sparse Autoencoder" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving figure sparsity_loss_plot\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk4AAAGFCAYAAAALnnwfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XmcjtX/x/HXuccwM/YtyU4RshdS+UqW7GuJJEtUtija\nkKUUypJC2WmZnyVEkSjJrmbyJUvIUt9sjTX7MOf3xzWmGTNjZpiZ675n3s/H434M576W9z0jfZxz\nrnOMtRYRERERSZjH7QAiIiIivkKFk4iIiEgiqXASERERSSQVTiIiIiKJpMJJREREJJFUOImIiIgk\nkgonERERkURS4SQiIiKSSBncDuCtjDG5gfrAAeCiu2lEREQkkQKAosBya+3x5L64Cqf41Qc+czuE\niIiI3JQngc+T+6IqnOJ3AODTTz+ldOnSLkeRvn37MnbsWLdjCPpZeBP9LG7swgVo0ACaNoUXX0z5\n++nn4R127txJ+/btIfL/48lNhVP8LgKULl2aypUru50l3cuePbt+Dl5CPwvvoZ/FjX38MZw7B2++\nCcWKpfz99PPwOikyzUaTw0VEJM2xFt5/H5o1S52iSdIP9TiJiEias2IF7NwJkya5nUTSGvU4iYhI\nmvP++1CxItSs6XYSSWvU4yQ+oW3btm5HkEj6WXgP/Szi9ttvsHQpzJgBxqTeffXzSB+MtdbtDF7J\nGFMZCAkJCYl3st8ff/xBWFhY6gYTSWZ58uShcOHCbscQSTY9e8K8eXDwIAQEuJ1GUltoaChVqlQB\nqGKtDU3u66vH6Sb98ccflC5dmvPnz7sdReSWBAUFsXPnThVPkiacOgUzZ8JLL6lokpShwukmhYWF\ncf78ea3zJD7t2nonYWFhKpwkTZg2DS5fhuefdzuJpFUqnG6R1nkSEfEOV67Ahx/CE0/A7be7nUbS\nKhVOIiKSJixcCAcOwPz5bieRtEzLEYiIiM+zFt59Fx5+GJx5wSIpQz1OIiLi89asgZ9+cpYhEElJ\n6nESERGf9957ULYsPPqo20kkrVPhJF7l4MGDeDweZs+eHdU2ZMgQPB79URWRuO3aBUuWQL9+qbvg\npaRP+r+RxDJr1iw8Hg+hoTHXDTtz5gz33XcfQUFBfPvtt8C/Rc2JEydSLI8xRoWTiMRr9GjInx+0\ncLekBv3fSOJkrvtn2z///EPdunXZvn07ixYtol69elHHXX9schs0aJAWGhWROB05ArNnQ+/ekCmT\n22kkPVDhJAk6e/Ys9erVY+vWrSxYsCCqaEotHo+HjBkzpuo9AS5evJjq9xSRpJkwATJmhOeeczuJ\npBcqnOSGzp07R/369dmyZQsLFizg0WSceXn69Gk6duxIjhw5yJkzJ506deLUqVOxjrt+jlO5cuWo\nU6dOrOOstRQoUIA2bdrEaBs3bhz33HMPgYGB3H777Tz33HOx7lO0aFGaNm3Kt99+y3333UdAQACT\nJ08GnAKqd+/e5M2bl2zZstG8eXMOHTqEx+Nh2LBhMa5z6NAhOnfuzO23305AQAD33HMP06dPj3HM\n6tWr8Xg8zJs3j+HDh1OoUCECAwOpU6cOv//+e6zPtWnTJho2bEiuXLnIkiULFSpUYPz48TGO+e23\n32jdujW5c+cmMDCQ++67jyVLlsT3rRdJE86dg4kT4ZlnIEcOt9NIeqHlCCReZ8+e5dFHHyUkJIQv\nvviCBg0aJOv1mzZtyvr163n++ee5++67WbhwIU8//XSsob/rhwPbtGnD0KFDOXbsGLfddltU+5o1\nazh8+DBPPPFEVFu3bt2YPXs2nTt35oUXXmD//v188MEHbNmyhXXr1uHn5xd1j127dtGuXTueffZZ\nunXrRqlSpQB4+umnmT9/Ph06dKBatWqsXr2aRo0axcp57NgxqlWrhp+fH7179yZPnjwsW7aMZ555\nhrNnz9K7d+8Yx48YMQI/Pz/69+/P6dOnGTlyJO3bt2fDhg1Rx6xYsYImTZpwxx130KdPH26//XZ2\n7tzJ119/HXW97du38+CDD1KwYEFee+01MmfOzNy5c2nevDkLFiygWbNmt/JjEvFaM2bA6dPQp4/b\nSSRdsdbqFccLqAzYkJAQG5eQkBB7o/d92cyZM60xxhYtWtRmypTJLl68ON5jhwwZYj0ejz1+/HiS\n7rFo0SJrjLGjR4+OaouIiLA1a9a0Ho/Hzpo1K9Y9rtm9e7c1xtgJEybEuGb37t1t1qxZ7cWLF621\n1q5Zs8YaY+z//d//xTju22+/tcYYGxwcHNVWtGhR6/F47IoVK2IcGxoaao0x9qWXXorR3qlTJ+vx\neOzQoUOj2rp06WILFChgT548GePYtm3b2pw5c0bl+uGHH6wxxpYtW9ZeuXIl6rjx48dbj8djt2/f\nbq219urVq7ZYsWK2ePHi9syZM/F9K+0jjzxiK1asaMPDw2O0P/DAA7ZUqVLxnmdt2v5zLGlbeLi1\nxYpZ27at20nE21z7ew2obFOgPlCPUyo5f955ZDYl3X03BAUl3/WOHTtGQEAAhQoVSr6LRlq2bBn+\n/v48F21igjGGXr16sWbNmhuee9ddd1GxYkXmzJlD9+7dAYiIiIjqXckUOUN0/vz55MiRg0ceeYTj\nx49HnV+pUiWyZMnCqlWrYvROFStWLNYQ4DfffIMxhuev2zG0V69ezJw5M0bbggULaNOmDVevXo1x\nv3r16jFnzhxCQ0O5//77o9o7d+4c1eMF8NBDD2GtZd++fZQpU4ZffvmFAwcO8P7775M1a9Y4vxcn\nT55k1apVvPnmm5w+fTrGe/Xq1WPo0KEcPnyY/Pnzx/v9FPFFCxfC/v3aXkVSnwqnVLJrV8pvAxAS\nAsm137AxhsmTJ9OnTx/q16/P2rVrueuuu5Ln4jjrNeXPn5+g6yq9a8NjCWnTpg0DBgyIKgpWrVrF\n0aNHY8xv2rNnD6dOnYoxnHeNMYZjx47FaCtWrFicOT0eT6z37rzzzhi///vvvzl16hSTJ0/m448/\nTtT9ri9Ic+bMCTjFEMDvv/+OMYayZcvGut41e/fuxVrLoEGDGDhwYLz3VeEkaYm1zoKXtWsn3995\nIomlwimV3H23U9ik9D2SU+nSpfnmm294+OGHqVu3LuvWraNAgQLJcm1rbZzLGFhnmDRBbdq04bXX\nXmPevHn07t2buXPnkiNHDurXrx91TEREBPny5ePzzz+P87p58+aN8fvAwMBE578+e0REBADt27fn\n6aefjvOc8uXLx/h99N6m6K5lTcz34tp9+/XrF+OzR3d9kSfi6374ATZv1vYq4g4VTqkkKMg3/2VU\npUoVvvzySxo2bEjdunVZs2YNuXPnvuXrFi1alFWrVnH+/PkYvU6//fZbos+vWrUqc+bMoUePHixc\nuJAWLVrg7+8fdUyJEiX47rvvqFGjRtTwXVIVKVKEiIgI9u/fT4kSJaLad+/eHeO4vHnzkjVrVq5e\nvUrt2rVv6l7Xu/POO7HW8uuvv8Z7zeLFiwPg7++fbPcV8XZvvw0VK2p7FXGHliOQBD388MMEBwez\nZ88eHn30Uc6ePXvL12zYsCHh4eFMmjQpqi0iIoIPPvgg0QtqtmnTho0bNzJt2jTCwsJiDNMBPP74\n41y5ciXWkgEAV69ejTUnKC7169fHWsvEiRNjtF+f0+Px0KpVK7744gu2b98e6zphYWGJ+kzRVa5c\nmWLFijFu3Lh4s+bNm5datWrx8ccfc+TIkWS5r4g327wZVq6E11/X9iriDvU4SZyuHyZq3rw5U6ZM\noUuXLjRu3Jjly5dH9eJYaxk9enSs+Uoej4fXXnstzus3adKEBx98kFdffZX9+/dTpkwZFixYwD//\n/JPojI8//jj9+vWjf//+5M6dm0ceeSTG+zVr1uTZZ59lxIgRbNmyhXr16uHv78/u3buZP38+48eP\np2XLlje8R+XKlWnVqhXjxo0jLCyM6tWrs3r1avbs2QPEHLIbMWIEP/zwA9WqVaNr166UKVOGEydO\nEBISwvfff5/kIsYYw8SJE2nWrBkVK1akU6dO5M+fn127drFjxw6WLVsGwIQJE3jooYcoV64cXbt2\npXjx4hw9epQNGzbw119/8csvvyTpviLe7J13oGRJSOA/XZEUo8JJ4hRXr0/Hjh05ceIE/fv35/HH\nH2fhwoVRx44YMSLW8RkyZIi3cDLGsHjxYvr06cNnn32GMYZmzZoxZswYKlWqlKg8BQoUoEaNGqxf\nv56uXbvGOWdo0qRJ3HvvvXz88ccMGDCADBkyULRoUTp06MADDzwQ4/rx9XR98skn5M+fn+DgYBYt\nWkTdunWZM2cOJUuWJCAgIOq42267jc2bNzNs2DAWLlzIpEmTyJ07N2XLlmXUqFEJfp642uvXr8+q\nVasYOnQoY8aMISIighIlStCtW7eoY0qXLs3PP//M0KFDmTVrFsePH+e2226jUqVKDB48OM77iPii\n7dth0SKYPh3imSIokuJMYifjpjfGmMpASEhICJXjmJwUGhpKlSpViO99Sdu2bNlC5cqV+eyzz2jr\nwzuL6s+x+JKnnoLVq2HvXmebFZG4XPt7DahirQ1N6Pik0hwnkQRcunQpVtu4cePw8/OjZs2aLiQS\nSX/27YPgYOjfX0WTuEtDdSIJGDVqFCEhIdSqVYsMGTKwdOlSli9fzrPPPptsyzOIyI29+y7kygVd\nuridRNI7FU4iCbj//vtZsWIFb731FmfPnqVw4cIMHTqU119/3e1oIunC4cPOvKYhQ5J3dwSRm6HC\nSSQBderUibUVi4iknjFjICAAIndYEnGV5jiJiIjXOnECJk2Cnj0he3a304iocBIRES/2wQcQEQEv\nvOB2EhGHCicREfFK//wD48dD164Qx17dIq5Q4SQiIl7pww/h7Fno18/tJCL/UuEkIiJe559/4L33\n4JlnoFAht9OI/EuFk4iIeJ0PP3SKp1dfdTuJSEwqnERExKv88w+MHq3eJvFOKpxERMSrTJgAZ85A\nPHuEi7hKhZPEadasWXg8HjweD+vXr4/zmEKFCuHxeGjatGlU27lz5xg8eDDlypUjS5Ys5MmTh0qV\nKtGnTx+OHDkSddzQoUOjrn/9y8/Pj2PHjqX4ZxQR73P2rDO3qUsX9TaJd/KJlcONMRmBN4EngVzA\nVmCgtXZlIs6tA7wOlMP5vLuBD6y1n6Zc4rQjMDCQzz//nBo1asRoX716NX/99RcBAQFRbVeuXOGh\nhx5i9+7dPP300/Tu3ZuzZ8+yfft2goODadmyJbfffnvU8cYYPvroIzJnzhzrvjly5Ei5DyUiXku9\nTeLtfKJwAmYDLYCxwF6gI7DUGFPLWht3dwhgjGkKLATWA4MBCzwOzDbG5LbWvp/SwX1dw4YNmTdv\nHuPHj8fj+beD8vPPP+fee+8lLCwsqm3hwoVs2bKF4OBg2rRpE+M6ly9f5vLly7Gu36pVK3LlypVy\nH0BEfMbZs85mvl26QOHCbqcRiZvXD9UZY6riFDuvWmtftdZOBR4BDgKjEji9B3AIeNhaO9FaOwmo\nA/yOU3zJDRhjaNu2LcePH2fFihVR7eHh4cyfP5927dphrY1q37dvH8aYWL1TABkzZiRLliypkltE\nfNPEieptEu/n9YUT0Bq4Aky51mCtvQRMA+43xhS4wbnZgJPW2ivRzr0KhAEXUiZu2lK0aFGqV69O\ncHBwVNvSpUs5c+YMTzzxRIxjixQpgrWW2bNnJ/r6x48fj/U6ffp0suUXEd9wrbepc2f1NsktuHAB\n3k/ZwSRfKJwqAruttWeva98c7f34/ACUNcYMM8aUMMYUN8YMAqqQcG+VRGrXrh2LFi3i0qVLgDNM\n95///CfGfCWAFi1aUKpUKQYNGkSxYsXo3LkzM2bM4O+//47zutZaSpUqRd68eWO87r///hT/TCLi\nXSZOhNOn1dskt8BaZ3+eOXNS9Da+MMcpP3A4jvbDgAHuuMG5w4BiwABgYGTbOaCVtXZJcoZMlMOH\nnVd8AgKgTJkbX2PHDrh4MXZ7/vzOKwU8/vjj9OnTh6+++or69evz1Vdf8eGHH8Y6LlOmTGzevJnh\nw4czd+5cZs2axcyZM/F4PHTv3p3Ro0fj7+8fdbwxhgULFpA1a9YY14lrsriIpF3XnqTr1AmKFHE7\njfisd9+Fzz6Dd95J0QrcFwqnQOBSHO0Xo70fn8s4T9HNAxYAfkA34DNjTB1r7eYbnJv8Pv4Yhg6N\n//0yZWD79htf47HHnOLpeoMHw5AhtxQvPnny5KFOnTp8/vnnnDt3joiICFq3bh3nsVmzZmXEiBGM\nGDGCP//8k++++4733nuPCRMmkCNHDoYNGxbj+IceekiTw0XSufffd3qbBgxwO4n4rK+/dpaZHzAA\n6tVL94XTBSBTHO0B0d6PzwSgqrW28rUGY8w8YDvwPpDgmFDfvn3Jnj17jLa2bdtSqlSphE6N7dln\nIdqaR7FEe7Q/XvPmxd/jlILatWtH165dOXz4MA0aNIjVSxSXQoUK0bFjR5o3b07x4sX57LPPYhVO\nIpK+nTzpdBQ8+6zmNknSBQcHEzx5MqxZA/nywX//y+k1a1L0nr5QOB0m7uG4a5XCobhOMsb4A52B\nkdHbrbVXjDHLgB7GGH9rbfiNbj527FgqV64cqz00NDQR0a9PnAzDaQkN5aWQFi1a8Oyzz7Jp0ybm\nJHH8OEeOHJQoUYLtCfWmiUi68+67EB4Or7/udhLxRW3r16ft669D2bKwdi1kzUpoaChVqlRJsXv6\nQuG0BahljMly3QTx6jjrMm2J57zcOJ/PL473/HEmxvvC5HivkDlzZj766CMOHDhAkyZN4jxm69at\nFChQgNy5c8doP3jwIDt27ODuu+9Ojagi4iOOHnWG6Xr3huueNRFJnBw5nEcxO3SARIyEJAdfKJzm\nA/1w5iaNgaiVxDsCG621f0W2FQKCrLW/RZ53DDgFtDDGvHFtSQJjTBagCbAzclkDiUf0NZoAnnrq\nqRsev2LFCgYPHkzTpk2pXr06WbJk4ffff2fGjBlcvnyZIdfNwbLWMm/evDjXd6pXrx558+a95c8g\nIt7rnXcgQwbo39/tJOKzPB4YNChVb+n1hZO1dnPkvKR3jDH5+Hfl8CJAp2iHfgLUJLIXyVobYYx5\nD2erlk3GmNk4n7cLUAB4OdU+hI8yxiTqmGvHtWrVirNnz/Ltt9+yatUqTpw4Qc6cOalWrRovvfQS\nNWvWjHVu9+7d47zuqlWrVDiJpGF//gmTJsHAgaDnQ8SXmOt7FbxRtL3q2gM5iWOvOmPMKuAha22G\n6859AngBKIkzyXwrMMpauyiBe1YGQkJCQuKd41SlShXie1/EF+jPsbilWzdYuBD27Uu1ERZJJ6LN\ncapirb2JCck35vU9TgDW2svAK5Gv+I55OJ72/wP+L4WiiYhIEu3ZA9Onw6hRKprE92hytIiIpKoh\nQ5wnx59/3u0kIkmnwklERFLNr79CcLAznzfwRssXi0R36ZLzNMEl95/pUuEkIiKpZtAgKFrUeYJc\nJFGsdVZIHToUdu50O41vzHESERHft2EDLFoEs2dDxoxupxGf8d57MGuWsw9dxYpup1GPk4iIpDxr\n4eWXoUIFePJJt9OIz1i8GF55xdmDrl07t9MA6nESEZFUsGSJsyPGN984axaKJGjbNqfKbtECvGif\nU/3xFRGRFHXlirNx/SOPOBvXiyTo8GFo3BjuussZ2/Wials9TrdopxdMVBO5WfrzK6lh5kxnTu8n\nn0AiNiQQcZ4euHLFGarLnNntNDGocLpJefLkISgoiPbt27sdReSWBAUFkSdPHrdjSBp1/jwMHgxt\n20IKblgvac0HH8C5c1CwoNtJYlHhdJMKFy7Mzp07CQsLczuKyC3JkycPhQsXdjuGpFHjxsHff8Pw\n4W4nEZ9y551uJ4iXCqdbULhwYf0PR0QkHmFhMHIkdO8OxYq5nUYkeXjPbCsREUlT3nrL+TpwoLs5\nRJKTCicREUl2+/bBxInOEjyaQidpiQonERFJdgMHQt680KeP20nEq1nrdoIkU+EkIiLJavNmZyPf\nYcMgKMjtNOK1zp+H+vXhu+/cTpIkKpxERCTZWAt9+zpbq3Ts6HYa8VoREdChA6xbBzlyuJ0mSfRU\nnYiIJJt582D9eli5Evz83E4jXuv112HBAli40OcW+FLhJCIiyeLiRWcyeJMmzvYqInGaMsVZp2LM\nGGjWzO00SabCSUREksW4cfC//8Hy5W4nEa+1cqWzsNfzz/vskwOa4yQiIrfs6FF4+23o0QNKlnQ7\njXilrVuhVSuoUwfGj/fZjQtVOImIyC174w3IkMH5KhKnBQugRAmYO9f5w+KjVDiJiMgt2bYNpk51\nNvPNlcvtNOK1Bg+GH3+ErFndTnJLVDiJiMhNsxZefNHZk7V7d7fTiFczBrJkcTvFLfPdvjIREXHd\n0qXOfN8vvwR/f7fTiKQ89TiJiMhNuXwZXnoJatd2liAQSQ/U4yQiIjflgw9gzx5nrq+PPiAlkmTq\ncRIRkSQ7cgSGDnXmNZUv73Ya8Sr//S9cueJ2ihSjwklERJLs1VchY0aneBKJEhICDzwAo0a5nSTF\naKhORESSZMMGmDULPv5Yyw9INAcOQOPGUKYMvPCC22lSjHqcREQk0SIioFcvqFwZunRxO414jZMn\noWFDCAqCr76CzJndTpRi1OMkIiKJNn26Mxqzdi34+bmdRrzCpUvQvLmz786GDXDbbW4nSlEqnERE\nJFFOnYLXXoP27Z1pLCJERECHDrBpE3z3XbrYqFCFk4iIJMrgwXDxIowc6XYS8RoDBsC8eTB/frqp\nplU4iYhIgn79FSZMgLffhjvucDuNeI327aF0aWjZ0u0kqUaFk4iI3JC10Lu3s7F9nz5upxGvUras\n80pHVDiJiMgNBQfDqlWwbJmzdpNIeqblCEREJF6nT8OLL0Lr1vDoo26nEXGfCicREYnXwIFw7hyM\nG+d2EhHvoKE6ERGJU0gITJwI774LBQq4nUZcZa12co6kHicREYnl6lV4/nln3m+vXm6nEVedPAm1\najmLW4p6nEREJLYpU+Cnn5wVwv393U4jrrlwAZo1c9ajyJHD7TReQYWTiIjEcOyYs0J4587pZk1D\nicvVq/Dkk/Dzz7BypbNek6hwEhGRmF5+GTwerRCerlkLPXvC4sWwcCHUqOF2Iq+hwklERKL8+CPM\nmuUM1eXJ43Yacc2wYfDRRzB1KjRp4nYar6LJ4SIiAsDly86E8OrVnWE6Sac++ACGDIHhw6FLF7fT\neB31OImICAAjRsDu3c4yBB79szp9sha2boWXXnImukksKpxERIRdu5wOhv79oXx5t9OIa4yByZP/\n/bXEosJJRCSdi4iAbt2gcGEYNMjtNOI6FUw3pMJJRCSdmzoV1qyB77+HwEC304h4N41ii4ikY4cP\nO8sPdO4MDz/sdhoR76fCSUQkHevVCzJlcvajE5GEqXASEUmnvvwSvvgC3n8fcuVyO42kqrAweOst\nZ4KbJIkKJxGRdOjMGejRAxo2hDZt3E4jqeqff6BBAxg/Hg4dcjuNz/GJwskYk9EYM9IY8z9jzHlj\nzEZjTJ0knN/GGLPeGHPWGHPSGLPOGFMrBSOLiHi111+HU6dg0iQ9RJWuXLwIzZs7C3YtXw4FC7qd\nyOf4ylN1s4EWwFhgL9ARWGqMqWWtXX+jE40xQ4BBwDxgBuAP3AMUSMG8IiJea8MGmDgRxo51liCQ\ndOLKFWjXDtavh2+/hUqV3E7kk7y+cDLGVAUeB16y1o6NbPsE+BUYBTx4g3Or4xRNfa2141MhroiI\nV7t0CZ55Bu6919nDVdKJiAjnB79kCSxaBA895HYin+ULQ3WtgSvAlGsN1tpLwDTgfmPMjXqO+gCH\nrxVNxpjMKRlURMTbDRsGe/bA9Ong5+d2GkkV1jqPT86e7bwaNXI7kU/zhcKpIrDbWnv2uvbN0d6P\nT23gJ2PMC8aYv4F/jDGHjDE9UiKoiIg3+/lnGDkS3ngD7rnH7TSSavbtg08/hSlToG1bt9P4PK8f\nqgPyA4fjaD8MGOCOuE4yxuQA8uAM5dUGhgB/Ap2AD4wxl621U+I6V0Qkrbl0CTp1cvahe+UVt9NI\nqipRAvbuhbx53U6SJvhC4RQIXIqj/WK09+OSJfJrLqCNtXY+gDHmC2AbMJBow38iImnZW285G/mG\nhIC/v9tpJNWpaEo2vlA4XQAyxdEeEO39+M4DCAe+uNZorbXGmDnAEGNMQWvt/2508759+5I9e/YY\nbW3btqWtujtFxEeEhsI77zgb+JYv73YakeQTHBxMcHBwjLbTp0+n6D2NtTZFb3CrjDHfAndYa++5\nrr02sBJoYq39Oo7zDHAOOGmtLXDde88CE4GK1tpt8dy3MhASEhJC5cqVk+fDiIikssuX4b77nLWa\nNm+GjBndTiSSskJDQ6lSpQpAFWttaHJf3xd6nLYAtYwxWa6bIF4dsJHvxxLZs7QFuNcYk8FaeyXa\n29cKqb9TJLGIiJd4+23YsQN++klFk0hy8IWn6ubjFHjdrjUYYzLiLIK50Vr7V2RbIWNMqevOnQP4\nAU9HOzcAeBLYbq09krLRRUTc89//wvDhzirhFW/0/LGkDd9+66wILinK63ucrLWbjTHzgHeMMfn4\nd+XwIjhPyF3zCVCTmMXgx8AzwITIouoPoANQCGic8ulFRNwRHg4dO0Lp0jBggNtpJMWtWgXNmjkr\ng0+b5naaNM3rC6dITwFvAu2BnMBWoJG1dl20YywQY5tna+1FY8zDOCuMdwIy4wztNbTWrkyN4CIi\nbhgxArZtg02bNESX5m3cCE2aOKuBT5jgdpo0zycKJ2vtZeCVyFd8xzwcT3sY0DmFoomIeJ2QEGeF\n8FdfBWeOrKRZW7ZAgwbOvnMLF0JAQMLnyC3xhTlOIiKSSBcuwFNPQblyzgrhkoZt3Qp16sCdd8JX\nX0Fm7SqWGnyix0lERBJnwABnh42QEA3RpWnbt8Mjj0Dhws6k8OvWG5SUo8JJRCSNWLUKxo6F0aOh\nbFm300iKGjUKChSAFSsgZ06306QrKpxERNKA06fh6aehVi3o08ftNJLiJk+Gs2chd263k6Q7KpxE\nRNKA3r2Gv6AqAAAgAElEQVSd4mnmTPBo9mralymT85JUp8JJRMTHLVgAs2c7RVORIm6nEUnb9O8S\nEREfduQIdOsGLVpAhw5upxFJ+1Q4iYj4KGuha1fIkAE+/tjZyFdEUpYKJxERHzVtmrN8z9SpkDev\n22kk2R06BJMmuZ1CrqM5TiIiPui33+CFF+CZZ6Cxdt5Me44cgdq14dw5aNsWcuRwO5FEUuEkIuJj\nLl929nItWBDGjXM7jSS7Y8ecxS3/+QdWr1bR5GVUOImI+JiBA50NfDdu1C4bac7Ro05P04kT8MMP\nznYq4lVUOImI+JCVK+Hdd+G996ByZbfTSLK6Njx36pRTNJUq5XYiiYMKJxERH/H3386SA3XrQt++\nbqeRZHXokFM0/fOPUzSVLOl2IomHCicRER9gLXTpAuHhMGuWVgdPc7ZscSavrV6t4Tkvp8JJRMQH\nTJoES5Y4r/z53U4jya5hQ9i5U9uo+AD9m0VExMv9+iu89BL07KmlB9I0FU0+QYWTiIgXu3DBWcbn\nzjth1Ci304jITRdOxpjKxphpxpjdxpizxpjzxpi9xpjZxpg6yRkynvsPMcZEGGNqpvS9RETc0r8/\n7NkDwcEQGOh2GhFJ8hwnY4wBRgN9gHDge+DLyF8XBxoCTxpj3rDWDk/GrNezkS8RkTRp/nyYMMF5\n3XOP22lEBG5ucvhwnKIpFGhtrT0Q/U1jTCagJ5D7ltPdmLazFJE0a98+5ym6xx6D5593O40ki717\noVcv57HI225zO43cpCQN1RljSgD9gTDg0euLJgBr7SVr7WhgcLTzchtjxhlj9hljLhpjjhpj5hhj\nysZxj2zGmGHGmO3GmH+MMaeNMXuMMTONMYUij1kFvBF5yg+RQ3YRxph9Sfk8IiLe6NIlaNMG8uSB\nKVPA6J+Jvm/3bqhVy6mIr1xxO43cgqT2OHXCKbY+staG3ehAa204gDEmD7ARKAb8AARH/ro10MgY\nU89auz7aqd8C9wHrgGVABFAEaALMBv4EZkQeWxOYCRyI/P2pJH4eERGv8/LLsHUrrF8P2bO7nUZu\n2a+/Qp06kCsXfPed1pPwcUktnGpEfl2VhHNG4RRKb1trB11rNMbMBpbiFEGlItvuAaoCC6y1raNf\nxBjjD/gDWGtnG2OKEVk4WWt/TOLnEBHxSgsXwvjx8MEHUKWK22nkloWGQr16zo7MK1ZA3rxuJ5Jb\nlNSn6m6P/Pq/xBwcWew8ARzHmRsVxVr7DbACuNMY88B1p168/lrW2nBr7fkk5hUR8Rn790OnTtCy\nJfTo4XYauWUbNjjbqJQoAatWqWhKI1J6Hae7gQBgs7U2VjHEvz1XFSO/7gS2Am2NMauNMX2NMZUi\nn+QTEUmzLl925jXlygXTpmlek8/buNHZVLB8eaenKWdOtxNJMklq4XQk8muBRB6fLfLr0XjeP4zz\ndFw2AGvtVeBh4EOgBPAeEAIcMcYMMsZowU4RSZNeecXZrmzuXMiRw+00cstKlnS6D7/5BrJlS/h4\n8RlJLUTW4RQ6jyTy+DORX/PF8/7tOGsxXTsOa+1Ja+0L1tqCQBmgB85Q31CcJ/pERNKUL7+EcePg\nvffg3nvdTiPJIlcuZ6JaUJDbSSSZJbVwmglcBboZY264TpMxJiOwC2e+0n3GmIA4Dns48uuWuK5h\nrf3NWjsJqBfZ1DTa21cjv/olLrqIiPfZuxeefhpatHCW+BER75akwsla+zvOU3J5gW+MMUWvP8YY\nk8kY8yIwOHJJguDI41+77rhHcQqiPdbadZFtRYwxReK49bVJ6ReitZ3A6f0qlJTPICLiLc6fh1at\nnLUQZ8zQvCYRX3AzK4cPBDIBfYHfjDHfA7/ibLlSDKgD5AIGRB7/CvAfYGDk03Ob+Hcdp7M4a0Nd\nUxFYYIzZDOzAmVNVAGiO08M0Ntqxq3CG+d6JXMbgNHDKWjvhJj6TiEiqstZZEXzPHti0Ses1ifiK\nJBdO1loL9DPGfA48j7OW0kM4vVeHgW+AGdba7yOPDzPGVAUGAc2AB3GKnAXAMGvtjmiX/xkYAdTC\n2fMuB07x9C3wrrX2p2g5dhpjOgIv4Wzxkgk4CKhwEhGvN3kyzJ4Nn3wC5cq5nUZu2q+/aiPBdMY4\ndZBczxhTGQgJCQmhcuXKbscRkTTkp5/gwQfhmWecDXzFB1kLgwbBO+/Af/+r4smLhIaGUsVZPbaK\ntTY0ua9/M0N1IiJyk8LCnHlNlSrBmDFup5GbEhEBPXvCpEnw7rsqmtIZFU4iIqnk6lV48km4cAHm\nzYNMmdxOJEl2+bLzGOTcuTB1KnTp4nYiSWUqnEREUsnQobByJSxfDoX0PLDvOX8eWrd2NuqdO9fp\nOpR0R4WTiEgq+PprePNNGD4c6tRxO40k2alT0KQJ/PKL88PUDzHdUuEkIpLCdu92huiaNIFXX3U7\njdyUpUthxw6nt6laNbfTiItUOImIpKAzZ6B5c7j9dmfpAY923PRN7dpBvXqQJ4/bScRlKpxERFJI\nRAQ89RT89Rds3qxFLn2eiiZBhZOISIoZOhSWLHFepUq5nUZEkoMKJxGRFLBwIQwbBm+9BY0auZ1G\nRJKLRttFRJLZ9u3QoYPz5Prrr7udRpIkIsLtBOLlVDiJiCSjkyedyeDFisGMGWCM24kk0SZNcpYZ\nuHTJ7STixVQ4iYgkk6tXoW1bOH4cFi2CLFncTiSJcm3fue7doUIF8Pd3O5F4Mc1xEhFJJq+9BitW\nwDffQPHibqeRRLlyBZ57DqZNg1GjoF8/dRPKDalwEhFJBjNnOvu9jhkDdeu6nUYS5fx5eOIJWLYM\nZs921o4QSYAKJxGRW7R2LXTrBl27Qp8+bqeRRDl+3FnK/b//ddaLePRRtxOJj1DhJCJyC/bvhxYt\n4IEH4MMPNcrjMzp3hj17YNUqqFrV7TTiQ1Q4iYjcpDNnnE6L7Nlh/nzImNHtRJJo48ZBeDiULOl2\nEvExKpxERG7CtSfo/vc/2LgRcud2O5EkSbFibicQH6XCSUTkJrz8MixfDkuXwt13u51GRFKL1nFy\nkcfjSfTLz8/P7bg3LU+ePJQvX97tGCLJZupU5+m5ceOgXj2304hIavKJHidjTEbgTeBJIBewFRho\nrV2ZxOusAB4BPrTW9k72oEk0ZMiQWG1jx47lzJkzDBkyBGttVLvx4Rmnvpxd5Ho//ADPP++8evRw\nO42IpDafKJyA2UALYCywF+gILDXG1LLWrk/MBYwxLYHqgE3o2NTyxhtvxGqbMWMGZ86cYdCgQS4k\nEpEb2bnTeYKuVi14/309QefVzp1zNgzs2lVLDUiy8vqhOmNMVeBx4FVr7avW2qk4vUYHgVGJvEYm\n4D1gBODTf9VdvHiRsWPHUrduXQoVKkSmTJnInz8/bdq0YceOHbGOnzBhAh6PhwULFjB//nzuv/9+\nsmbNGmPoLDw8nCFDhlC8eHECAwO5++67GTduHL/++isej4fevWN3zh0+fJiePXtSokQJAgICyJcv\nH23btmX37t1Rx2zfvh2Px8OJEyeirnXtNWbMmJT5BomkkKNHoWFDKFDAeYJOu3J4sSNHnOp2+XJn\nOxWRZOQLPU6tgSvAlGsN1tpLxphpwHBjTAFr7V8JXOMVnILpPZwhP5/1119/8fLLL1OrVi2aNm1K\ntmzZ2Lt3L19++SXLli1j48aNlClTJsY5xhimT5/OypUradq0KQ8//DAXL16Mev+JJ55g4cKFlCpV\nit69e3P27FmGDx/OqlWr4hxm27FjB7Vr1yYsLIwGDRrQunVrDh06xIIFC/jmm29Ys2YN99xzD7fd\ndhtDhgxh5MiRZM2alR49ekQNP9aoUSNlv1Eiyej8eWfZgYsXYfVqZ/kB8VI7dzoV7qVLsGYNVKrk\ndiJJa6y1Xv0CvgV+jaO9NnAVaJTA+YWBc8Bjkb+PAMYn4r6VARsSEmJTU9GiRa3H44n3/QsXLthj\nx47Fag8NDbWBgYG2ZcuWMdo//PBDa4yxGTNmtBs3box13qJFi6wxxj744IP28uXLUe1//PGHzZkz\np/V4PLZXr14xzqlQoYINCAiwa9eujdG+bds2GxQUZGvUqBGjPU+ePLZcuXLxf2gRL3blirXNmlmb\nObO1qfzXgSTV6tXW5shhbdmy1h486HYacUlISIjFmZZT2aZAXeL1Q3VAfuBwHO2HcXqR7kjg/NFA\nqLV2XnIHc0NAQAB58+aN1V6pUiXuv/9+vvvuuzjPa9euHdWqVYvV/umnn2KMYejQofhHG3soVKgQ\n3bt3jzFBHWDt2rVs3bqV5557jgceeCDGe/fccw9PPfUUGzdu5I8//riZjyfidV56ydmRY84cqFzZ\n7TQSr+BgZ5PASpWcPXAKF3Y7kaRRvjBUFwhciqP9YrT342SMeRhnUnmaWk9/8+bNvPfee2zYsIFj\nx44RHh4e9Z4xhnPnzpE5c+YY59x3331xXmvr1q0AsYqg+No2btwIwB9//MHQoUNjvb9v3z4Adu3a\nRWH9xSU+7v33ndfEidCokdtpJF4//wzt2jmb9E6dqiXcJUX5QuF0AcgUR3tAtPdjMcZ4gPeB2dba\n0BTKlupWrlxJw4YNyZgxI/Xq1ePOO+8kc+bMGGOYM2cOu3bt4tKlSzEKJ2MM+fLli/N6Z86cIUuW\nLGTKFPtbHNc5J06cAGDRokUsWrQozmteK95EfNmiRdC3L/Tv7yw9IF7s3nvh66+hQQM96igpzhcK\np8PEPRyXP/LroXjO6wiUBLoZY4pEtl37LyprZNsxa22chdc1ffv2Jft1M0Hbtm1L27ZtExE9+b35\n5ptERESwfv36WItKrlixIt7z4ltLKVu2bBw7doxLly7FKp6OHj0a5/HGGGbOnMlTTz11E59AxPtt\n3ux0YLRuDSNGuJ1GEqVhQ7cTiAuCg4MJDg6O0Xb69OkUvacvFE5bgFrGmCzW2rPR2q+tybQlnvMK\nAf7A9es8WeBpoAPOMN7iG9187NixVPaiiQ379u2jUKFCsYqmM2fOsG3btiRfr0KFCuzdu5d169ZR\nu3btGO+tW7cu1vHVqlXDWsv69esTXTj5+flx9erVJGcTccO+fdC4sTNVZtYs8PjCTFCRdCqujozQ\n0FCqVKmSYvf0hb8S5uMUeN2uNUSuJN4R2GgjlyIwxhQyxpSKdl4wTmHU/LqXAb6O/PWmVMifrIoU\nKcLhw4fZv39/VNvVq1fp3bs3Z86cSfL1nnzySay1DB48OMZcqT///JNJkybF6qmqVasW5cqVY9q0\naXz11VexrhcREcGaNWtitOXKlYsjR44QERGR5Hwiqenvv521ErNnhy+/hMB4Z1CKSHrl9T1O1trN\nxph5wDvGmHz8u3J4EaBTtEM/AWoSWQxaa3cDu7lOZCGw31q7JGWTp4xevXrRtm1bqlatyuOPP06G\nDBn47rvvOHXqFDVq1GDDhg2xzrn+ybjomjVrRrNmzVi8eDHly5enadOmnD17lrlz5/LQQw+xePFi\nPNH+yW2MYe7cudSpU4emTZvy4IMPUrFiRTJmzMjBgwdZv3494eHhHDt2LOqc2rVrM2nSJBo2bEj1\n6tXx9/enTp06cT7lJ+KWs2ed0Z4zZ2D9esiTx+1EEkt4uFYeFdd5feEU6SmchSvbAzlx9qprZK2N\nPpZkcdZoSsi19R281o32dmvTpg3WWkaNGsWsWbPImjUr9erV4+2336ZPnz5xnpvQXnFz585l+PDh\nzJo1i/Hjx1OkSBEGDhxIhQoVWLx4MdmyZYtxfKlSpdiyZQujR49m8eLFTJs2jQwZMlCgQAHq1q1L\nmzZtYhw/fPhwzp8/zzfffMN3331HREQEgYGBKpzEa1y+DK1awW+/OQtcFi/udiKJISICXnsNfvkF\nli0DH970XHyfuVFvRHpmjKkMhISEhHjVHKfUNG7cOF588UU++eQTnnzySbfjiKSIiAjnKfb5853/\nJ1831U/cdu4ctG/vjJ2OHg19+ujJObmhaHOcqqTEU/W+0uMkKejo0aOxlh44cOAAI0eOJCAggAYN\nGriUTCRlWQv9+jlrJ86Zo6LJ6/z1FzRt6nQFfvmls++NiMtUOAkDBw5k3bp1PPDAA+TJk4cDBw6w\nZMkSLly4wOjRo8mVK5fbEUVSxHvvwdix8OGH8NhjbqeRGH76CZo3dx5rXLcOKlRwO5EIoMJJgKZN\nm0YVSydPniQoKIhq1arRq1cvmjdv7nY8kRQxeza8/DIMHAg9eridRmL47DPo0gUqVoSFCyF//oTP\nEUklKpyEJk2a0ERd4JKOLF0KnTvDM8/AsGFup5EYLlyAQYPgiSfgo48gICDhc0RSkQonEUlXNm50\nhuUaNYJJkzTP2OsEBsKmTc56EPrhiBdS4SQi6cbWrc52ZpUrw//9H2TQ34DeKW9etxOIxMsXVg4X\nHzJkyBA8Hg8//vij21FEYti9G+rWddZo+uorrQouIjdHhZMkK2NMrAU3Dx48iMfjoXPnzi6lkvTu\njz+gTh3InRuWL3e2VBERuRkqnCRZ9erVi507d1K1alW3o4gAcPSoUzRlyAArVmgrFa9w6RKoV1p8\nlEb4JVnlypUr1rpPWp1e3HLyJNSr5+xDt3YtFCjgdiLh0CFo3Rp+/RUOHoScOd1OJJIk6nHyAl98\n8QX/+c9/yJcvH4GBgVF7vi1YsACIOdS1Y8cOGjVqRM6cOcmaNSv169cnNDT2ivKhoaH07NmTcuXK\nkSNHDoKCgihfvjwjR47kypUrsY4vWrQoxYsX5/Tp0/Ts2ZPChQvj7+/P7NmzAThy5AgvvPACJUuW\nJCgoiJw5c1KmTBmef/55/vnnn6jrXD/HadasWRQvXhxjDDNnzsTj8eDxePDz8+PHH39k4MCBeDwe\nvvjiizi/N9OmTcPj8TBy5Mhb/j5L+nJt096//nJ6mrT/nBdYuxaqVHHGTlesUNEkPkk9Ti6bNGkS\nPXr04I477qBly5bkzp2bI0eOsHnzZhYtWkTLli2jjv3999954IEHqFKlCt27d+fgwYPMmzePmjVr\nsmrVKu67776oY6dMmcJXX31FzZo1adSoEefPn+eHH37gtdde4+eff2bevHkxchhjuHTpErVr1+bc\nuXM0a9aMDBkykC9fPi5cuECNGjX4448/qFevHi1btuTy5cvs37+fTz/9lP79+5M1a9ao60Sf41Sx\nYkX69OnDuHHjqFixYowFNYsWLUq3bt0YMWIEU6ZMoVWrVrG+P1OnTsXf35+OHTsm17dc0oGLF6FZ\nM9i+HVatgrJl3U6UzlkLEyc6+8zVqAFz58J12zyJ+AxrrV5xvIDKgA0JCbEpqUqVKjYgIMCGhYXF\neu/EiRPWWmsPHDhgjTHW4/HYAQMGxDjm22+/tcYYW6FChRjtf/zxh42IiIh1zS5duliPx2PXr18f\no71o0aLW4/HYhg0b2osXL8Z4b8mSJdYYY1966aVY1zt37py9fPly1O+HDBliPR6PXb16dVTbtfyd\nOnWK83vQsGFD6+fnZw8ePBijfceOHdYYY1u1ahXneSJxuXzZ2qZNrQ0IsDbaH0Nxy/nz1nboYC1Y\n+8ILzg9IJAWFhIRYwAKVbQrUBxqq8wL+/v74+fnFas95XTd2jhw5eP3112O01a1bl0ceeYRt27bx\nyy+/RLUXKlQo1tNtAN27d8day8qVK+PMMmrUKDJlyhTnewFxrOAbFBSEv79/nMcn1nPPPUdERATT\npk2L0T558mSMMXTt2vWWri/px5Ur8NRTsGwZLFgANWu6nSidCw93fgjz5sEnn8C4cXCLf1+IuE2F\nk8ueeOIJzp07xz333MPLL7/MsmXLYswZiq5SpUoEBQXFan/ooYcAYhRO4eHhjBkzhmrVqpE9e3b8\n/PzweDzce++9GGM4dOhQrOsEBARQNo4xjZo1a5I/f35GjBhB48aN+eijj9i5c+fNfuRYGjVqRMGC\nBZkxY0bURPLw8HA+++wzChUqRP369ZPtXpJ2Xb0KHTvC/PnO4pYNGridSPD3d/a2Wb8e2rd3O41I\nslDh5LJ+/foxbdo0ChQowJgxY2jUqBG5c+emRYsWHDhwIMax+eKZE5AvXz6stZw+fTqqrVWrVvTr\n148zZ87wxBNP8PrrrzNkyBD69OmDtZZLly7Fus5tt90W5/WzZcvGpk2b6NChA5s2baJHjx6ULVuW\nIkWKMGnSpJv/8JE8Hg9dunThr7/+YtmyZQAsWLCAsLAwunTpcsvXl7QvIsLZdy44GD7/HKJNDRS3\nPf+8s1mvSBqhwskLdOzYkU2bNvH333+zaNEiWrVqxZdffkmTJk1iPMp/9OjROM8/evQoxhiyR67q\n9/PPP/PVV1/RoEEDduzYwccff8ybb77JG2+8QZs2beLNEdfQ3jUFCxZk+vTpHDt2jF9++YVRo0Zh\nraVnz57MmTPnJj/5v7p27YrH42HKlCmAMyncz8+PTp063fK1JW2LiIDnnoPZs53RoMcfdzuRiKRl\nKpy8SM6cOWnatCnBwcHUrl2bHTt2sHfv3qj3f/nlF86fPx/rvGuP/leqVAlwnr4DaNiwYaxi6Fa3\nQjHGUL58efr168fnn3+OtZbFixff8Jxr87euXr0a7zF33HEHjRo1YunSpWzYsIHvv/+eRx99lIIF\nC95SXknbrIWePWHqVJg+Hdq1czuRiKR1Kpxctnr16lht4eHhHD9+HCDGRO1Tp07x1ltvxTh2+fLl\nfP/995QrVy6qcCpSpAgAa9eujXHs9u3bGTFixA17luKyY8cOjh07Fqv9yJEjQNyTxqPLmTMnxhj+\n/PPPGx737LPPEh4ezmOPPQagSeFyQ9Y6T7dPmgRTpsDTT7udKB07d87tBCKpRus4uax58+Zky5aN\n6tWrU6RIEcLDw1mxYgU7d+7kscceo3Dhwhw8eBBwJoF/9NFHbNq0ierVq7N//37mz59P5syZmTp1\natQ1q1atStWqVZk7dy6HDh2ievXqHDx4kCVLltC4ceNYazglZMWKFfTv358HHniAkiVLkjt3bvbt\n28fixYsJDAykR48eNzw/c+bM3Hffffz444906NCBu+66C4/HQ4cOHShUqFDUcY8++ihFihTh4MGD\n3H777TRu3DhJOSX9sBb694fx453CSVPhXHLpErz4IqxbB5s2QTxP5IqkKSmxxkFaeJFK6zh99NFH\ntnnz5rZYsWI2KCjI5s2b11avXt1OnjzZXrlyxVobcx2kHTt22MaNG9scOXLYrFmz2vr169tffvkl\n1nXDwsLsM888YwsWLGiDgoJshQoV7EcffWT3799vPR6P7dy5c4zjixYtaosXLx5nxp07d9q+ffva\nKlWq2Lx589rAwEB755132s6dO9udO3fGODaudZystXbPnj22cePGNleuXNbPzy/OY6y1dtCgQdYY\nE2u9KpFrIiKsffVVZ1mg8ePdTpOO7d9v7b33Wpsxo7WTJjk/GBEvkNLrOBlrtY9YXIwxlYGQkJAQ\nKleu7GqWgwcPUqxYMTp27Mj06dNdzZLSGjduzDfffMPu3bsprj0y5DrWwoAB8M47MHq009khLvjq\nK+jQAbJnd9Z/qFLF7UQiUUJDQ6ni/JmsYq2NvSfZLdIcJ/EaO3bsYNmyZdSrV09Fk8RiLbz8slM0\nvfuuiiZXXLkCr78OTZrAgw9CaKiKJkl3NMdJXBccHMyuXbv45JNPMMYwePBgtyOJl7EW+vaF9993\nFp9+4QW3E6VDx49D69awZg2MHAn9+oFH//aW9EeFk4+4fvPctGTy5MmsXbuWIkWKMH36dKpVq+Z2\nJPEiERHQuzdMmOC8und3O1E6lTkzZM0K330H//mP22lEXKPCyQcUKVLkhmsg+bpVq1a5HUG8VESE\ns/D05Mnw8cfQrZvbidKxgABIYM02kfRAhZOIeKWrV51CacYMZ3FLLSIvIt5AA9QJSGDNRhFJAVev\nOoXSzJkwa5aKJhHxHiqcEnDokNsJbt62bdsoWLAg27ZtczuKSKJdueI86f755/Dpp/DUU24nSkfC\nw50fgIjES4VTAi5ccDvBzcuTJw/PPPMMefLkcTuKSKJcuuRs0jtnDgQHQ9u2bidKR/bvd5YYGDbM\n7SQiXk1znBIQx566PiN//vwMGTLE7RgiiXLuHLRsCatXw8KFzlJBkkrmz4dnnoFcuUBbHYnckHqc\nEnDxotsJRNK+06ehfn1ny7OlS1U0pZoLF5zHFh97DOrVg19+gapV3U4l4tXU45QAXx6qE/EFf//t\nFE3798PKlVC9utuJ0olt26BdO9i711nroWtXSKNrxYkkJ/U4JcCXC6eTJ08yb948Tp486XYUkTj9\n9RfUrOk8hLF6tYqmVPPZZ3Dffc6vN2921n1Q0SSSKCqcEuDLc5z279/P448/zv79+92OIhLLvn3w\n0EPO3KYff4Ty5d1OlI6UKeMswf7TT1CunNtpRHyKhuoS4Ms9ThUqVOD06dNkzpzZ7SgiMWzfDnXr\nQpYs8MMPULiw24nSmUqVnJeIJJl6nBLgy4WTn58f2bJlw8/Pz+0oIlF++snZ6ixvXme/WBVNIuJL\nVDglQE/ViSSf5cvh4YehZEmnpylfPrcTiYgkjQqnBPhyj5OIN/nsM2eJoFq1nKfncuZ0O1EadvWq\nM+lbRJKdCqcE+PLk8N9++43777+f3377ze0oks6NHQvt2zuvhQshKMjtRGnYgQNQp47zuOKRI26n\nEUlzVDglwJd7nAICAihbtiwBAQFuR5F0ylp45RV48UV49VWYPh38/d1OlUZZC9OmOU/J7d8Py5bB\n7be7nUokzdFTdQnw5cKpSJEiTJ061e0Ykk6FhztrKs6a5fQ49enjdqI07PBh55v99dfQpQuMGQPZ\nsrmdSiRNUuGUAE0OF0m68+edzXqXL3fmNrVr53aiNGzOHGdNpowZYckS7TUnksJUOCXAl+c4ibjh\n+HFnr7mtW50OkHr13E6Uhu3e7VSlrVvDxImQO7fbiUTSPBVOCfDlobqzZ8+ybds2ypUrR5YsWdyO\nI55lNHoAACAASURBVOnA779DgwZw8iSsWvXvrh6SQkqWhC1btPq3SCrS5PAEhIc7T/b6ot27d1Oj\nRg12797tdhRJBzZudPaaM8b5tYqmVKKiSSRVqXBKhHPn3E5wc0qXLs2vv/5K6dKl3Y4iadyCBc7C\nlnffDevXQ4kSbicSEUkZKpwS4exZtxPcnMDAQMqWLUtgYKDbUSSNstZ5Yq51a2jWDFas0DSbZHfl\nitsJRCQaFU6J4Ks9TiIp6epVeOEFZ42m/v3h889BS4Yls8WLnXlMu3a5nUREIqlwSgQVTiIxnT8P\nrVrBhAkwaRKMHAke/W2SfMLCnKflmjWDMmUga1a3E4lIJP1Vlwi+OlS3b98+HnvsMfbt2+d2FElD\njhxx5jOtXOksG/Tcc24nSkOshblznWJp+XL49FPnm1yggNvJRCSSCqdEOHXK7QQ35+rVq5w5c4ar\nvvpYoHid//4XqlaFP/+EH3+Ehg3dTpSGHD7sdOO1aQP/+Q/s2AFPPuk8pigiXkPrOCXC8eNuJ7g5\nd911F8uXL3c7hqQRixc7o0elSjm/VidIMjp50llWwM8P5s93CigR8UrqcUpAUJAz3UAkvbIWRo2C\n5s3h0UedniYVTcksZ054/32nl0lFk4hX84nCyRiT0Rgz0hjzP2PMeWPMRmNMnUSc19IYE2yM+d0Y\nc84Ys8sY854xJnti7509u+/2OIncqkuXoHNneOUVGDDAmX6TObPbqdKoJ5/UWg4iPsBXhupmAy2A\nscD/t3ffcVKV1+PHP0ekSVVBBGkqqBRlWRXFgoAoQdCgBiMBFZEYv/nFgl1sKCaWGGJPCPaGCgYj\nKk1AUSwgC0qRoqgUaSq9LSzn98e5K8Mwuzs7s7t3Z+a8X695ze6d+9w5M5edOTzPc8/zDdAPeE9E\nOqrqJ4W0GwasAF4ClgLHAn8BuolItqruKOqJa9dO3R6n3Nxc1qxZwyGHHEKlSpXCDselmLVrrfNj\n+nRfqNc55/KV+8RJRNoBFwE3qOo/g20vAXOBh4DTCml+oapOjTpeDvAC0Ad4tqjnr107dXuc5s6d\ny/HHH8/MmTPJzs4OOxyXQubNs4V6t2yxNefatw87ojSwaxfsX+4/cp1zRUiFobrfAbuA4fkbgp6i\nZ4D2IlLgbIvopCkwOriPax2SVO5xatasGePGjaNZs2Zhh+JSyJgxlijVqGG9TZ40JUkVRoyA5s3B\nS4M4l/JSIXHKAhapanQ1pekRjxdH/eA+rnQolec41axZk65du1KzZs2wQ3EpYPduGDIEzjsPunSB\njz+GJk3CjirFLVkC3brZOOcJJ4Avf+RcykuFxKk+sDLG9pWAAA2KebxbsB6sUfHsnMo9Ts7Fa9Mm\nm890112WPI0a5cWqk7JtG9xzjxWynD/fuvFGjoT69Ytu65wr11JhwL0qEGsS9/aIx+MiIn8A+gMP\nqOq38bTJ73FS9Tp0Lj0tXmylBpYts/pM554bdkQp7t134Zpr7A298Ua7HNEvRXQubaRC4rQNqBxj\ne5WIx4skIqcDTwNjgTviffK33x7Irl21OOccqFjRtvXu3ZvevXvHe4jQLF++nKFDh3L99dfTsGHD\nsMNx5dDYsdC7N9SrZ/OZjjkm7IhS3Pz50KMHnHUWvPeeVQt1zpWaESNGMGLEiL22bdiwoVSfU1S1\nVJ8gWSIyAWigqq2jtncG3gfOVdV3izhGG2AKsAjorKpb43jebGDmv/41k//7v2y+/RaOOCLhlxGK\n+fPn06tXL0aOHEnLli3DDseVI6rwwAPWGdK9uy2JVivu6mauUNOnw4knehe1cyHJycnh+OOPBzhe\nVXNK+vipMMdpNnCUiFSP2n4yoMHjBRKRI4FxwCrgnHiSpki1a9v92rXFaVU+tGzZknnz5nnS5Pay\nebMthzZoENxxB/zvf540lah27Txpci6NpULiNAobUrwyf4OIVMKKYH6mqiuCbY1EZK9+cRGpB0zA\nJoP/RlV/Ke6T5xfyXbUqseCdK0/mz7fOkLFj4c034d57Yb9U+BRwzrlyotzPcVLV6SIyErg/SITy\nK4c3AS6P2PUloAN7J4PjgaZYoczTZe//Ba5W1feLev4DD7S5TcuXJ/MqnAvfa6/BgAFWYmDGDJ/P\nlJDNm2HmTDjjjLAjcc6FpNwnToFLgCFAX+BA4Cugu6pOi9hHgd1R7Y4N7m+OccwPsTlShdpvP1vQ\n1BMnl6pyc+GGG+CJJ6yc0LBhUD164NsVbvdueOkluO02qwC+dClUqVJ0O+dc2kmJTnpVzVXVW1T1\nMFU9QFVPju4tUtVOqrp/1LYKhdw6x/v8DRumZuKUk5ODiJCTU+Jz41yKWLbMOkeGDYMnn7RJ4J40\nFdMnn8DJJ0O/ftChg3XXedLkXMZKicQpbKmaODVu3Jjhw4fTuHHjsENxIZg4EbKzYcUK+Ogj+POf\nfc5ysSxbZl10p55qPU4ffWTjnV5O3bmM5olTHFI1capTpw4DBgygTp06YYfiylBenlX/7trVEqec\nHDjppLCjSjHDh1sNpilT4NlnrcTAaYWtJ+6cyxSpMscpVPmJk1cPd+XdqlVwySUwaZItn3LnnVCh\nQthRpaAjjoBrr7WaDb72jHMugidOcWjYELZvh19+2VOewLnyZuJE6NvXkvuJE+HMM8OOKIWdeaa/\ngc65mHyoLg75q5UsWxZuHMW1evVqhg4dyurVq8MOxZWiXbusY6RrV2jTBr780r/znXOutHjiFIem\nTe3+u+9CDaPYVq5cyeDBg1m5cmXYobhSsnSpXTX30EPwt7/BuHG27pwrwrJlsHNn2FE451KQJ05x\nOOQQm+aweHHYkRRPVlYWGzduJCsrK+xQXCl4+23IyrL5d1Onwq23ehXwIq1fD7fcAs2bwzPPhB2N\ncy4F+cdsHETsczbVEieXnrZvt3nLv/2tlRWaNQtOOSXsqMq5HTvgkUfgyCOtEugtt0CfPmFH5ZxL\nQT45PE6eOLnyYM4c+75fuBAefRSuvtqv9CzUrl3wwgu2KN/y5bbmzODBUL9+2JE551KU9zjFqVkz\nT5xceHbvtg6TE0+0shgzZsA113jSVKhFi6BlS0uWTjoJ5s2zEuqeNDnnkuCJU5yaN4cff4QtW8KO\nJH5z5syhYcOGzJkzJ+xQXBJ+/BF+8xsYOBCuusqSpuOOCzuqFNCkiRWtzMmBN97wVY2dcyXCh+ri\n1Ly53X/zjV3ynQq8cnjqGz0a/vhHqFgRxo+Hs88OO6IUUrmyVf12zrkS5D1OcWrRwu7nzw83juKo\nX78+gwcPpr4PTaSczZtthOmCC+D0021ukydNzjkXPk+c4nTggdCoEXz1VdiRuHQ3bRq0bQsjRtiS\naf/9L3inYQzTpsHnn4cdhXMuw3jiVAzHHeeJkys927bBjTdaD1PdujB7tvU6+QTwKFOnWmn0006D\np54KOxrnXIbxxKkY2rRJrcRp3bp1jBw5knXr1oUdiivC9OmQnQ2PPw4PPggffbRnXp3DLiWcPBk6\ndrRS6T//DG++Cc89F3ZkzrkM44lTMRx3nJWC+eWXsCOJz3fffcdFF13Ed6m2VkwG2bEDbr8d2reH\n6tWtmOVNN0GFCmFHVk6o2orFHTpYL9OmTfDWW/ZGXXCBl0p3zpU5v6quGPKvpps9Gzp3DjeWeLRp\n04YNGzZQrVq1sENxMcyeDZdeCgsWwD33WDHrihXDjqqcWbcOevaE1q3hnXfgnHN87NI5FypPnIrh\nqKOgZk2bj5oKiVOFChWoWbNm2GG4KLm58MADMGSI1WecMSN1SlyUuYMOsgyzWTNPmJxz5YInTsWw\n337Qrh189lnYkbhU9fnnNuH766/httvgzjuhUqWwoyrnfLKXc64c8QkCxXTyyZY4qYYdiUslW7ZY\n5e/27S1R+uIL63HK+KRp3TpYsSLsKJxzLm6eOBXTySfDmjWQCvOtFy5cSPv27Vm4cGHYoWS0CRNs\nis6wYfDQQ9brlJUVdlQhW7YMrr/eiqMNGhR2NM45FzdPnIqpfXubavHRR2FHUrQqVarQqlUrqlSp\nEnYoGennn6FfP+jaFY44wqp/33gj7J/JA+Rz5sDll9sb8txzcN118Pe/hx2Vc87FLZM/whNy0EE2\nkXfyZLjssrCjKVyTJk14+umnww4j46jC66/DtdfaRPBnnrFcIWPnNu/eDe+9B488ApMmwWGH2ez4\nK6+EGjXCjs4554rFe5wS0LkzTJni85zcvhYvht/8Bnr3tgrg8+dD//4ZnDSBrYx97rlWg2nECBvn\nvuEGT5qccynJE6cEdOpkUzS++SbsSFx5sW0b3H23zWVavNhKDo0aBb6+MlbHY948u6ri4ou9WJVz\nLqV54pSAM86wq6HGjg07ksJt3ryZTz/9lM2bN4cdSlobNw6OPRbuvx9uvhnmzoXu3cOOqpxp2TLD\nu92cc+nCE6cE1KhhS2a9807YkRRu0aJFnHLKKSxatCjsUNLS8uXQqxd06wZNm9q85yFD4IADwo6s\nDG3ebJcL/vhj2JE451yZ8MQpQT16wAcf2LSN8qpFixbMnTuXFi1ahB1KWsnNhX/8A1q0gI8/tmk7\nEyfC0UeHHVkZmj8frr4aGjSAP/8ZPvww7Iicc65MeOKUoHPPhZ07YcyYsCMpWNWqVWnVqhVVq1YN\nO5S08d57Nix38812pdyCBTZtJyNGoXbuhJEjbZJfq1bwxhtwzTXw/fc2G9455zKAJ04JatoUTj0V\nXn457EhcWViwwIbkuneHhg1t+bTHHoNatcKOrIyMGAFNmsBFF0Fenv2+bBncd58VsXTOuQzhiVMS\n+va1qtCrV4cdiSst69fbUinHHguLFsHo0fD++/Z7RqlbF3r2hC+/hKlTrZst49eLcc5lIk+cktCr\nly38+/rrYUcS25IlS+jVqxdLliwJO5SUk5dnc56bN4fhw23S97x5ljtkxLBctC5d4Kmn4Ljjwo7E\nOedC5YlTEg4+GM45p/wO1+Xl5bFx40by8vLCDiVlqFp5gexsuOoqG5pbtAhuvRXSduWaDRusxpJz\nzrkieeKUpL59YcYM640ob5o3b8748eNp3rx52KGkhJkzrWOlWzebu/T55/D883bhWNrZuRPefdeG\n3A491LrSPMF2zrkieeKUpPPOs+rQjzwSdiQuUUuW2EVhJ5wAq1bB22/b1fXt2oUdWQlThVmzbNJW\nw4ZWU2PePLj3XssaK1QIO0LnnCv3PHFKUqVKtpjriy/al65LHT/9BNddB8ccY/Odn37a5j6fe24a\nzmNas8bmJ2Vnw6uvQp8+lkR99RXcdJMtvOucc65InjiVgD/9yRKoJ54IO5K95ebmsnz5cnJzc8MO\npVzZtAn++lc48kh49lkYPNjWl7viCth//7CjKyV168KZZ1ohqhUrYOhQyMpKwwzROedKlydOJaB2\nbRgwwC46Kk/Lws2dO5dGjRoxd+7csEMpF7Zuhb//HY44wkan+vWDb7+FQYMyYJkUERtP7tYtjbND\n55wrfZ44lZCBA2HLFluKo7xo1qwZ48aNo1mzZmGHEqrt2+HRRy1hGjQILrwQvvnGttWtG3Z0Sdq4\n0S7r7NnTfnbOOVeqPHEqIY0b21ynhx4qP+ud1qxZk65du1KzZs2wQwnFjh3WC3jkkXDDDVY6YtEi\n+Pe/U7zY9dq18MwzNrn7kEPgkkts28qVYUfmnHNpzxOnEjRoEFStCnfdFXYkmW3rVlsOpVkz+Mtf\noHNn+Pprm890+OFhR5cgVZtE17GjlQ/44x+th+n+++GHH2DatAxbZdg558Lhkx1KUO3acPfd1vP0\npz/BiSeGHVFm2bDBepj++U/45Rf4wx/gttugRYuwIysBIrbeywEHWEnz886z3ibnnHNlynucSthV\nV0HbtjbxePv2cGNZvnw5119/PcuXLw83kFK2di3cfrsNlw4ebHOYFi+2EhFpkTTlmzjRroobMMCT\nJuecC4knTiWsYkV44QX74h48ONxYNm7cyPjx49mYppOGv/0WrrkGmjSxid5XXgnffw//+lcKDckt\nXWoB9+hhM9YLs5//uTrnXNh8qK4UtG4N99wDd9wBXbtCp07hxNGyZUvmlce1YJKgCh9/bGWI/vc/\nWy/wppssgTr44LCji8P27TYfaeJEW/Jk7lyr2H3aaTbW6JxzrlzzxKmU3HQTTJoEF10EX3xhvSIu\ncTt3wsiRljDNnGlDcMOG2VqBVauGHV2cfv97W89l+3aoV8+y6jvvhLPPtglyzjnnyj1PnErJ/vvD\na6/Z+mcXXGBLelSrFnZUqWfVKrsa7qmnrOD12WfD2LF2n3IjV23awEknwVlnWbekV+12zrmU44lT\nKapTB956y0Zhzj8fxoyBypXDjqr8U4UpU6ze0ujRNm/sD3+wdeVatw47uiiqNjdp6lSrp1SpUsH7\nDhpUdnE555wrFan2f/aUk5VlCdNHH8HFF0NZLhuXk5ODiJCTk1N2T5qEn3+2obhjjrFl1ebOtUrs\nK1bYArzlImnavdsCe+opG3pr0ACOOspmps+fH3Z0zjnnSpn3OJWBTp1g1CjrdTrvPPu5evXSf97G\njRszfPhwGjduXPpPlqC8PHj/fSsd8Oab1oFz4YUwfDicfno5Gs3avt1O4Oefw7p11g124olWd6JD\nBzj1VMjQCu3OOZdJPHEqI927w7hxtqRYp042hHfYYaX7nHXq1GHAgAGl+yQJmjvXkqWXX7aVQlq0\nsIV3L7+8nK4fV6WK1U667jpLktq3z4CVgZ1zzkXzxKkMde4MH35ovU5t28KIETYklSl++MF62159\nFXJyrHxA795w2WVw/PFl3LuUm2tDazk5dtuwAV56qfA2L7xQNrE555wrt1JijpOIVBKRB0VkuYhs\nFZHPRKRLnG0biMgbIrJORDaIyFsiElp5xLZt7Xu6TRu7uOqGG2xttXT1/ffw8MN2MVnTplbhu0kT\nm/T944/w+ON25WGpJ00rVths8yuvtCesUcNOxoABVjdiv/1snNA555wrRKr0OL0InA/8E/gG6Ae8\nJyIdVfWTghqJSDXgA6AGcB+wC7ge+EBEslR1XSnHHVPdujZsN3SolfF5+237uUePkk0gVq9ezSuv\nvEKfPn2oV69eyR24EKrw5Ze2Msjo0VbDqkoV6NbNepq6dw9pKtDcuXD11dCqFWRn29yk7Gw47riy\nmXDmnHMuPahqub4B7YDdwMCIbZWBxcDHRbS9GcgDsiO2HQ3sBO4rom02oDNnztTStGCBapcuqqDa\nqZPq5Mmqu3eXzLFnzZqlNWrU0FmzZpXMAQuwfr3qqFGq/fur1q9vr6V6ddULL1QdMUJ148bkn+PV\nV1+1H/LyVH/4QfX991Wfekp14EDVHj1UX3ih8APs2KG6bVvygbg958KFzs9F+eLno3yYOXOmAhr5\n3V+St1QYqvsd1lM0PH+Dqu4AngHai0hhU6wvBGao6q/X46vqQmAScFFczz5xIsyYAWvWlMpQztFH\nw4QJ8M47djl+5852sdZrr8GOHckdOysri40bN5KVlVUywQa2bLG35Y47rEZVnTrwu9/Bp59avaVJ\nk+y1jBplJRhq1Ej+OUfccQcce6xVEW3SBLp0sR6kMWOsREBRBbIqVbKuL5e0ESNGhB2CC/i5KF/8\nfGSGVBiqywIWqermqO3TIx5fEd1IRAQ4Dkuwok0HzhKRaqq6pdBnv/XWPT9XrWoTdZo0sQJN998f\n72solIgNYZ1zjiUkDz1kk6Zr17aEpHdvS1AKq61YWlRtMd0vvrDbJ59YHrlrlyVMZ5xhC+x26xbn\nwrrbt1s58JUr7bZ8OaxdC0OGFN5u//3tsv8rroDmza12UtOmVhbAOeecKyOpkDjVB1bG2L4SEKBB\nAe0Owob0CmpL0HZxoc8+eTLUqmWXhH3/vd1++AGWLSs68uuu25NhHHzwvrdDD91roTURW0rk7LPh\n66/hlVdsXtDTT1tHS8eOdhXeCSdY3lYSPTn5VC1/WbDAnnvBApgzx9aFW7/e9mnaFNq1s/XhOna0\nEgKSG3SLFdbjM2mS9Q6tXLnnYPkqVbJE9M47C88Mjz4annwymZfonHPOJS0VEqeqQKxBq+0RjxfU\njgTb7lGrlk0izs4uctd9rFgBCxfCTz/Z2FV02fChQ2HgwJhNW7SA+y7/liFbnmTllhosWF6d+Ysq\ns2B8Jb7aVZlcKnNgvUp837oHDZtVoXFjOOggOPBA66mqXt0SscorllBx3WrydtkQ2+ZNyvaNuWxZ\nl8u6NTtZurE2E7eeyg8/WF1HgAoV4MgjoWVLuPFGuHTalRwia6mcuxmWboanNsEDG+01bd0KzzwD\n/fsX/D4ceqhdQli//t63Qw+1BDLlFp1zzjmXqVIhcdqG9RxFqxLxeEHtSLDtr/t8/fXXRcVXsNtu\n2/OzKmzbZvWC1q+3+0aNrDZBQebMgbdGw5Yt1N62jVN27qR9Xi5CMNdqNdzZbBKTJ9dmzRrYtMmm\n+0S6lb/RizcBOCC4Rapb63iWdPkPp55qPUqHHw4NG+49ArZ2wkLW5uVZwcdDDrEdq1WzpLJWLUt+\nilrW5bLL9v59507rtYun5w7YsGFDyiwdk+78XJQffi7KFz8f5UPE93apTGwVLee1a0RkAtBAVVtH\nbe8MvA+cq6rvxmgnwFbgGVX9S9Rj9wK3A7VizJ3K3+cPwCsl8yqcc845V8b6qOqrJX3QVOhxmg10\nFJHqUUnOydjlhrNjNVJVFZE5wAkxHj4JWFJQ0hQYD/QBvmfP0J5zzjnnyrcqQFPse7zEpUKPUzvg\nM+BGVR0abKsEzAXWquqpwbZGwAFBuYH8tjcD9wMn5pckEJGjg7YPqertZfpinHPOOZfSyn3iBCAi\nrwM9gUfYUzn8BKCzqk4L9vkA6KCq+0W0qw7MwiqHP4zVgxqIXY3XVlV/LrtX4ZxzzrlUlwpDdQCX\nAEOAvsCBwFdA9/ykKaBYhfE9G1Q3i8gZ2FItt2Nr800BrvekyTnnnHPFlRI9Ts4555xz5YEX0HHO\nOeeci1PGJU4iUklEHhSR5SKyVUQ+E5EucbZtICJviMg6EdkgIm+JSDwLjbgYEj0XInKBiIwQkW9F\nZIuILBCRh0WkVlnEnY6S+buIOs5EEdktIo+VRpyZINlzISK/F5FPRGRz8Fk1TUQ6lmLIaS3J74wu\nIjJZRNYG5+JzEelb2jGnKxGpJiL3iMhYEfk5+Ky5tBjta4nIf0RkTfD3MVlE2hY3joxLnIAXgeuA\nl4FrsAnj74nIKYU1EpFqwAfA6cB9wF1AW+ADETmwNANOYwmdC2AY0AJ4CbgaGAv8BfhERIpY7dcV\nINFz8SsRuYA9ZUJc4hI+FyIyGHgVWIpdCHM78CVQ2GLornCJfmech10OXxG4GxiE1RZ8UUSuLdWI\n01cd4E7gGKwUUdyfNUFtx/eAi4HHgJuAuth3+JHFikJVM+YGtMMmkA+M2FYZW6/u4yLa3gzkAdkR\n244GdgL3hf3aUu2W5LnoEGPbJcHx+of92lLtlsy5iNp/CfZFvRt4LOzXlYq3JP8uTg4+o64J+3Wk\nyy3J8zEeWAbsH7GtQtB2VtivLRVvWBJ6SPDz8cG5uTTOthcF+58fsa0O8AvwcnHiyLQep99h/1sY\nnr9BVXcAzwDtRaSw/5VdCMzQoB5U0HYhMAk7Ia54Ej4Xqjo1xubRwX2LkgwyQyTzd5HvFqzMx8Ol\nEmHmSOZcXAesVNXH4NdecpecZM5HTWCdqu6KaJsH/EThy325AqjqTlVdk2DzC4FVqpr/XYGq/gS8\nAfxWRCoW2DJKpiVOWcAi3bdi+PSIx/cRdPEdB3wR4+HpwJH+IVVsCZ2LQtQP7n9KKqrMlNS5EJHG\nWOJ0c/Cl4hKXzLnoDMwQkWtFZC2wSUR+FJH/VxqBZohkzscHQCsRuVdEjhSRI0TkTqyn5KGSD9UV\noS0QayHB6dgyrkfFe6BUqeNUUuoDK2NsX4n9b7lBAe0OwrpnC2pL0HZxsgFmkETPRUFuwf5nOCrJ\nuDJRsufiH0COqo4s6cAyUELnQkRqY8MOp2EJ1GBsmOhy4HERyVXV4bHaukIl87dxL3A4Nnx9R7Bt\nC3Chqo4pySBdXOoDH8bYHvkdPi+eA2Va4lQViPU/4u0RjxfUjgTbutgSPRf7CBZk7g88oKrflkBs\nmSbhcyEinYDzsbkgLnmJnovqwf1BwO9VdRSAiLwJzMG+uD1xKr5kPqdygUXASOC/2PymK4FXRKSL\nqk4vpK0reYWdS6EY3zmZljhtw3qOolWJeLygdiTY1sWW6LnYi4icDjyNXVl3RxG7u9gSOhcish/w\nKPBi5Nw/l5RkP6N2Am/mb1RVDZasGiwiDVV1eYlFmhmS+Zx6Eminqtn5G0RkJNar8SjQvqSCdHEp\n7FwqxfgOz7Q5TivZMxcmUv62Hwto9wuWqRbWNlZ3ritYoufiVyLSBvgftgRPL1XdXUQTF1ui56If\nNi/gPyLSJLg1DR6rEfzuPbHFk8xn1HbgZw0uF4qQP5nWy6YUX0LnI5ho3B94N3J7MFF8LHBCcSYj\nuxKR9HdOvkxLnGYDRwWL/0bKrz0zO1aj4INoDrawcLSTgCUxJg+6wiV0LvIFdTfGAauAc1R1a6lE\nmRkSPReNsMuDPwG+C25LgjaXBT+fVRoBp7FkPqNmA3VFJHokIf/Kr7UlGWiGSPRv42BsRKdCjMcq\nYt+9mfb9G7bZQHaM7Sdj9bUWxXugTDtxo7B/zFfmbxCRStj/nD9T1RXBtkYicnSMtieKSGS369HY\nRMw3SjnudJTwuRCResAEbDL4b1T1l7IKOk0lei5GYPObekbdBPufdk/g8zKIP50k8xn1OvZFfVlE\n2ypAH2Ceqq4q3dDTUqLnYw2wHjg/MpENErBzga/9CtTSIyKHisjRIhKZuI4C6gWFevP3q4OVnHhb\nVXfGffx9e3XTWzDe3xN4BPgG+wM4AeisqtOCfT7AiizuF9GuOjALqIHVqtmFVeYVoK2q/lx28dNW\n+wAACgNJREFUryI9JHEuZgPHYpf0zo067GpVfb/Ug08ziZ6LAo61G3hCVa8pzZjTVRJ/F1WAGUBz\nrDLyUuBS7JL5Hqo6oexeRfpI4nwMAoZgPR0vYgnYFVjh5D6q+lrZvYr0EZTXqI31pF6FTbyfFTz8\nmKpuEpHnsX/7TVV1adBuP+BjoBX2Hf4T8GegMXCCqsZ/VXzYlUDL+gZUAh4EVmDdc58BXaL2mQLs\nitG2Afa/unXABuAt4IiwX1Oq3hI9F1h15IJuk8N+Xal4S+bvIsax8oBHw35NqXpL8jOqDvAsNiy3\nFRtG7VIWcafrLcnzcTHwKfAzsDk4Hz3Dfk2pfMOmBBT0+d842Oc5rHOjcVTbWsB/sB7BTVgB67bF\njSHjepycc8455xKVaXOcnHPOOecS5omTc84551ycPHFyzjnnnIuTJ07OOeecc3HyxMk555xzLk6e\nODnnnHPOxckTJ+ecc865OHni5JxzzjkXJ0+cnHPOOefi5ImTc84551ycPHFyzqUkETlDRHaLyF1h\nxxIWEfleRJaEHYdzmcQTJ+cyhIg8GyQaa0WkYgkcr0lwvGdLIj6XEA1uvxKRwcF56RBSTM6lNU+c\nnMsAIlIN6AXsBg4CeoYbkSshnYEuUdv2SaaccyXHEyfnMkNv4ADgn9iX6hUlcEwpgWO4JKjqd6r6\nXdRmPy/OlSJPnJzLDFcAu4CHgClAFxFpVNDOIlJXRP4hIgtEZJuI/Cwin4rI9cHjlwFLsCSsXzA0\ntFtE8vKHiETk+WBb4xjH32c4SUQqisjVIjJORJaKyHYRWS0ib4pIVrJvgIjUFJF7RWSeiGwSkQ0i\nsjiIs1HEfr/GJiJXiMhXwXuwXESGikj1GMfuLyJvich3Ee/XOBHpGGPfX+dmiUh7ERkvIutEJC9i\nn04iMlZEVgTvwyoRmSoif4w61l5znERkCpA/5+uDiPOyRMwPwVBtpQLeo49EJFdEGhT7DXYuQ+wf\ndgDOudIlIi2Ak4B3VHWtiLwInAlcDtwbY/+jseSqHvAxMBqoBrQGBgFDgdnAI8B1wc9vRRzi++C+\nsCGjWI8dhPWITQXeBdYBRwDnAd1E5HRVnRnv645hAnAiMA0Yiw1bNgHOBV4ElkXFdgM2FPY68A42\nJHYdcJKInKGquyKO/QT2PkwE1gKHYcOh74vI+ao6JkY8pwK3A5OBYUBjABHpDrwdvP7/ASuBukAb\noC8wPOIY0e/hc8F9B+B59pyLdaqqIjIcuAe4AHgtsqGIHBXENEZVf4wRr3MOQFX95je/pfEN+AeQ\nB/QKfq8GbAK+K2D/GcH+/WM81iDi5yZY8vFsAcd5LjhO4xiP3R081iFiWyWgfox9WwAbgfFR288I\nnv+uON6D1sG+o2I8VhE4ICq23cA2oFXUvi8HcQ+M2t4kxnHrAcuBBQXEnQdcGqPdqOCx1jEeOzDq\n9++AJUW9txGP1QdygUkxHvt70K5H2P9m/ea38nzzoTrn0piI7I/1UmzEejFQ1S1YL1JjEekStf+J\nwPHAh6q6z9VyWoo9Eaqaq6orY2z/GusB6yAiFZJ8mu0xjr9TVbfG2PcFVZ0XtW0QlvT0izrGDzGO\nuxp4E2hewLDoLFV9sZixritk/yIF7+8Y4AwROSJ/e/Dv5BKsd+u9ZJ7DuXTniZNz6a0nNszzhqru\niNj+IjaJuH/U/u2C+4llENs+RKSNiLwazMXZkT9HBxtOqwTUSfDQXwNfAb1F5EMRGSgibUWkoInU\nig1T7r1RdSk2pNcqSDby4z5cRIaLyDfBHKf8uK8Odok1Z2h6Ac+dP4T2uYg8LiI9ReTgOF5jvIZh\nn/0DIrb9FjgE6z3cXYLP5Vza8TlOzqW3K7Ak4KWo7ZOBFUBPEamtquuD7bWC/VeUXYhGRE4BJgXP\nPwFYDGwOfj8fOA6onMixVTVPRDoBg4ELgYexxHGtiDwB/DVGwrC6gMOtxoYpawDrRORIbHizOtYz\n9jbWw7cb6ITNN4oVd8zjq+ooEekJXA/8Cfgz/Drx+wZV/TKe11wQVZ0QTCi/TETuCF73gCBer8nl\nXBE8cXIuTYnIYcBZwa9TC+hcUWwo74ng9/VYQnFYCYSQn4jE+pypFWPb7Viv0mmq+mnkAyLSHkuc\nEhYMc10LXBtMgO+M9Qjdg837eTCqSb0CDlUPe982Bb9fj72evqo6Iiru+ljiFDOkQmIdA4wJruA7\nBZvMPQAYKyLHqOrGgtrGaTjwN+BcEZmJ/Tt5X1W/T/K4zqU9H6pzLn31x/7GPwaejnF7HkuSIms6\n5Q8fnR3H8fMvny9o3lH+fJxYSVh2jG1HAL/ESJqqFrB/wlR1oar+iz2v87yoXQQ4PbpdUFqhETBP\n91xVlz9X6O0YT3VaknFuVtUJqnoVdr7qYVdIFqao8wLWs7QLS8b6Y693eCH7O+cC3uPkXPrqh/X6\nXBpr8jL8WnrgZBHJVtUcVf1CRKZjE7EHqOrTUfs3iJggvg7rNSmoHtQM7Au5H/BRxDF+h/XCRPe4\n/IBNpG4RTAhHRPbDrgqsG2P/uIlIE4g5ifvQ4H5bjGaXisgTqjonYtv9WDL6XMS2/GOeBoyPeM7b\ngFbFjVtETgemxRg6zO8B22fSeJRfsPe9wDpdamUp3sKGLU8AfsJKHzjniuCJk3NpSETOBA4HphSU\nNAWeA9pjvU45wba+2FydYSJyCfApUAVLArKwJAZV3SIiM7Ak60VsTtJu4EVVXYZ9EX+LFchsDMzC\nSgt0wuo0nRMVy+NYD9A0EXkDSxA6YhOrP8Au409UFvDfICmcD6xiT62lPKx+VCTFkqBPReQ1rDZT\nF+yKw0/YM7QJ8G+sJtZoEXkd+Bk4GWiL1X/qXsxYHwMaiMjHWB0mxZKyE7Fzsc+k9ShTgjb3i0hr\nYAOwXlWfjNrv39gyPIcAD+vedamccwXwoTrn0lN/7Mvz+SL2ex3rbblYRCoDqOo32NDYo1jSci3Q\nB6v/NCSqfV+smGR3rH7QvVjChqpuxwptvoV96V+FzWHqAHwRHYiqvov1gHwbPF9vLMlph/XqxOq5\niXddti+AB7DE7hxsXtIZ2CT0U4LnjjYUmwPVDnsP6mMJVtfIJENVZ2NzhL7AJrFfjvX6nAoUVLCz\nsLj/hk3ePxa4EjuXlYCbgbNVNbrdXr8HvXX9sGTvL9g5uWGfAFSnAEuDX58pIBbnXBTZ92/QOecy\nl4jcjS1b0klVp4YdT2kRkUOxxGmaqnYKOx7nUoX3ODnnXGYaiE0g/1fYgTiXSnyOk3POZQgRqYnV\nhWqCzWubB4wMNSjnUownTs45lzkOxOZQbcMWU/6/GHOmnHOF8DlOzjnnnHNx8jlOzjnnnHNx8sTJ\nOeeccy5Onjg555xzzsXJEyfnnHPOuTh54uScc845FydPnJxzzjnn4uSJk3POOedcnDxxcs4555yL\n0/8Hd8OCmpCJs7oAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "p = 0.1\n", + "q = np.linspace(0, 1, 500)\n", + "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n", + "mse = (p - q)**2\n", + "plt.plot([p, p], [0, 0.3], \"k:\")\n", + "plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n", + "plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n", + "plt.plot(q, mse, \"r--\", label=\"MSE\")\n", + "plt.legend(loc=\"upper left\")\n", + "plt.xlabel(\"Actual sparsity\")\n", + "plt.ylabel(\"Cost\", rotation=0)\n", + "plt.axis([0, 1, 0, 0.95])\n", + "save_fig(\"sparsity_loss_plot\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def kl_divergence(p, q):\n", + " \"\"\"Kullback Leibler divergence\"\"\"\n", + " return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28 * 28\n", + "n_hidden1 = 1000 # sparse codings\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.01\n", + "sparsity_target = 0.1\n", + "sparsity_weight = 0.2\n", + "\n", + "#activation = tf.nn.softplus # soft variant of ReLU\n", + "activation = tf.nn.sigmoid\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n", + "\n", + "weights1_init = initializer([n_inputs, n_hidden1])\n", + "weights2_init = initializer([n_hidden1, n_outputs])\n", + "\n", + "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n", + "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n", + "\n", + "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n", + "biases2 = tf.Variable(tf.zeros(n_outputs), name=\"biases2\")\n", + "\n", + "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n", + "outputs = tf.matmul(hidden1, weights2) + biases2\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate)\n", + "mse = tf.reduce_mean(tf.square(outputs - X))\n", + "\n", + "hidden1_mean = tf.reduce_mean(hidden1, reduction_indices=0) # batch mean\n", + "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n", + "loss = mse + sparsity_weight * sparsity_loss\n", + "training_op = optimizer.minimize(loss)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train MSE: 0.0814597 \tSparsity loss: 0.394392 \tTotal loss: 0.160338\n", + "1 Train MSE: 0.0547652 \tSparsity loss: 0.130117 \tTotal loss: 0.0807885\n", + "2 Train MSE: 0.0491941 \tSparsity loss: 0.1048 \tTotal loss: 0.0701541\n", + "3 Train MSE: 0.0428305 \tSparsity loss: 0.0548378 \tTotal loss: 0.053798\n", + "4 Train MSE: 0.039173 \tSparsity loss: 0.258404 \tTotal loss: 0.0908538\n", + "5 Train MSE: 0.0361353 \tSparsity loss: 0.0223408 \tTotal loss: 0.0406035\n", + "6 Train MSE: 0.0327397 \tSparsity loss: 0.0539304 \tTotal loss: 0.0435258\n", + "7 Train MSE: 0.0290739 \tSparsity loss: 0.0547827 \tTotal loss: 0.0400305\n", + "8 Train MSE: 0.0270405 \tSparsity loss: 0.0859135 \tTotal loss: 0.0442232\n", + "9 Train MSE: 0.0253977 \tSparsity loss: 0.103084 \tTotal loss: 0.0460145\n", + "10 Train MSE: 0.0226754 \tSparsity loss: 0.142739 \tTotal loss: 0.0512232\n", + "11 Train MSE: 0.0230119 \tSparsity loss: 0.185111 \tTotal loss: 0.0600341\n", + "12 Train MSE: 0.0205853 \tSparsity loss: 0.0905618 \tTotal loss: 0.0386977\n", + "13 Train MSE: 0.0192056 \tSparsity loss: 0.0623178 \tTotal loss: 0.0316691\n", + "14 Train MSE: 0.0210757 \tSparsity loss: 0.0744971 \tTotal loss: 0.0359751\n", + "15 Train MSE: 0.0175043 \tSparsity loss: 0.107595 \tTotal loss: 0.0390232\n", + "16 Train MSE: 0.0171826 \tSparsity loss: 0.103815 \tTotal loss: 0.0379455\n", + "17 Train MSE: 0.0161081 \tSparsity loss: 0.0794732 \tTotal loss: 0.0320028\n", + "18 Train MSE: 0.0154329 \tSparsity loss: 0.0585922 \tTotal loss: 0.0271513\n", + "19 Train MSE: 0.0197791 \tSparsity loss: 0.377908 \tTotal loss: 0.0953607\n", + "20 Train MSE: 0.0150577 \tSparsity loss: 0.0548583 \tTotal loss: 0.0260294\n", + "21 Train MSE: 0.0150628 \tSparsity loss: 0.0414338 \tTotal loss: 0.0233496\n", + "22 Train MSE: 0.0150796 \tSparsity loss: 0.237394 \tTotal loss: 0.0625584\n", + "23 Train MSE: 0.0147269 \tSparsity loss: 0.134953 \tTotal loss: 0.0417176\n", + "24 Train MSE: 0.0173978 \tSparsity loss: 0.200894 \tTotal loss: 0.0575766\n", + "25 Train MSE: 0.0146077 \tSparsity loss: 0.113924 \tTotal loss: 0.0373926\n", + "26 Train MSE: 0.0141484 \tSparsity loss: 0.0858619 \tTotal loss: 0.0313208\n", + "27 Train MSE: 0.0152365 \tSparsity loss: 0.103625 \tTotal loss: 0.0359616\n", + "28 Train MSE: 0.016218 \tSparsity loss: 0.0481832 \tTotal loss: 0.0258546\n", + "29 Train MSE: 0.015513 \tSparsity loss: 0.354282 \tTotal loss: 0.0863694\n", + "30 Train MSE: 0.0161261 \tSparsity loss: 0.0611825 \tTotal loss: 0.0283626\n", + "31 Train MSE: 0.0129792 \tSparsity loss: 0.0651063 \tTotal loss: 0.0260005\n", + "32 Train MSE: 0.0132758 \tSparsity loss: 0.269341 \tTotal loss: 0.0671439\n", + "33 Train MSE: 0.0139204 \tSparsity loss: 0.0999084 \tTotal loss: 0.033902\n", + "34 Train MSE: 0.0117337 \tSparsity loss: 0.0827167 \tTotal loss: 0.0282771\n", + "35 Train MSE: 0.0133443 \tSparsity loss: 0.114992 \tTotal loss: 0.0363427\n", + "36 Train MSE: 0.0133315 \tSparsity loss: 0.0640423 \tTotal loss: 0.02614\n", + "37 Train MSE: 0.0134189 \tSparsity loss: 0.0827036 \tTotal loss: 0.0299596\n", + "38 Train MSE: 0.0133511 \tSparsity loss: 0.187742 \tTotal loss: 0.0508995\n", + "39 Train MSE: 0.0129 \tSparsity loss: 0.160532 \tTotal loss: 0.0450063\n", + "40 Train MSE: 0.0131213 \tSparsity loss: 0.187431 \tTotal loss: 0.0506075\n", + "41 Train MSE: 0.0139679 \tSparsity loss: 0.142237 \tTotal loss: 0.0424154\n", + "42 Train MSE: 0.0133992 \tSparsity loss: 0.106155 \tTotal loss: 0.0346303\n", + "43 Train MSE: 0.0123227 \tSparsity loss: 0.131862 \tTotal loss: 0.0386951\n", + "44 Train MSE: 0.0145169 \tSparsity loss: 0.381131 \tTotal loss: 0.0907431\n", + "45 Train MSE: 0.0120986 \tSparsity loss: 0.196967 \tTotal loss: 0.0514921\n", + "46 Train MSE: 0.0120502 \tSparsity loss: 0.0845151 \tTotal loss: 0.0289532\n", + "47 Train MSE: 0.0108891 \tSparsity loss: 0.129996 \tTotal loss: 0.0368882\n", + "48 Train MSE: 0.0119541 \tSparsity loss: 0.116771 \tTotal loss: 0.0353082\n", + "49 Train MSE: 0.0112326 \tSparsity loss: 0.31247 \tTotal loss: 0.0737267\n", + "50 Train MSE: 0.0120686 \tSparsity loss: 0.133666 \tTotal loss: 0.0388017\n", + "51 Train MSE: 0.0112876 \tSparsity loss: 0.125641 \tTotal loss: 0.0364159\n", + "52 Train MSE: 0.0128882 \tSparsity loss: 0.228499 \tTotal loss: 0.058588\n", + "53 Train MSE: 0.0111112 \tSparsity loss: 0.14206 \tTotal loss: 0.0395232\n", + "54 Train MSE: 0.0124159 \tSparsity loss: 0.376808 \tTotal loss: 0.0877774\n", + "55 Train MSE: 0.0121291 \tSparsity loss: 0.371292 \tTotal loss: 0.0863876\n", + "56 Train MSE: 0.0112408 \tSparsity loss: 0.239637 \tTotal loss: 0.0591682\n", + "57 Train MSE: 0.0111381 \tSparsity loss: 0.105095 \tTotal loss: 0.032157\n", + "58 Train MSE: 0.0108619 \tSparsity loss: 0.0852712 \tTotal loss: 0.0279162\n", + "59 Train MSE: 0.011571 \tSparsity loss: 0.267429 \tTotal loss: 0.0650569\n", + "60 Train MSE: 0.0109666 \tSparsity loss: 0.209801 \tTotal loss: 0.0529268\n", + "61 Train MSE: 0.0119152 \tSparsity loss: 0.137225 \tTotal loss: 0.0393603\n", + "62 Train MSE: 0.0117323 \tSparsity loss: 0.11996 \tTotal loss: 0.0357243\n", + "63 Train MSE: 0.0111296 \tSparsity loss: 0.249816 \tTotal loss: 0.0610927\n", + "64 Train MSE: 0.011342 \tSparsity loss: 0.140469 \tTotal loss: 0.0394357\n", + "65 Train MSE: 0.0188621 \tSparsity loss: 1.00517 \tTotal loss: 0.219895\n", + "66 Train MSE: 0.019563 \tSparsity loss: 0.400298 \tTotal loss: 0.0996226\n", + "67 Train MSE: 0.0143541 \tSparsity loss: 0.182415 \tTotal loss: 0.050837\n", + "68 Train MSE: 0.0188449 \tSparsity loss: 0.525975 \tTotal loss: 0.12404\n", + "69 Train MSE: 0.0114148 \tSparsity loss: 0.849677 \tTotal loss: 0.18135\n", + "70 Train MSE: 0.0229842 \tSparsity loss: 0.282243 \tTotal loss: 0.0794328\n", + "71 Train MSE: 0.0135722 \tSparsity loss: 0.564019 \tTotal loss: 0.126376\n", + "72 Train MSE: 0.0151828 \tSparsity loss: 0.36578 \tTotal loss: 0.0883389\n", + "73 Train MSE: 0.0171349 \tSparsity loss: 0.396833 \tTotal loss: 0.0965014\n", + "74 Train MSE: 0.0221404 \tSparsity loss: 0.431197 \tTotal loss: 0.10838\n", + "75 Train MSE: 0.0174526 \tSparsity loss: 0.488249 \tTotal loss: 0.115102\n", + "76 Train MSE: 0.01674 \tSparsity loss: 0.204111 \tTotal loss: 0.0575622\n", + "77 Train MSE: 0.0157648 \tSparsity loss: 0.340083 \tTotal loss: 0.0837813\n", + "78 Train MSE: 0.0294703 \tSparsity loss: 0.143162 \tTotal loss: 0.0581027\n", + "79 Train MSE: 0.0174752 \tSparsity loss: 0.280303 \tTotal loss: 0.0735359\n", + "80 Train MSE: 0.0304021 \tSparsity loss: 0.463337 \tTotal loss: 0.123069\n", + "81 Train MSE: 0.020485 \tSparsity loss: 0.281656 \tTotal loss: 0.0768161\n", + "82 Train MSE: 0.0207744 \tSparsity loss: 0.303879 \tTotal loss: 0.0815502\n", + "83 Train MSE: 0.0154509 \tSparsity loss: 0.162027 \tTotal loss: 0.0478562\n", + "84 Train MSE: 0.0166876 \tSparsity loss: 0.932988 \tTotal loss: 0.203285\n", + "85 Train MSE: 0.0260509 \tSparsity loss: 0.573328 \tTotal loss: 0.140716\n", + "86 Train MSE: 0.0122943 \tSparsity loss: 0.0993213 \tTotal loss: 0.0321586\n", + "87 Train MSE: 0.0141513 \tSparsity loss: 1.45436 \tTotal loss: 0.305023\n", + "88 Train MSE: 0.0178403 \tSparsity loss: 1.11933 \tTotal loss: 0.241705\n", + "89 Train MSE: 0.0266467 \tSparsity loss: 1.23002 \tTotal loss: 0.27265\n", + "90 Train MSE: 0.0219565 \tSparsity loss: 0.138038 \tTotal loss: 0.0495642\n", + "91 Train MSE: 0.0132129 \tSparsity loss: 0.279289 \tTotal loss: 0.0690708\n", + "92 Train MSE: 0.0180262 \tSparsity loss: 0.560036 \tTotal loss: 0.130033\n", + "93 Train MSE: 0.0139025 \tSparsity loss: 0.369372 \tTotal loss: 0.0877769\n", + "94 Train MSE: 0.0131638 \tSparsity loss: 0.153498 \tTotal loss: 0.0438635\n", + "95 Train MSE: 0.0159171 \tSparsity loss: 0.169834 \tTotal loss: 0.0498838\n", + "96 Train MSE: 0.0204874 \tSparsity loss: 0.247977 \tTotal loss: 0.0700828\n", + "97 Train MSE: 0.032796 \tSparsity loss: 0.521566 \tTotal loss: 0.137109\n", + "98 Train MSE: 0.0185355 \tSparsity loss: 0.36058 \tTotal loss: 0.0906516\n", + "99 Train MSE: 0.01732 \tSparsity loss: 1.32711 \tTotal loss: 0.282742\n" + ] + } + ], + "source": [ + "n_epochs = 100\n", + "batch_size = 1000\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = mnist.train.num_examples // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch})\n", + " mse_val, sparsity_loss_val, loss_val = sess.run([mse, sparsity_loss, loss], feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n", + " saver.save(sess, \"my_model_sparse.ckpt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAIBCAYAAACP2yFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3GuMXWX5N+Bn2jlPz6VQoAgOUC14ANGAqKgkoomIASXx\nlHhMNCaaSDxEExM1MX5UvhglGEXkA2qCaAyIGkkQhSDISRSqlYOVoYXSTud86vvhDf/kffN/7mey\n706nTK/r65177bX3Xt3z60rWr+vQoUMFAIDOrVruEwAAeLETqAAAkgQqAIAkgQoAIEmgAgBIEqgA\nAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjqXu4TqDi03CcAK0jXcp8AR97OnTvD\n39HVq1cvyet2dcWX26FDnf+8d3fHf7Lm5+eX5HVbovfcet1Vq+L7GtF+5j21XndhYaHjY2dkrp+l\nfE/Dw8PN31F3qAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCApKO1NgGAhFbFQPT4eevR9U6PmzU3\nN9fxbus9Zd5zpFVP0XpP0Xm1dqNroFUhEL1utp4gUwWRqahYau5QAQAkCVQAAEkCFQBAkkAFAJAk\nUAEAJAlUAABJahMAVqDWI+SZR8yjx+Zbx40eqW9VDLSO3drvVKsGIPo8Dh48GO4u1TmXUsr8/Hx1\n1tfX1/FxM9UGi9mPRO9pKT/LxXCHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJL0UAGs\nQK0uoIzJycnqrNVv1N1d/7OT7ZmK9lufR9Rv1DI9PV2d9fb2hrut+dTUVEev2zr2zMxMuBt1bw0M\nDIS7s7Oz4TwSdXqVEn/Hmeun1TW2GO5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahMAVqDW\nI+SZWoVWNUIkeqS+VYvQqjaI9ufm5uITC7QeqY8+y/7+/nC3VV8QVVS0jh19Hq3POvq8JiYmwt3W\n9RFdm1FNRCnxZ93T0xPuZqoxFsMdKgCAJIEKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEjSQwVw\nDIq6lVativ+vHfUItfqtent7O37dVs9Q1H/U6pKK+o8yHVat3aiXq5RShoaGqrNWD9XBgwers+np\n6XA36r9qvactW7aE86gDq9WftlQyvWwvcIcKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAA\nkvRQAaxArV6daN7qAsr0UEUdRN3d8Z+k+fn5cD42NladtXqXog6sVt/T2rVrq7NWd1bUy9U6r9b3\nNDo62tFxW6KOqlLa7ynq3mrtRtdXq9Mrun4yn8f/HCN9BACAY5xABQCQJFABACQJVAAASQIVAECS\nQAUAkKQ24TC56667wvnVV18dzk8++eTqbGBgINz98Ic/XJ1t2rQp3G3NgZWp9cj9UomqDXbt2hXu\nPvjgg+F8ZGSkOms9Fn/GGWdUZ69+9avD3fXr11dng4OD4W6rCmLz5s3VWVRBUUop+/fvr85aNRIL\nCwvVWavaoPV3Jao+aL2naD4zMxPuRlrfw2K4QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlU\nAABJXcvVRdJwVJ5U5GUve1k437lz5xE6k/9X1I9SSikXXHDBETqTo8Npp50Wzr/85S+H85e85CWH\n8WyOmHrpCyvWrl27wt/RqJcp6glqafUbPfnkk9XZzTffHO7ecccd4fzpp5+uzk466aRwN+pWmpyc\nDHdPPfXU6mzbtm3h7vj4eDiPXvs///lPuBt1WLU6m6Lv8eyzzw53r7jiinC+Y8eO6qx17c3OzlZn\nrfeUueaHh4eb/yjcoQIASBKoAACSBCoAgCSBCgAgSaACAEgSqAAAkrqX+wRWil/84hfh/P777w/n\n0WOof/vb38Ldu+++uzprPYb8m9/8pjp76UtfGu7++9//DucZ3d31S/PEE08Md5966qmOX7dVq/Cl\nL32p42PDkdR6DDyqzIkeLy+llPn5+Y5fN9LX1xfOX//614fzTZs2VWdzc3PhblRB0PoNfvzxx6uz\n6DH/Ukp57rnnwvmBAweqs927d4e7UWXDli1bwt0777yzOrv33nvD3e3bt4fz8847rzprVVRE1QgL\nCwvhbuu6znKHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJK6oi6SZXRUntSL0dTUVDiP\n+lNaPVS7du3q5JQWpbe3tzpr9VBF5713795w96abbgrn7373u8P5UarzYiBetJ544onwdzTq7Gn1\n9USdTq3epfHx8epsz5494W7r3/7g4GB19uyzz4a70W/D6OhouLt27drq7IQTTuj4dUspZf/+/dXZ\n2NhYuBu9dqvD6vOf/3x11tPTE+7ecMMN4fzCCy+szlp/s6IeqtZ1G+WdVn/a8PBw83fUHSoAgCSB\nCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBI6l7uE2Bp9ff3h/OXv/zlHR97x44dHe9m3H333eE8\n6ps5//zzw91LLrmko3OCo02rYzDq7Il6pkopZXp6ujpr9fn09fVVZ6ecckq4u23btnAedWu1fgvP\nPvvs6qzVjbR69erqLHq/pcQ9U63XHhoaCnfn5+ers1tuuSXcjT7LN7/5zeHuaaedFs4nJyers1aP\nWfRZZ675w9HJ6Q4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkNoGj0vj4eHV2+eWXh7vR477f\n+c53wt2BgYH4xOBFIvp3UEpcbxA9Xl5KKb29vUvyuj09PeFu67wmJiaqs+icS4lrFVqP1Eev29rN\nVDK0jr179+7q7Lbbbgt3o+/x4osvDndbFRWtaoRIpvogqgNpXVuL4Q4VAECSQAUAkCRQAQAkCVQA\nAEkCFQBAkkAFAJAkUAEAJOmh4qj0ox/9qDobGRkJdzdv3lydnXrqqZ2eEhwzWn0+UZdUNCullO7u\n+p+d1u7o6Gg4j7qTWj1DUffd2NhYuBt1SbW67fr6+sL54OBgdRb1TJVSys9+9rPq7He/+124e845\n51Rn559/frgbdWeVEl9frc8jOvb09HS4ezi6psLjL+nRAQCOAQIVAECSQAUAkCRQAQAkCVQAAEkC\nFQBAkkAFAJCkh4pl8a9//SucX3XVVR0f+89//nN1tnXr1o6PC+RFPVT9/f2pY0c9VK0Oq9nZ2eps\nfn6+49ednJwMd9esWRPOo/0777wz3L3tttvCeeRDH/pQdXb88ceHu1EvVymlDA0NVWetDqto3upP\n6+3trc7m5ubC3cVwhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASFKbwLL41a9+Fc6jR5ivvPLK\ncHd4eLijc4KVpKurK5xH/8aix8tLiWsColqEUuLH3lv1BH19feE8eu3p6elwN3pPrTqH6JH7iYmJ\ncLcl+p7uu+++cPeRRx6pzi666KJw9+KLL67O1q9fH+4eOHAgnA8ODobzSPQ9tWoTomugVdewGO5Q\nAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIeKpZM1J9y0003hbtR38y3vvWtcPdw9InA\nShf9O4m6fkppd1xFomNHvxmltPuxov2hoaFwN+qLeuaZZ8LdqIeq1dnU6rj697//XZ2NjY2Fuy97\n2cuqs1af38knn1ydRe+3lFJWrYrv1czMzFRnre84eu3W60bXfOs9LYY7VAAASQIVAECSQAUAkCRQ\nAQAkCVQAAEkCFQBAktoElswPfvCD6uyOO+4Idz/wgQ9UZ8PDwx2fExwr5ufnw3n0iHmrFqG7u/6n\no1V9ENUmTE9Ph7utx+KjR+57eno63o3ebymlrFmzpjrbvHlzuLtnz55w/tOf/rQ627dvX7j7jne8\nozq7+OKLw93o+mnVNbQ+r0OHDlVnmWsgU+eR2X2BO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUA\nQJJABQCQpIeKjt1///3h/DOf+Ux1tmHDhnD3G9/4RkfnBPxfq1evDudRF1A0W8y8U5n+olLijqKo\nZ6qUuD8r6pkqJe6ampqaCndvvPHGcH7zzTdXZ2eddVa4e+mll1ZnW7duDXdHR0ers9Zn2eoLi7rI\nWj1mrT6xSNSt1TrnxXCHCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJL0UFE1OTkZzt//\n/veH86jz44Mf/GC4Ozw8HM6BWKuzKfr3memDynQQtUTn3Jo/99xz4e6BAweqs1YPVfSe77vvvnD3\nmmuuCed79uypzj7+8Y+Hu9u3b6/OWn1P0e9/qwsq2ycWib7j1nG7urqqs8x1+QJ3qAAAkgQqAIAk\ngQoAIEmgAgBIEqgAAJIEKgCAJLUJx7joUdF3vvOd4e6jjz4aznfs2FGdff3rX49PDFhS0aP+mcfa\nW4/MT09PV2etWoSBgYFwHv2etWpgos9j7dq14e6zzz5bnV133XXh7sjISDh/73vfW529733vC3ej\n9xTVRJRSysTERHW2bt26cDdT2bF69epwN2Mpj12KO1QAAGkCFQBAkkAFAJAkUAEAJAlUAABJAhUA\nQJJABQCQpIfqGLdv377q7Pbbb08d+/rrr6/ONm3alDo2EOvq6lqyY0f9Rq3X7enpqc76+/vD3dax\nM11S0Xm1+oui38rbbrst3N22bVs4f/vb316dDQ0NhbtR91ar8ys6dm9vb7jbOnak9R1Hx27tRj1l\nme61F7hDBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkqU1Y4Q4cOBDOL7jggo6P/ZOf/CScn3vu\nuR0fG1g+rZqASPRoeuvYrWqDlunp6eqs9ah/dN5PPPFEuBtVI7SqDd73vveF80svvbQ6i6oeSolr\nE/r6+sLd6POanZ3t+HVbx27VF2QqO1rXZpY7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAF\nAJCkh2qF++EPfxjOd+3a1fGx3/jGN4bzVicIsHyif5/z8/PhbtTn0+r6mZub63h3cHAwnEdav0dR\nd9I999wT7t51113VWdSNVUopr33ta8P5mjVrqrNW31Ok9XlE10Crh2rjxo3hPDrvVg9VdF6Z/rTD\nwR0qAIAkgQoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASNJDtQLs3LmzOvva17525E4EeNGIOp8y\nHXKtHqFoPjMzE+6uWtX5PYCxsbFwfv/991dnN998c7i7Z8+e6qzVM3XaaaeF86i3q9UHFWl91pFW\n31OrHyv6HlvXT6R13UbHPhy9ie5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahNWgDvuuKM6\nGx0d7fi4O3bsCOcDAwMdHxtYXplH16N5q9ogeuS+9bqtR/2jioHx8fFw95lnnqnOnnrqqXC3r6+v\nOrvkkkvC3bVr14bz6Lynp6fD3ejz6u7u/M9/VLmxGEtZy9Gpnp6e9DHcoQIASBKoAACSBCoAgCSB\nCgAgSaACAEgSqAAAkgQqAIAkPVTHuAsvvLA6++1vfxvu6qGCo1erD2p+fn5JXrfVExT1ULXOaXJy\nMpxPTU1VZ1HPVCmlHDx4sDrbunVruHvuuedWZ+eff364OzQ0FM6jz6TVBxV1TbW+p6grKrPb2s90\nXGU6qmZnZzvefYE7VAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAUlfmMcMldFSeFLxIxc8wsyLt\n2rWr49/RVuVC9Gh75pH5lqhyoZS4YqBVyRA9Nt96lL+np6c6a32WUbVBa791XtFn3fqeotdtVQxk\njp25PjLX7djYWLj7qle9qvk76g4VAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJB2tPVQA\nAC8a7lABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJA\nBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJA\nBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJA\nBQCQJFABACQJVAAASQIVAECSQAUAkCRQAQAkdS/3CVQcWu4TgBWka7lPgCNv9+7dS/Y72tVVv6QO\nHYpfNpqvWpX7P/7CwkLHu9Frt95T9Hm0zqn1njOfV+u8O7WU7yn6LJdS6z1t27ateWLuUAEAJAlU\nAABJAhUAQJJABQCQJFABACQJVAAASUdrbQIAy6T1uH2mYmD16tXVWevR9dYj9ZnzyrxupLs7/jM7\nOzvb8WsvVS3CUh87IzqvTL3F4ahrcIcKACBJoAIASBKoAACSBCoAgCSBCgAgSaACAEhSmwBwDIoe\nMY/qB0pp1xtE5ubmqrOoUmExWucdiT6P+fn5cLenp6c6a9UitObRsaNZaz4+Ph7uZr6nzPXTqj7I\nnNdSV0G4QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJeqhWgBtuuKE6a3WN3HvvvdXZ\nNddc0/E5lVLKV7/61ers4osvDnff8pa3pF4bjnWtzp1MZ1PUI9TqAormrX6rqIOolFImJiaqs717\n94a7O3furM6eeuqpcHffvn3VWeuc161bF87POuusjmallLJ58+bqLHN9tPqvurq6wnl0DbSun+7u\nemyZmZkJdzPX/GK4QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJKlNeBH49Kc/Hc6///3vL8nr\nZh8x/eY3v1md3XTTTeHuH//4x+ps/fr1HZ8THCta/36jR9uXsnIhOvb8/Hy4Ozo6Gs6feOKJ6uxP\nf/pTuBtVyPzjH/8Id3fv3l2dtWoRzjnnnHAefSYbN24Mdzds2FCd9fb2hrvRZ92qgohet5S4dmF2\ndjbcja7bVuVCVMtxOCoV3KECAEgSqAAAkgQqAIAkgQoAIEmgAgBIEqgAAJIEKgCAJD1UR4moa2qp\neqZKKeXcc8+tzt7znveEuzt37gzn1113XXX2yCOPhLs///nPq7OPf/zj4S7Q7pJaqmO3+o0i09PT\n4fy///1vOP/DH/5Qnf36178Od59++unqbHh4ONx9xSte0fFu1MlUSin//Oc/q7ORkZFw94orrqjO\nzjrrrHA36oNqdUUNDg6G8+gaaX0eUS9X1FFVSvu8s9yhAgBIEqgAAJIEKgCAJIEKACBJoAIASBKo\nAACS1CYcIU8++WQ4v/baazs+9ute97rq7NZbbw13o8dbW48/R4+vlhI/7nvnnXeGu88++2w4B5ZO\nq3Khu7v+p6P1uzE3N1edjY2Nhbt33313OH/ggQeqs9HR0XD3ggsuqM4++tGPhrs7duyozlavXh3u\n7tq1K5xHn9fjjz8e7rZ+ZyObNm2qzlrVBhMTE+E8qjdoVR9kdhcWFqqz6JpeLHeoAACSBCoAgCSB\nCgAgSaACAEgSqAAAkgQqAIAkgQoAIEkP1RHS6lWKel+inqlSSvnd735Xna1ZsyY+sYQf/ehH4fye\ne+7p+Njvfve7O94F2l1Src6eSNRBNzs7G+4ePHiwOmt1MkW7pZSyefPm6uycc84Jdy+77LLq7K1v\nfWu429/fX52NjIyEu62Owr/85S/VWevzGh4ers4GBgbC3eeee646O+6448Ld559/PpxPTU1VZ9Fn\nWUrc65Xpkoo6qhbLHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEmgAgBI0kN1hLzmNa8J51FP\nVW9vb7jb6hNZKtdee204n5mZOUJnAvz/Vq3q/P/LUc9U69hzc3Ph7vT0dHUWdQyVUsr5558fzs87\n77zqbP369eHu1q1bq7Oenp5wN+pV2rNnT7jb+qyjLsEzzzwz3F27dm111npPJ510UnW2cePGcDf6\nPEqJr59WD1XUn5a5bg8Hd6gAAJIEKgCAJIEKACBJoAIASBKoAACSBCoAgCS1CUeJ1iO9y+H6668P\n5w888EDHx77kkkvC+emnn97xsYFSDh06FM6jx89b9QUZg4OD1dkpp5zS8W4ppfT19VVnrUfqoyqA\n6LillLJv377q7OGHHw53//rXv4bz/fv3V2fHHXdcuBvVJkTffymlbNiwoTqLqhxKaV8/Q0ND1Vnr\ns15YWKjOZmdnw93o30R03MVyhwoAIEmgAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACS9FAd46IO\nlE9+8pPh7vT0dDg/8cQTq7Orr7463O3p6QnnQKzVMxR18rR2I62+p+7u+p+d448/Ptzt7+8P53v3\n7q3O1q1bF+5GHVdzc3Ph7jPPPFOd3XvvveHuk08+Gc7POOOM6uzCCy8Md7ds2VKdRV1QpcQ9VC2t\nXsWo86nVB9XqmopE13Xmmn+BO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJLahGPcn//85+qs\nVYvQ8qlPfao62759e+rYQCyqRSglfky8VX0QaT3WHlWitM55ZmYmnK9Zs6Y627RpU7jb19dXne3Z\nsyfcve2226qzBx98MNw97rjjwvlZZ51VnbXe0+bNm6uzVkVFVKvQ+p7Gx8fDefS3pbe3N9yNroHV\nq1eHu615ljtUAABJAhUAQJJABQCQJFABACQJVAAASQIVAECSQAUAkKSHaoX72Mc+Fs5vvPHGjo/9\nuc99Lpx/8Ytf7PjYQE7UM7WUu62un6jDaGFhIdxtnVfUQ9Xqx3rssceqsxtuuCHcffjhh6uzubm5\ncHfHjh3hfHh4uDprdUlt3LixOst8T60eqtZ7juatayDS3R1HmqhfrbW7GO5QAQAkCVQAAEkCFQBA\nkkAFAJAkUAEAJAlUAABJahNWgLGxserslltuCXenpqaqsxNOOCHc/cpXvhLOe3t7wzmwdKJHxEuJ\nH5tfytqEVavq/49vPTLf398fzqPH+VuP8t96663V2W9+85twd3p6ujo7++yzw90NGzaE86gK4vTT\nTw93o+8i+u0vpZTR0dHqbHBwMNxdt25dOJ+YmKjOWvUWrXkkuq5b/14Wwx0qAIAkgQoAIEmgAgBI\nEqgAAJIEKgCAJIEKACBJoAIASNJDtQJceeWV1dmePXs6Pu5nP/vZcL5p06aOjw0srajvqZS4s6kl\ns9vdXf+z0+o3anVcRV1T+/btC3ejeeu3Lvqst2/fHu6+4hWvCOdRD9XMzEy4G31eretjfHy8o3Mq\nJf6OWzL9aa3rY6m5QwUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJeqheBO69995wfvvt\nt3d87CuuuKI6u+qqqzo+LrC8urq6wnnU95Pp8+nr6+t4nu0R2rt3b3V2/fXXh7s33nhjdbawsBDu\nnnfeedXZqaeeGu5u2bIlnE9PT1dnTz/9dLg7NDTU0ayU+LuI+r5KKWViYiKcR++p1WEVzXt7e8Pd\nqLcr0632AneoAACSBCoAgCSBCgAgSaACAEgSqAAAkgQqAIAktQlHicnJyersy1/+crgbPQraEj3u\n23oEFTh6tR4DX7Wq/v/pqFKhpaenp+N565xb5xVVzNx0003h7sjISHX2tre9Ldx917veVZ296U1v\nCndblQxjY2PVWfR3o5RSBgYGqrPo+2/tzs7OhrtRLUIppQwODlZnreqMqA6k9Z6iY2eu+f95/fQR\nAACOcQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJCkh+oo8b3vfa86+/3vf9/xcT/2sY+F86uu\nuqrjYwNHr0wPVabPp9VDFfUutc756aefDuf33XdfdbZ///5w93Wve111dvnll4e7UU/V5s2bw91W\nZ1PUB9j6rPv7+6uz8fHxcHdoaKg6a3UfRq/bOnbUu1VKKVNTU9VZ67rt7q5Hnta1txjuUAEAJAlU\nAABJAhUAQJJABQCQJFABACQJVAAASWoTjhJf+cpXluS43/72t8N59EgusHJl6gui2oT5+flwN5pH\n51RKKQ899FA4v+uuu6qz1uP4p5xySnW2bdu2cDcyOzsbzkdGRsL5wYMHq7NMJcPc3Fy4G1UQtHZb\novf86KOPhrtR3cOpp54a7m7durU6Oxx/C92hAgBIEqgAAJIEKgCAJIEKACBJoAIASBKoAACSBCoA\ngCQ9VCtcq3sl6hpZSn19feE803MTda+0TE5OhvOrr76642NHovdbStxT1tPTc7hPhxWg9W+71Y8U\nif4Nto47NTVVnbX+/T322GPh/J577qnOWp9H1L31z3/+M9yNzntmZibcffzxx8N5f39/ddbqoerq\n6qrORkdHw90TTzyxOlu7dm24OzExEc7//ve/V2cPPvhguLtu3brqrPUdb9mypTprdaAthjtUAABJ\nAhUAQJJABQCQJFABACQJVAAASQIVAECS2oQV7uSTT17uU/hffepTnwrnJ510UnU2MjIS7n73u9/t\n6JyOZtH3+IlPfOIIngkvFlENQClxVUdrN5q3agI6PadS2jUBAwMD1dmzzz4b7kbVCK3zeuqpp6qz\nVhXE0NBQOI9+C88444xwN3rPzz33XLib0arOOHjwYHX2/PPPh7uvfOUrq7PWZxnVSPT29oa7i+EO\nFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQJFABACTpoTpKfPCDH6zOfvjDHx7BMzkyvve97y3L\n63Z3x5d8q28m8pGPfCScv/71r+/42G94wxs63uXYFHXuLKXp6elwvmpV/f/xUY9UKaVcdNFF4fzy\nyy+vzn75y1+Gu08++WR19vjjj4e74+Pj1dmaNWvC3cHBwXAedTq1jj02NladPfTQQ+HuwsJCR7NS\nSlm/fn04f8lLXlKdnXnmmeHuy1/+8urshBNOCHej3/+5ublwdzHcoQIASBKoAACSBCoAgCSBCgAg\nSaACAEgSqAAAkgQqAICkrkOHDi33OfxvjsqTWi4//vGPw/nMzMySvO4DDzwQzr/73e8uyeuWUsoX\nvvCF6uyMM87o+LiXXXZZOD/++OM7PvZRbHkKiVhWu3fv7vh3NOqKKiXuIWr9TYn6sVo9VK35I488\nUp3deusN6VEMAAADVklEQVSt4e7f//736mzfvn3hbk9PT3XW398f7h533HHhfGhoqDpbu3ZtuLt3\n797q7MCBA+Fu1HG1ZcuWcHfdunXhfPv27dVZq5dr48aN1dmGDRvC3aiHqtXbtm3btubvqDtUAABJ\nAhUAQJJABQCQJFABACQJVAAASQIVAECS2gRY+dQmHIP++9//hr+j0W9/6+9Cq1ahU319feG8t7c3\nnEePxc/NzYW7IyMj1VmrYiB63ah+oJR2bUJ03vv37w93Dx48WJ1F1Ret1219D5s2bQrnq1ev7uh1\nW7uZ67JVm3DSSSepTQAAWGoCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQVC/PAOCY1OrkiTqM\nWrtRj9D09HS4+/zzz4fz6LWHhobC3bVr11ZnUc9UKaVs3LixOmt1erX6oGZmZqqz/v7+cDfq9Zqf\nnw93Z2dnq7NMH1gp8ffU09MT7i6V1vewGO5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJahMA\nVqDW4/qteoNOd1vHzTwyv2bNmnCeefQ9qglovadot/WeJicnO55Hr1tKKQMDA9VZVKnQ2m19zq1r\nL5q3dpdKVOexWO5QAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJIeKoAVaNWq+P/LS9UF\nND8/H86jXqZWv9FS9UyVUkp3d/3PYatLKnrPrddtzaPvsdUlFX2PU1NT4W70eWS6xkqJO59a109G\ndF6Ho//KHSoAgCSBCgAgSaACAEgSqAAAkgQqAIAkgQoAIEltAsAK1KoYiB4hz+y26hqiY7cemY8e\nty8l9+j7ctVIRPUELYfjUf+azPWRuQZaovNq1TVErzszM9PxOb3AHSoAgCSBCgAgSaACAEgSqAAA\nkgQqAIAkgQoAIEmgAgBI6lrKHgsAgGOBO1QAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQ\nJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQ\nJFABACQJVAAASQIVAECSQAUAkCRQAQAkCVQAAEkCFQBAkkAFAJAkUAEAJAlUAABJAhUAQJJABQCQ\nJFABACQJVAAASf8H0ruJ1KdPKhgAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "show_reconstructed_digits(X, outputs, \"my_model_sparse.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Variational Autoencoder" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28*28\n", + "n_hidden1 = 500\n", + "n_hidden2 = 500\n", + "n_hidden3 = 20 # codings\n", + "n_hidden4 = n_hidden2\n", + "n_hidden5 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "activation = tf.nn.elu\n", + "initializer = tf.contrib.layers.variance_scaling_initializer(mode=\"FAN_AVG\",\n", + " uniform=True)\n", + "\n", + "X = tf.placeholder(tf.float32, [None, n_inputs])\n", + "\n", + "weights1 = tf.Variable(initializer([n_inputs, n_hidden1]))\n", + "weights2 = tf.Variable(initializer([n_hidden1, n_hidden2]))\n", + "weights3_mean = tf.Variable(initializer([n_hidden2, n_hidden3]))\n", + "weights3_log_sigma = tf.Variable(initializer([n_hidden2, n_hidden3]))\n", + "weights4 = tf.Variable(initializer([n_hidden3, n_hidden4]))\n", + "weights5 = tf.Variable(initializer([n_hidden4, n_hidden5]))\n", + "weights6 = tf.Variable(initializer([n_hidden5, n_inputs]))\n", + "\n", + "biases1 = tf.Variable(tf.zeros([n_hidden1], dtype=tf.float32))\n", + "biases2 = tf.Variable(tf.zeros([n_hidden2], dtype=tf.float32))\n", + "biases3_mean = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n", + "biases3_log_sigma = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n", + "biases4 = tf.Variable(tf.zeros([n_hidden4], dtype=tf.float32))\n", + "biases5 = tf.Variable(tf.zeros([n_hidden5], dtype=tf.float32))\n", + "biases6 = tf.Variable(tf.zeros([n_inputs], dtype=tf.float32))\n", + "\n", + "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n", + "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n", + "\n", + "hidden3_mean = tf.matmul(hidden2, weights3_mean) + biases3_mean\n", + "hidden3_log_sigma = tf.matmul(hidden2, weights3_log_sigma) + biases3_log_sigma\n", + "noise = tf.random_normal(tf.shape(hidden3_log_sigma), dtype=tf.float32)\n", + "hidden3 = hidden3_mean + tf.sqrt(tf.exp(hidden3_log_sigma)) * noise\n", + "\n", + "hidden4 = activation(tf.matmul(hidden3, weights4) + biases4)\n", + "hidden5 = activation(tf.matmul(hidden4, weights5) + biases5)\n", + "logits = tf.matmul(hidden5, weights6) + biases6\n", + "outputs = tf.sigmoid(logits)\n", + "\n", + "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits, X))\n", + "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_log_sigma) + tf.square(hidden3_mean) - 1 - hidden3_log_sigma)\n", + "cost = reconstruction_loss + latent_loss\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(cost)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "\n", + "n_inputs = 28*28\n", + "n_hidden1 = 500\n", + "n_hidden2 = 500\n", + "n_hidden3 = 20 # codings\n", + "n_hidden4 = n_hidden2\n", + "n_hidden5 = n_hidden1\n", + "n_outputs = n_inputs\n", + "\n", + "learning_rate = 0.001\n", + "\n", + "initializer = tf.contrib.layers.variance_scaling_initializer()\n", + "\n", + "with tf.contrib.framework.arg_scope([fully_connected],\n", + " activation_fn=tf.nn.elu,\n", + " weights_initializer=initializer):\n", + " X = tf.placeholder(tf.float32, [None, n_inputs])\n", + " hidden1 = fully_connected(X, n_hidden1)\n", + " hidden2 = fully_connected(hidden1, n_hidden2)\n", + " hidden3_mean = fully_connected(hidden2, n_hidden3, activation_fn=None)\n", + " hidden3_gamma = fully_connected(hidden2, n_hidden3, activation_fn=None)\n", + " noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n", + " hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n", + " hidden4 = fully_connected(hidden3, n_hidden4)\n", + " hidden5 = fully_connected(hidden4, n_hidden5)\n", + " logits = fully_connected(hidden5, n_outputs, activation_fn=None)\n", + " outputs = tf.sigmoid(logits)\n", + "\n", + "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits, X))\n", + "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n", + "cost = reconstruction_loss + latent_loss\n", + "\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", + "training_op = optimizer.minimize(cost)\n", + "\n", + "init = tf.initialize_all_variables()\n", + "saver = tf.train.Saver()" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 Train cost: 18660.0 \tReconstruction loss: 14909.5 \tLatent loss: 3750.54\n", + "1 Train cost: 16801.5 \tReconstruction loss: 13146.9 \tLatent loss: 3654.59\n", + "2 Train cost: 16557.8 \tReconstruction loss: 12804.5 \tLatent loss: 3753.33\n", + "3 Train cost: 16497.7 \tReconstruction loss: 12671.7 \tLatent loss: 3826.02\n", + "4 Train cost: 16174.9 \tReconstruction loss: 12373.4 \tLatent loss: 3801.54\n", + "5 Train cost: 16035.1 \tReconstruction loss: 12275.9 \tLatent loss: 3759.2\n", + "6 Train cost: 15680.6 \tReconstruction loss: 11804.0 \tLatent loss: 3876.62\n", + "7 Train cost: 15986.4 \tReconstruction loss: 12102.2 \tLatent loss: 3884.18\n", + "8 Train cost: 15905.0 \tReconstruction loss: 12168.1 \tLatent loss: 3736.95\n", + "9 Train cost: 15573.1 \tReconstruction loss: 11884.5 \tLatent loss: 3688.53\n", + "10 Train cost: 15964.7 \tReconstruction loss: 12237.8 \tLatent loss: 3726.87\n", + "11 Train cost: 15424.4 \tReconstruction loss: 11624.2 \tLatent loss: 3800.19\n", + "12 Train cost: 15412.8 \tReconstruction loss: 11593.3 \tLatent loss: 3819.51\n", + "13 Train cost: 15994.3 \tReconstruction loss: 12164.7 \tLatent loss: 3829.56\n", + "14 Train cost: 15079.8 \tReconstruction loss: 11436.3 \tLatent loss: 3643.48\n", + "15 Train cost: 15979.3 \tReconstruction loss: 12097.2 \tLatent loss: 3882.11\n", + "16 Train cost: 14951.6 \tReconstruction loss: 11165.4 \tLatent loss: 3786.25\n", + "17 Train cost: 15493.5 \tReconstruction loss: 11697.3 \tLatent loss: 3796.21\n", + "18 Train cost: 15461.6 \tReconstruction loss: 11713.8 \tLatent loss: 3747.77\n", + "19 Train cost: 14935.8 \tReconstruction loss: 11180.2 \tLatent loss: 3755.56\n", + "20 Train cost: 14609.6 \tReconstruction loss: 10986.4 \tLatent loss: 3623.16\n", + "21 Train cost: 15187.6 \tReconstruction loss: 11438.1 \tLatent loss: 3749.49\n", + "22 Train cost: 15600.6 \tReconstruction loss: 11827.5 \tLatent loss: 3773.18\n", + "23 Train cost: 15116.5 \tReconstruction loss: 11350.9 \tLatent loss: 3765.54\n", + "24 Train cost: 14949.6 \tReconstruction loss: 11219.4 \tLatent loss: 3730.12\n", + "25 Train cost: 14937.5 \tReconstruction loss: 11313.6 \tLatent loss: 3623.88\n", + "26 Train cost: 15221.8 \tReconstruction loss: 11436.5 \tLatent loss: 3785.3\n", + "27 Train cost: 14935.9 \tReconstruction loss: 11222.6 \tLatent loss: 3713.28\n", + "28 Train cost: 14333.8 \tReconstruction loss: 10778.5 \tLatent loss: 3555.31\n", + "29 Train cost: 14920.6 \tReconstruction loss: 11206.3 \tLatent loss: 3714.29\n", + "30 Train cost: 14940.8 \tReconstruction loss: 11202.7 \tLatent loss: 3738.09\n", + "31 Train cost: 15194.3 \tReconstruction loss: 11453.7 \tLatent loss: 3740.55\n", + "32 Train cost: 14906.3 \tReconstruction loss: 11218.1 \tLatent loss: 3688.22\n", + "33 Train cost: 15060.4 \tReconstruction loss: 11375.2 \tLatent loss: 3685.17\n", + "34 Train cost: 14896.8 \tReconstruction loss: 11287.9 \tLatent loss: 3608.9\n", + "35 Train cost: 14583.5 \tReconstruction loss: 10907.3 \tLatent loss: 3676.15\n", + "36 Train cost: 14944.4 \tReconstruction loss: 11272.8 \tLatent loss: 3671.56\n", + "37 Train cost: 14900.2 \tReconstruction loss: 11129.7 \tLatent loss: 3770.56\n", + "38 Train cost: 14330.5 \tReconstruction loss: 10598.1 \tLatent loss: 3732.47\n", + "39 Train cost: 14748.5 \tReconstruction loss: 10982.6 \tLatent loss: 3765.95\n", + "40 Train cost: 14476.3 \tReconstruction loss: 10742.6 \tLatent loss: 3733.77\n", + "41 Train cost: 14663.8 \tReconstruction loss: 10947.7 \tLatent loss: 3716.1\n", + "42 Train cost: 14832.6 \tReconstruction loss: 11118.1 \tLatent loss: 3714.55\n", + "43 Train cost: 14790.0 \tReconstruction loss: 11085.9 \tLatent loss: 3704.08\n", + "44 Train cost: 14288.0 \tReconstruction loss: 10724.2 \tLatent loss: 3563.8\n", + "45 Train cost: 14553.8 \tReconstruction loss: 10962.5 \tLatent loss: 3591.27\n", + "46 Train cost: 14820.9 \tReconstruction loss: 11063.5 \tLatent loss: 3757.46\n", + "47 Train cost: 14408.0 \tReconstruction loss: 10874.2 \tLatent loss: 3533.85\n", + "48 Train cost: 14888.1 \tReconstruction loss: 11207.9 \tLatent loss: 3680.17\n", + "49 Train cost: 14823.5 \tReconstruction loss: 11127.2 \tLatent loss: 3696.3\n" + ] + } + ], + "source": [ + "n_epochs = 50\n", + "batch_size = 150\n", + "\n", + "with tf.Session() as sess:\n", + " init.run()\n", + " for epoch in range(n_epochs):\n", + " n_batches = mnist.train.num_examples // batch_size\n", + " for iteration in range(n_batches):\n", + " print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n", + " sys.stdout.flush()\n", + " X_batch, y_batch = mnist.train.next_batch(batch_size)\n", + " sess.run(training_op, feed_dict={X: X_batch})\n", + " cost_val, reconstruction_loss_val, latent_loss_val = sess.run([cost, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n", + " print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n", + " saver.save(sess, \"my_model_variational.ckpt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Encode:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_digits = 3\n", + "X_test, y_test = mnist.test.next_batch(batch_size)\n", + "codings = hidden3\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " codings_val = codings.eval(feed_dict={X: X_test})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Decode:" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " outputs_val = outputs.eval(feed_dict={codings: codings_val})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot the reconstructions:" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAJ2CAYAAACevsIBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAHgJJREFUeJzt3WuMXWXZBuC1O9OWKYXSKW05KpaWQ0mAQhAQhGJaFE8U\nEGMLBAImEBMMaAhBPKCJEomE8IcIVmlFiMSSoiZGgSIJEiBQaCOHQCnHYCu20paOPUxn5vvhj+/T\n73l3u8pM97NnruvnvWev9XaIa9/zZj++jYGBgQoAIKtRrV4AAEAzygoAkJqyAgCkpqwAAKkpKwBA\nasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkFpnqxdQMNDqBUANjVYv\nAAKeo7STps9ROysAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJCa\nsgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJCa\nsgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJCa\nsgIApNbZ6gUAMLwMDAzUyutep9FohPmoUf7+Hq78lwUAUlNWAIDUlBUAIDVlBQBITVkBAFIzDbSL\nnnrqqTC//fbbw/zggw8O866urjC/9NJLi/fu7u6ulQPsCaVpne3bt4f5k08+GeYPP/xwmD/33HNh\n3tPTE+Yf+chHwryqqmru3Llhfuqpp4b51KlTw3z06NG18v7+/jDv6OgI86qqP9VUmo4q5e3IzgoA\nkJqyAgCkpqwAAKkpKwBAasoKAJBao+5ZDXtIukUdeeSRYb5q1aohv/eECRPC/JRTThnyew+1ww47\nLMxvuOGGMG/2bf8WGj5fuWc4GfLnaF9fX5ivX78+zG+99dYwv+OOO8K8NPVT98ygqqqqsWPHhnnd\nyZu6Uz9jxowJ86OOOqp4jxkzZoT5uHHjwvzLX/5ymJc+I0q/ixZr+hy1swIApKasAACpKSsAQGrK\nCgCQmrICAKRmGmgXvfzyy2G+YsWKMD/mmGPC/MUXXwzzp59+unjv3/72t2H+1ltvhfnHPvaxMH/j\njTeK96ijs7N8pNSBBx4Y5u+8806te9x8881hfv3119e6zh5iGoiMhvw5WpqA6e3tDfOHHnoozH/4\nwx+G+d/+9rcwf//998N8r732CvOqKk/xbNiwIcxL5xuVlCajStNG++yzT/Fapammbdu2hXlpWrV0\npp1pIACAQaasAACpKSsAQGrKCgCQmrICAKRmGqgNbN26NczffPPNMC9NA73++uuDsp7SWRdVVZ4G\nKq3pH//4R5gvXbo0zM8999ydrK4lTAORUcueo6XPldKETWm651//+leYl6Z+9t13311Y3X9as2ZN\nmG/ZsiXMS8+/0nO6q6srzEvPvqqqqmuuuSbMV65cGebTpk0L8xdeeCHMS5NRLWYaCABoX8oKAJCa\nsgIApKasAACpKSsAQGrKCgCQWvlEOtIojekdddRRta5z9NFHD8ZymiodyLhu3bowP/nkk8P87LPP\nHrQ1AXtW6SC+0gF6U6dOHZTr744ZM2YM2rXqOOigg4qvlQ4sLB2KWHpednR01F9YUnZWAIDUlBUA\nIDVlBQBITVkBAFJTVgCA1BxkSG09PT3F10rfrC8dFvbUU0+FeWlKKCkHGZKR52hizz33XPG1M888\ns9a1SgcWfvSjH611nRZzkCEA0L6UFQAgNWUFAEhNWQEAUlNWAIDUnA1EbYsWLSq+tnbt2jCfNGlS\nmLfZt9UBaunr6wvzK6+8sviezZs3h/nxxx8f5occckj9hbUZOysAQGrKCgCQmrICAKSmrAAAqSkr\nAEBqzgaiaPXq1WE+c+bM4nt6e3vD/JVXXgnz0llCbcbZQGTkOZrAiy++GOazZs0qvqe/vz/M77nn\nnjCfP39+/YXl42wgAKB9KSsAQGrKCgCQmrICAKSmrAAAqTkbiKLf//73YV6a+KmqqrrwwgvDfNq0\naYOyJoCMShM8X/nKV8K8dGZQVVXV4YcfHuYXXHBB/YUNE3ZWAIDUlBUAIDVlBQBITVkBAFJTVgCA\n1EwDUZzuWbp0aZiPHTu2eK2bb745zDs6OuovDKBNvPHGG2H+0ksvhXmjUT4K57rrrgvz0aNH11/Y\nMGFnBQBITVkBAFJTVgCA1JQVACA1ZQUASM00ENXPf/7zMH/88cfDfMGCBcVrOQMIGM527NgR5uec\nc06Yl84MKp3/U1VVdckll4R5swmi4c7OCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkZnR5BFmx\nYkWYX3311WG+3377hfkPfvCDQVsTQEYDAwNhft9994X56tWrw7yrqyvMb7nlluK9mx0WO1LZWQEA\nUlNWAIDUlBUAIDVlBQBITVkBAFIzDTQMbdmyJcznz58f5n19fWF+0UUXhbnDCoHhbuPGjWF+zTXX\nhHlpemjevHlh/oUvfKF475F8YGGJnRUAIDVlBQBITVkBAFJTVgCA1JQVACC1RukbzC2WclHZ9Pf3\nh/mcOXPC/LHHHgvzo48+Oswff/zxMO/u7t754kYWX90nI8/RXVB6jl555ZVhvmjRojA/9NBDw/zZ\nZ58N84kTJxbXNEKngZr+o+2sAACpKSsAQGrKCgCQmrICAKSmrAAAqZkGamPr1q0L8ylTptS6Tunb\n6ieccELtNY1QI/Kr+6TnOfp/lD7rVq5cGeZnnXVWmPf29ob5fffdF+af//znw3zUKHsF/8U0EADQ\nvpQVACA1ZQUASE1ZAQBSU1YAgNQ6W70Adm7jxo1hfsopp9S6zq9+9aswnzVrVu01AbSTzZs3h/ll\nl10W5qXn7uzZs8P805/+dJib+hkcfosAQGrKCgCQmrICAKSmrAAAqSkrAEBqpoHawN133x3mr7/+\neq3rnH766WHeaDjaBmh//f39xdfuuuuuMH/11VfDvLMz/nhcsGBBmI8ZM2Ynq+PDsLMCAKSmrAAA\nqSkrAEBqygoAkJqyAgCkZhooiVWrVhVfu+mmm/bcQgDaVE9PT/G1Rx55JMx7e3vDfP/99w/z888/\nP8xNVQ4tOysAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJrR5SQef/zx4mubNm2qda2jjz46zLu6\numpdByCj0oGFr7zySvE9pbHmvffeO8wvu+yyMJ8wYULzxTEk7KwAAKkpKwBAasoKAJCasgIApKas\nAACpmQZqY5/4xCfC/OGHHw5z00DAcFA6NLC7u7v4nlNPPTXM582bF+YLFiwI846Ojp2sjqFgZwUA\nSE1ZAQBSU1YAgNSUFQAgNWUFAEitMTAw0Oo1RFIuCgri0QRoLc/RXVD6DCxNHDFkmv7C7awAAKkp\nKwBAasoKAJCasgIApKasAACpZZ0GAgCoqsrOCgCQnLICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwA\nAKkpKwBAasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwA\nAKkpKwBAasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwA\nAKkpKwBAasoKAJBaZ6sXUDDQ6gVADY1WLwACnqO0k6bPUTsrAEBqygoAkJqyAgCkpqwAAKkpKwBA\nasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKl1tnoB\n/Nu9995bfK2npyfMly9fHuZ33XVXrXt/5zvfCfNPfepTYT579uxa1wfYHf39/bXyDRs2FK+1Zs2a\nMC89X995550wf/vtt8N83LhxYT5nzpwwnz59ephXVVU1Go3iayOVnRUAIDVlBQBITVkBAFJTVgCA\n1JQVACA1ZQUASK0xMDDQ6jVEUi5qMHzta18L8zvvvHMPr2TnZs6cGeZ/+ctfiu+ZMGHCUC0nM3OG\nZNQ2z9HS59D27dvD/MEHHwzzBx54oHiPZ599NsxL485bt24N897e3jAfM2ZMmB955JFhvmzZsjCv\nqqqaOHFi8bVhrOlz1M4KAJCasgIApKasAACpKSsAQGrKCgCQmmmgIbInpn5mzZoV5hdccEGYr1q1\nKswXL15c677NDkq84ooral1rmDANREZt8xwtTdgsWbIkzL/5zW+G+d///vfiPTo743N7R42q9zf7\njh07auXjx48P8/vvv794j3POOSfMh/kBh6aBAID2pawAAKkpKwBAasoKAJCasgIApBZ/PZpd9vbb\nb4f5woULa13npJNOKr72xz/+MczHjRsX5qUzKvr6+sL8tddeC/MnnngizNetWxfmALvj3XffDfPS\nc/S9994L82aTPaeddlqYf/e73w3zSZMmhfkjjzwS5rfddluYb9u2LcxffvnlMK+qqvrMZz4T5sN8\nGqgpOysAQGrKCgCQmrICAKSmrAAAqSkrAEBqpoE+pNJkTOnMpdLUT+kb5lVVPluirkWLFoX5M888\nU+s655577iCsBhhpSs/FLVu2hHlpsvH4448P89LET1WVp34mTpxYfE9k8uTJYf7SSy+F+aOPPjoo\n9x3p7KwAAKkpKwBAasoKAJCasgIApKasAACpmQb6kE444YQwL00Jlb7d3tXVNWhrKimds7F9+/Yh\nvzdAyUEHHRTmCxYsCPMPPvggzL/0pS8V71GavimdJ1SaXNqwYUOYl575pemhadOmhXlVjewzgErs\nrAAAqSkrAEBqygoAkJqyAgCkpqwAAKmZBhoiEyZMaNm977nnnjBfuXJlreucffbZYX744YfXXhNA\nSen8s/PPPz/Me3t7a12nqsoTNqWpn9LE0S233BLmpbOBPvvZz4b5scceG+ZVZRooYmcFAEhNWQEA\nUlNWAIDUlBUAIDVlBQBIzTRQG3v++efD/Morrwzzbdu2hfmBBx4Y5rfffnuYjx49ehdWB/CfSlMu\nHR0dYT5u3Lgw7+vrC/PSOT/NlM5Gu/fee8P8oYceCvPSv2HevHlh3sqJ0XZkZwUASE1ZAQBSU1YA\ngNSUFQAgNWUFAEhNWQEAUjO63MaefPLJMC+NKJdcddVVYX7EEUfUXhPAYKk76txMaUT5F7/4RZjf\neOONYV46RPHcc88N8xNPPDHMd2fMeiTz2wIAUlNWAIDUlBUAIDVlBQBITVkBAFJrDAwMtHoNkZSL\napXLL788zO+///4w37p1a5hfe+21Yf6jH/0ozMeMGbMLq6OqqnhkAVprxD1Hm32erVy5Msw/97nP\nhfl7770X5tOnTw/zxx57LMwnT54c5qaB/p+mz1G/LQAgNWUFAEhNWQEAUlNWAIDUlBUAIDXTQEls\n3ry5+NqMGTPCvPRt9alTp4b5Cy+8EObd3d07WR07YRqIjEbcc7Snp6f42plnnhnmK1asCPO99tor\nzH/3u9+F+VlnnRXmpfON+H9MAwEA7UtZAQBSU1YAgNSUFQAgNWUFAEits9UL4N8uvPDC4mulqZ+S\nr3/962Fu6gcYDnp7e8O8dI5aVZWnfkpn9MydOzfMzzjjjDA39TO07KwAAKkpKwBAasoKAJCasgIA\npKasAACpmQbaw5YvXx7mjz32WO1rnX/++WH+jW98o/a1ALIpnV136623hnnp3J5mSmcGLV68OMw7\nOjpq34MPz84KAJCasgIApKasAACpKSsAQGrKCgCQmmmgIbJly5Ywv+GGG8J8+/btte9x4oknhvmY\nMWNqXwsgmzfffDPMf/zjH4f51q1bi9c67rjjwnzhwoVhPn78+DB3BtB/Kk1sDfbvyc4KAJCasgIA\npKasAACpKSsAQGrKCgCQmmmgIfLTn/40zJctW1b7WpdffnmYOwMIGA76+vrCvPQc3bRpU5iPGlX+\n+3v27NlhPnny5OaLo+rv7y++Vvpv98EHH4R5d3f3bq3BzgoAkJqyAgCkpqwAAKkpKwBAasoKAJCa\nsgIApGZ0eYh861vfGrRr3XbbbWHuwEJgONi8eXOYP/PMM2FeGlGeMGFC8R5z5swJ89JYbumAvtKo\nbrPx3khnZ/njd7AOASz9G0pr/ec//xnm77zzTvEepc+hQw89dCerq8fOCgCQmrICAKSmrAAAqSkr\nAEBqygoAkJppoDZQ+qZ8s0O7BsPYsWPDvKOjo/ie0jflt23bVuveW7ZsCfPbb7+91nWaKf07SpNc\no0ePHrR7A/9r1apVYb5mzZowL/1vsdkhec8//3yYH3744WFeer6++uqrYf7iiy+GeelAv9NPPz3M\nq6qqpk+fHuY9PT1hXjrY8dFHHw3z0nTPypUrw3yvvfYK86qqqvPOOy/M58+fH+bNJraasbMCAKSm\nrAAAqSkrAEBqygoAkJqyAgCkZhqoDRx88MEtue9VV10V5gcddFDxPWvXrg3zO+64Y1DWtCeUft9f\n/epX9/BKYHgpnVVTOl9m+/btta6zfv364r3vvPPOMP/1r38d5qVnWWlNpUnIuutpdq3SNFBpsnHH\njh1hXneysdkz/5hjjgnz/fffv9Y9dsbOCgCQmrICAKSmrAAAqSkrAEBqygoAkFqj9K3qFku5qDpK\nkyN33333Hl5Jbp2d8UBas/OHIpdddlmYn3rqqXWXVJ122mlhPm3atNJbGrVvAkMv3XO09HlTOv/s\nZz/7WZjfeuutYb5x48bivUsTR6U1bd26Ncz7+/tr5V1dXWFeOnut2b1LU0KNRvwIKk0D7bvvvmFe\neh6fccYZYV5VVbVw4cIwHz9+fJiX1lrt5DlqZwUASE1ZAQBSU1YAgNSUFQAgNWUFAEjNNNAe9stf\n/jLMS+dN7I6VK1eG+WCdz3PdddcVX5s+fXqta33xi18M8ylTptS6TouZBiKjtnmOlj6HStMvpTOA\nli5dWrxH6T0ffPBBmJcmaaZOnRrmJ510UphPnDgxzPfbb78wr6qqWr16dZivWbMmzDdt2hTmEyZM\nCPOZM2eGeWlSp9nk0owZM8K87kRnZRoIAGhnygoAkJqyAgCkpqwAAKkpKwBAaqaB4MMzDURGnqPD\nTN3P69LPNzmfp7ZBvJZpIACgfSkrAEBqygoAkJqyAgCkpqwAAKl1tnoBAMDO1Z28Gcypn1azswIA\npKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJCasgIA\npKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJBaZ6sX\nUNBo9QIA2pznKMOGnRUAIDVlBQBITVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgNWUF\nAEhNWQEAUlNWAIDUlBUAIDVlBQBITVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgNWUF\nAEhNWQEAUlNWAIDUlBUAIDVlBQBITVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgtc5W\nL6BgoNULgBoarV4ABDxHaSdNn6N2VgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgNWUFAEhNWQEA\nUlNWAIDUlBUAIDVlBQBITVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBS62z1AgCA/zUwMBDmjUZjD68k\nDzsrAEBqygoAkJqyAgCkpqwAAKkpKwBAasoKAJCa0WWK1q1bF+ZTpkwpvuc3v/lNmF9wwQWDsiaA\njErjxmvXrg3zxYsXF6+1bdu2ML/66qvDvLu7eyera392VgCA1JQVACA1ZQUASE1ZAQBSU1YAgNRM\nA1H0yiuvhPmoUeWOe8ghhwzVcgDS6u/vD/OXXnopzJcsWVK81rHHHhvmo0ePrr+wYcLOCgCQmrIC\nAKSmrAAAqSkrAEBqygoAkJppIIqefvrpMN9nn32K7zn55JOHajkALVc6A6iUb9y4Mcz7+vqK95g2\nbVqYd3V17WR1w5edFQAgNWUFAEhNWQEAUlNWAIDUlBUAIDXTQFRr1qwJ8+9973thfu211w7lcgD2\nmNIUz2Bd589//nOY77///sVrXXHFFWHe0dFRf2HDhJ0VACA1ZQUASE1ZAQBSU1YAgNSUFQAgNdNA\nVG+99VaY9/T0hPnFF188lMsBGHR1p34ajUat65Sel3/605/C/OMf/3jx3qVJodKaRgI7KwBAasoK\nAJCasgIApKasAACpKSsAQGqmgahuvPHGMJ8+fXqYH3bYYUO4GoDd09/fX3ytNMUzatTg/M3+4IMP\nhvn7778f5nPmzCleq7PTR/N/s7MCAKSmrAAAqSkrAEBqygoAkJqyAgCk1qh7XsIeknJR7W7Dhg1h\n3t3dHebHHntsmK9YsWLQ1jRMjNwDO8hs2D5HS59bfX19xfeUztXp6Oiode9t27aF+UknnRTm69ev\nD/Ply5cX73HAAQfUWtMw0fQ5amcFAEhNWQEAUlNWAIDUlBUAIDVlBQBITVkBAFJzWtII8txzz9X6\n+UMPPXSIVgIw+Jr9X3GURpfrXuuvf/1rmL/22mthftFFF4X55MmTa61npLOzAgCkpqwAAKkpKwBA\nasoKAJCasgIApGYaaAR55plnav3897///SFaCcDgGzWq/t/fpamf0oGFt912W5iXpo2uuOKKMK97\ngOJIZ2cFAEhNWQEAUlNWAIDUlBUAIDVlBQBIrdHsLIUWSrmodvH666+H+cknnxzmM2fODPNly5aF\neWenIbL/Uu/QEdgzRtxztNnnWd3PunfffTfMZ82aFeajR48O81dffTXM99lnn1rrGQGaPkftrAAA\nqSkrAEBqygoAkJqyAgCkpqwAAKkZ6xiGSlM869atC/PjjjsuzE39AO2kdD5Ps9f6+/vDfPny5WG+\nfv36MJ89e3aYjxs3rrgmdp2dFQAgNWUFAEhNWQEAUlNWAIDUlBUAIDXjHsPQs88+G+alb8NffPHF\nQ7kcgLRK00A/+clPwrw0JXnTTTeFeUdHx26ti/9kZwUASE1ZAQBSU1YAgNSUFQAgNWUFAEitMTAw\n0Oo1RFIuKpvNmzeH+ZFHHhnmU6ZMCfPnn39+0NY0QpUPJIHW8RzdBe+9916YH3LIIbXyFStWhPm+\n++67ewsbeZo+R+2sAACpKSsAQGrKCgCQmrICAKSmrAAAqTkbqI0tWbIkzNesWRPm8+fPH8rlAKRV\nmnz99re/Hea9vb1hPnfu3DAfP3787i2MXWJnBQBITVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSM7rc\nxlavXl3r5ydNmjREKwHIbceOHWH+xBNPhHnpeXn99deHeaPhPNOhZGcFAEhNWQEAUlNWAIDUlBUA\nIDVlBQBIzTRQG7vnnntq/fx55503RCsByG3Dhg1hvnnz5jA/6qijwnzKlCm17ls6QLHZ9FDpPSUj\nYRLJzgoAkJqyAgCkpqwAAKkpKwBAasoKAJCaaaA2sGrVqjB/99139/BKAPJqNkWzaNGiMO/sjD8G\nZ8yYUese/f39YT5qVLwn0GytpWvVnfoZTlNCdlYAgNSUFQAgNWUFAEhNWQEAUlNWAIDUTAO1gQce\neCDM+/r6wvyTn/xkmB9xxBGDtiaAbLZs2VJ87Q9/+EOYT5w4McxPOOGEMC9N8ezYsSPMS9NGpSmh\nqipP8dQ9Z2h3ziXKys4KAJCasgIApKasAACpKSsAQGrKCgCQmmmgJHp7e4uv3X///bWudemll4Z5\ns2+fA7S7TZs2FV8bO3ZsmE+aNCnMTz/99DDv6Oiole/Oc7c0rVM3H058egEAqSkrAEBqygoAkJqy\nAgCkpqwAAKmZBkqi2TfGDzjggDCfNWtWmF9yySWDsiaAdrL33nsXXytNSa5duzbMp06dGuZjxowJ\n89I00O5M6pTO9BnJ7KwAAKkpKwBAasoKAJCasgIApKasAACpNZJ+6zjloqBg+B/MQTvyHKWdNH2O\n2lkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgNWUFAEhNWQEAUlNWAIDUlBUAIDVlBQBI\nTVkBAFJTVgCA1JQVACA1ZQUASE1ZAQBSU1YAgNSUFQAgNWUFAEhNWQEAUlNWAIDUlBUAIDVlBQBI\nrbPVCyhotHoBAG3Oc5Rhw84KAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoA\nkJqyAgCkpqwAAKkpKwBAasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoA\nkJqyAgCkpqwAAKkpKwBAasoKAJCasgIApKasAACpKSsAQGrKCgCQmrICAKSmrAAAqSkrAEBqygoA\nkNr/AJFSZIFRH/pCAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n", + "for iteration in range(n_digits):\n", + " plt.subplot(n_digits, 2, 1 + 2 * iteration)\n", + " plot_image(X_test[iteration])\n", + " plt.subplot(n_digits, 2, 2 + 2 * iteration)\n", + " plot_image(outputs_val[iteration])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate digits" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "n_rows = 6\n", + "n_cols = 10\n", + "n_digits = n_rows * n_cols\n", + "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " outputs_val = outputs.eval(feed_dict={codings: codings_rnd})" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhcAAAFMCAYAAAB4eJ7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzsnWdgVNXWhp8pyaQXAoQioAhEFFHpoHIBUSyAvXu99vrZ\nu1iwXbvXil2xgWLBigrCBSyA0ov0JgQSCCGFTMqU8/04d62ciUFFJjNR9vMHHENmn3P23metd5Xt\nsiwLg8FgMBgMhmjhjvcADAaDwWAw/L0wxoXBYDAYDIaoYowLg8FgMBgMUcUYFwaDwWAwGKKKMS4M\nBoPBYDBEFWNcGAwGg8FgiCrGuDAYDAaDwRBVjHFhMBgMBoMhqhjjwmAwGAwGQ1QxxoXBYDAYDIao\nYowLg8FgMBgMUcUYFwaDwWAwGKKKMS4MBoPBYDBEFWNcGAwGg8FgiCrGuDAYDAaDwRBVjHFhMBgM\nBoMhqhjjwmAwGAwGQ1TxxnsAu4kV7wEYDAaDwfAXxtUQv9QoFwaDwWAwGKKKMS4MBoPBYDBEFWNc\nGAwGg8FgiCrGuDAYDAaDwRBV/uoJnTHHsuwc0u3bt1NZWQlAy5YtcbuNnWaoxbIsXK4GyZMyGAyG\nRo95IxoMBoPBYIgqRrnYBYLBIHPmzAHg/PPPx+PxAPDNN9/QvHlzAOOt7sFYlkU4HAbseRDruSCq\nmmVZEX8XZL4KjXGuihro8/l+Uw0MhULU1NQAsHHjRrKzswHIyclplNdlMOxpGOPiDxAMBgEYPXo0\nd9xxBwA7duyge/fu+jPyUpENMVYbnPPl4aS+76/70nFu3n/lDdl5XUDMQlTyzDdu3AjAtGnTaNu2\nLQC9evUiKSkJiP293bFjB1VVVQDMmzePWbNmAfDLL7/g9/sBaNWqFcceeywAmZmZtGjRAoDExESy\nsrIA8Hq9MRm/PLvq6moSExN/9Z3OZ1teXg7Ao48+ynvvvQdAUVEReXl5AHz88cfG0DcYGgEmLGIw\nGAwGgyGqGOXidwiHw4waNQqAO+64Qz3C5s2bc/XVVwO2FFtXcm7oMVVXVwO211ZYWAjAtm3b9GdW\nrFgBQEVFhX6Wn5+v42zatClDhw4FoGPHjqSkpOjP/VU8PlEO5syZw+jRowHIzs7m3nvvBRpWwbAs\nS73oG264AYAffviB/fbbD4BnnnlGvelYzg2A1NRUduzYAcDkyZOZN28eAIsXL9ZQQmJiIh9//DEA\nGRkZOsYOHTpw8sknA3DYYYcB9lxvqHvpVJ0SEhLqvVcyHy3L0uv64osv2LBhAwA1NTUsXrwYgI8+\n+ohLL7004t8Zfp+6CugfvXf1Kafmvu8almURCoUAOyxYUlIC2Ip5cnIyYO9r9al6jRljXPwO+fn5\nPPTQQ4D9ohap+4EHHtCXc6xfHgDFxcUAvPLKK0ycOBGADRs2qNERCAQAOzYtkzIhIUHl7/3335+W\nLVsC0KlTJ/29sZ644XBYF5bb7f7DYaVwOMy6desAGDFiBGvWrAFgv/32+1WIKlo4cyr8fj/jx48H\n0LBDWVkZW7ZsAewXXqzvpXyf2+2mSZMmAAwcOJBNmzYBsGrVKn3mbdu21bni8/lYv349AHPnzmX+\n/PkAtGvXDoCHHnqIAw88EIj+XN+V3BTLsli1ahUAhYWFOm+cv2PlypV6XbIxxxvnywNqjWJ5Mbtc\nLr2vdedsQ80hp1FXU1OjRltRURFg7xsyV1JTU3V8zn8XDoc1zLZp0ybS09MBdI9JSEjQ8cdyLTjH\nF6v7+WeQcRYVFfH4448DMG7cON3bnXtIbm4uY8eOBaBnz55/ierExj9Cg8FgMBgMfymMcrETxNO4\n5ZZb2Lp1K2B7FRdddBEAZ599tia8xQPxJIqKivjll18A23OWMYllm5ycrAl6eXl5DBs2DLDl7169\negG25xprRJ6fPXs2K1euBKBfv37svffegJ1MuLOkVLCTFs866ywA1q1bpyGI++67L6rPxalWWJal\n1QybN29W5aRVq1aA7RW1adMGsJMTd5Zs29C4XC4SEhIA+57K+C6++GL1KjMyMnR8xcXFLF26FIAP\nPviAKVOmAGjY4fbbb+edd94BICsra6fen9MTbwhCoRCzZ88GaqtKwA7xyBxeu3atet977bVX3Cp2\nwuGwqgGvvvoqn3zyCQCrV6/W0GpqaipgX5eMv2XLlrouzzzzTFq3bg3YYcy6aztahEIhHZOE+mbM\nmEFGRoaOSVSgxMRETZj96KOPePPNNwF7Dh111FEA/Oc//wHsPSrWqm44HCY/Px+Ad999V9eohIy7\nd+/OqaeeCtQqLBCpfgWDQb3XoVBIE/rl2aanp0dlXsm9vv3221WVqK6ujlB75Ds3btzISSedBMCD\nDz7IaaedBqBKemPEGBf1YFkWb7zxBoDGpQEOOOAAHnzwQYC4GhZut1sNhuzsbDWE0tPTdWM69NBD\nATte3rt3b8CWwmUTc7vdcZMIg8GgbrYvvfSSLhCR3utDFpmEe7788kvWrl2rnx1zzDGA/YyiSTgc\n1o1X/pTPBw8eDKA5L1u3btWNqKCgIG7GBdQan+np6XTu3BmwxyyfOzf9Zs2a6Uvsp59+0muQebV6\n9Wp9mcu8q4+GlPDBvqfPPPMMYBvS8nl2djZ9+/YF4Nhjj1XjolmzZjHdfC3LUqN55syZPPzwwwBM\nnTpVP3eGFUT+htp7t27dOi13//rrrznzzDMB+7rEgE5JSdnte+389z6fT0MaEubo0qWLjqOioiKi\neaB8PnXqVDWgEhMT1Vhyhnhivcds375dHagVK1ZEhKMAxo8fz2OPPQbYzpYY3s2bN9e/p6SkaGjN\n7/frs9t3330BOO+883b7ukKhEF9//TUAY8eO1e9o0qQJXbt2BewqLjGKNmzYQGlpKQAjR47Uve+m\nm24CbCeyMYV8wIRFDAaDwWAwRBmjXNRDaWkpL774ImB72ZLYNG7cuEYjQ4knISERgM6dO3P77bfr\n38GWzWLVr+D3kDFPnz6dJ554Qj+T7P4DDjhgp2OVf7t8+XIAnnzySf2Z4447jiuuuAJAwwHRwpls\nl5SUpN+51157kZOTA9SGRcrKyrTnRbNmzeJ+v8H2Hp1qRX2SK6Ce2ubNm9XbEyn8oIMOiqgmijXO\neSP3NxQK6fguv/xyLrzwQsBWVsT7Likp0WcU7XnhRMJm4XBYE2MfffRRZs6cCdSqbWDf97phDbfb\nrfPesixVFzt27Mjxxx8PQOvWrRtkboOtwkoIRJ5zkyZNVIkoLCzUiqMZM2awYMECoFblADt0cu21\n10b8jlgmHYradv/992uYNRQK6VicSq8oEJmZmXTs2BGA3r1768/6fD6ttps5c6b+vIQlonFdJSUl\nPPvss4A9P/baay8AxowZo9/n9/v1fVNSUqLPYNasWTRt2lQ/B3ttN7ZqEqNcGAwGg8FgiCpGuXAg\nHsiUKVO05M3n85GbmwvYCTiS3Onz+fTnU1NTf5WD0dDxRon/L168WD27wYMHa7xOvJ/GYMXK+KQX\nwSWXXKLW/y233KIx0sTExJ0mcYqFPnLkSADWrFmjXsfDDz9MWloaEP3rdSZHOvF4PBonlbizk44d\nOzaacjEZR90yQvH2li9frkrS7Nmz1YuVPIZ77rlHY/LxKCksKysD7JJjUVUSEhK45JJLALj55psj\nFEX5e1VVlSoyO0sQjsYYZZyVlZUsWrQIsL13Z36TqABt2rRRz1Q8zUMOOUST+37++WcyMzMBuPrq\nqzXB2ePxNGjCrPxOmetNmjRRb7+4uFj3uvnz56sS4/P51NsfPXo0++yzj15vrJES8I8++kgVrSOP\nPJJHHnkEsJVEsOe9s1TdOVZZDy6XS+/7oEGD9Nk5Fag/izzDVatWsXnzZsC+j6+++ipgd/Z19nWR\nvzdr1kznzfHHH/+r3kbO0mGnQhlPjHHhQJLWxowZEyF1Stb8HXfcoZJbWlqaLsQhQ4ZwzjnnAGhi\nXEpKSoO2ApcX2pYtW9SwGT58eIPKv38Gy7K0B8UZZ5wB2FLezTffDNjJUTu7T84kTnn5TZ8+HbA3\nwWuuuQZo2PMknFnkzsXuDJdITwmv16sv5MzMzEaxwKHWaK6pqdFnMW3aNCZMmADA0qVLtdlahw4d\nuO222wA0STYnJyfmLwzLsjQx8+677wbsXgoyjuOOO04NTWe4CmrnUXV1tRqAPp+vwdaGzNMtW7bo\nZu/3+7VPSOfOnTU8c+CBB+o5KNJwLScnR0MMHo9HjQufzxfhtNQ3DxsKp3OUlZVFhw4dAHvfEQMv\nJyeH66+/HrCrkuJlTIdCIe0T4Xa7ueqqqwDbcalbCec0LizLikhsdlbj1Nd+XozBaCHz2+12a5WZ\nc7+pO6ed1SxSBfP9998DMGzYsIjql8ZA43CtDAaDwWAw/G0wysX/cPYwyMjIUCnM2SVt3rx5ESVZ\n4oGsWbNGa6vFenTWSke71tuyLA2LWJalno6zJttp/YpHEY/SsLKyMo488kig1lI/9thjue6664Df\nvjfiYaxbt47XXnsNqC2PHDRokNbVx0qtqatiiFck1/fjjz9qDX1jCYlAree1fv16VdhWrlyp9zc9\nPZ3+/fsDdthJOrbGMxG4srKSV155BUB7ALhcLv7xj38A8OKLL2qoBiL7S0go5Pvvv1d1pn///px4\n4okAUU1OdXbftCxLu6GuXbtW1cXCwsIIZUJUDFG5Dj/8cLp16wbYYRPxkHe2NmL1PJxz/b///S8A\nixYt0ns9YMAATaSNR5diGcfatWu1W25SUpLOi6qqKt0PJQQxbdo0DXnst99++iycPTnqdo2N5v2W\n39WkSRMNs9TU1GjrgzvuuKPexMxwOKzvp7lz5/LWW29F/F6fzxeXZ/BbGOPifwSDQX35JScnRzRD\nkhe52+3WuGK/fv10ozvkkENUGndmYDfkeQzjxo0D7DwQ56mtYmhMnjwZsDc2kYbz8vL0PJTDDz+8\nwfIUoDYn5Pbbb9f6bKlgeemll37XIHAae2+++ab+PrnPF198sS7Cum1+GyLvAuznL/fa4/Hod0ps\nev/9949rVUV9OA3RkSNHat6Ls3HTkCFDtH9Eenp6XA0jZw8ImePyks7MzOS5554DIkM1oVBIDYrK\nykqtKHr77bf1pSjrAdDma9GYJ86cC7fbrWGR4uJi7WMRDAYjTiOWF92SJUsAeOedd7THy1FHHaWN\n+uLRBMyJ05GSngzl5eX68j7ssMPiWj0na/HLL7/UvduyLP79738DdrMp2Xuc/S5k3+jZs6dWbLRv\n317Xbizuebt27TjooIMA+2ykl19+GbD3EHGavF6vhjGff/55fvzxR8AOK0v4XcJB7du3j0ur9d+i\n8bhXBoPBYDAY/hbsscqFWOVi0ZaWljJp0iTA9ookvJGfn68eSLNmzRgwYAAAF154oSZs1ec178qB\nTLtKIBDg/fffB+zEMfFM33jjDVUp6usIuHHjRr799lsAevTooZKxhICihWVZWuP/0Ucf6T2W5K+d\nfZ+z1XZ5eTlTp04F7K56cg1i7bdt2zbiGkUJaYgQiVO5cI5VxiQeVEVFhXrNInnHG8uytBvqV199\npffX5XKpcnHJJZdohn1jCeesWLFCew3Ife7Tp49K2s4+HVu2bNH25TNmzNAQ5bx58zRR1e/3q4oh\nrZOjMVecib0ej0dVrPT0dK1ykflRd9zyZ1VVFXPnzgVg2bJlFBQUAHYb7Vh603Vx3t9ly5bpZ/K5\n9FqIF7LvydggMok3FArpfBc8Ho+qLevXr9eQ28UXX6z7eSxITEzkgw8+AOxkTAmnPfnkk6rYrV69\nmtWrVwO1fWjAVtYlGVjG3FDVULvDHmlchMNhnYBS0jN+/Hg9tyAUCmmL25qaGu2lf+qpp3L00UcD\n9mJ3Tty6FQ8NuUn7/X5tnhUOhyNON6wbz+3Ro4duouPGjdPrmj17tk7uc889N6oTMxAIqFxXXFys\nuSkSsgmHwxHxXIk9bt++XRvFzJ49W18YJSUlKmVKKMfr9arR4qzMaUicDancbrc+f5Fep0+fHpGr\n82dfXtEsOSwrK9NTfWtqajSPwufzaYhvxYoVGvN3nmQZD+Tax4wZo2tUXgYnnXRSRLMpCS+MGDGC\nn376CbDDfQcffDBgzxF5efj9fjVWZKOOliEqY/b5fHTv3l3HLC2a8/PzNY8lKytLq8+kbHXDhg0a\n+vH7/VoBUFRUFFFFEC+KiooiTi4WQ7SqqipuLe6dRk5ZWZmWTzvPPhk8eLBWqMmxAG63Wx2AUaNG\nqbM1fPjwmBoXgLY4+Pe//63ND/Pz89U4dpaLQ+06aNOmjc6t7777DoATTzwxrkdS1EfjcFMMBoPB\nYDD8bWhcpk4DI57mjh07NFFGPJtVq1apJ+PxeNRDtiyLPn36ALa8KT9/0EEHaYZ9Tk6OJhrGonLB\n6/WqClBaWqre9JAhQ7RtuagFTo/noosu4vTTTwdg4sSJqlycc845Uc00Lioq0qY2brdbPUXJ/ncm\nYC5evFhbJsvPy5+S8FZRUaENs4YMGQLYiZ3yjGKZJe3MKHc2XQNbAZBmX5dffvkfCjXU9fzqJqf+\nWWRszz77rCYZZmRkaDOhvfbaS73RqVOn0r59ewC6du2qc+uP3Ne6Ej/snmonasUPP/ygc1fUuIED\nB+pcKiws1JbvmzZtUkXxlltuUS922bJlTJw4EbDvR9u2bf/0uH4L52mZAwcOBOwKIme4xHkv5dmI\nV/rVV19p9VR+fr7OoaVLl0acuBsv9cLZ6t7v9+v1rl69OiLBOdbjk7De1Vdfrfc9HA5rm27nCb7O\nsR1yyCGArYTdf//9gB0ylkMfY4WM6dBDD9XDJSdPnqzqZ7du3XTO5uXl6TOYMGECn332GYCeStut\nW7e4NjGrjz3GuLAsS0Mgn332mcphIkeuX79eZSqfz6ebbTgc1n83depU3Xi9Xi89e/YE7M0vln3d\n09LSGDVqFABXXHGFZm8//fTTOgHrIykpicMOOwywX4giDwYCgai+oFNSUlSa3rJli75k5Ttee+01\nNcJSUlJUskxLS9OfmTFjhoYbsrKyVN484YQT9FpinR1dt7GOVLNIGVx1dbXOp7Vr12pMum4JsLxc\nQqFQhDEl/082792ROSV7/osvvtAyyMzMTM0Zat68uYbW0tPTWbhwIWC/0I444ggAzW+oGyqpr8un\nZVlayXHDDTf86XE7u106z3QBOzwm+Qh33XWXGqW9e/fmyiuvBOw5JM/g9ddfV2MkOTlZXyrO0Eo0\nTreU7/B4PBFzfWfN4erO2wMOOEB/1nl8fLxDVPLdSUlJEUfcy+ehUEiNwYSEhAZtGlgfMj+6du2q\nJ8a6XK7fPdtExte+fXvdw+W5xQOfzxeRBzRo0CDADuvIe8UZhs3Ly9PQyfz58wF47rnntEqmsZx/\n1ThMHIPBYDAYDH8b9hjlorq6mieffBKw+7FLJrfU/ZeWlmqiVfPmzdWSXb9+vWYle71ebYN79tln\na/JWrJuXuFwubdz09NNP67kEEprZGeFwmG+++Qaw1QpnL/pokpGRoSckHn/88VqTLWrF1q1bVfrb\nd9991TrPz89XmW/ixInqYZx22mnqmcb7zBRnlVHdaobExMSIBDzxoAOBgI7b2XciOTlZ1RmR8qPR\nZ8KyLE30kjkNtocsPRRyc3O1kiIpKUnn0E8//aQ195KEePTRR+tccV5vQUGBeu05OTnajn13kGtv\n1aqVZtALkydP1nDbtm3bdHzdu3fXZOyvvvpKW8SXlpbqfe/WrRsnn3wyEP35Xl/DpT/SsE68/s8+\n+0wr0hISElRh6d27d8zVgPpISEiICPHJXBg0aFDcTuJ0homc99qZ6Pl7pKenq+KRm5sbt3vsdru1\n+uOoo47SEEndtvYyb1u0aMEFF1wAoK36ne+pxqJc7DHGxaxZs7TU1Hkeh2zuzljijh076pWuc3Jy\nuPzyywG7AUs8O6LJd+fm5mqJXbdu3SLkQcEZI5VqEZfLpRUC0b4Ot9uth+x07NjxV79fFpIg0vrc\nuXM1xFBSUqIvg7vuuivuRoV8t1xLaWmpVivIXKkb1pFD7izLUuOhSZMmmg+TkZGh87Bu6Gh3cYYU\nJLfCGcNNTk7WcYTDYZWSO3bsqOGGWbNmAXYuiXT2TEhI0DBcWVmZGonRirnLvczOzlZjVF7C48eP\n1+9IS0vTTfSjjz7S/BwxksAulbz44osBuOCCC/RY62iH02R/8Pv9et9/K6Ql1yjVUG+++WbEYWCH\nHnoosPOD/GKBZVlqOL766qt6lkU4HKZHjx6AvY6dsn2skXldWVmpxll2dnbE2SE7OwgR7DFH4zCy\naCB7YHp6+u+GaNxuN126dAGI6Nopc1/KoeONCYsYDAaDwWCIKn975UKs2NWrV6tstH379l9lbDsb\nODnVioSEBPW07733Xq1WiPYJebuKWN9ffvmlnrvx888/8+677wKodxkOh/nhhx8AO5Qj1m3Lli0Z\nMWIE0DBex2/dn7qNhLZv3w7ABx98oMmz++67r56Empqa2mgaxMg4kpOT1bMWD6qqqkrrzydNmqQJ\nk8OGDdNqF2fyr9vt/lW78Ghdp0jrffr0Ua++oKCAn3/+GbCTwuS7/H6/enDO8xgkLLF+/XpVXkKh\nkCacVVZWamgwMTFR18/uqC8yL8rLy/W+yjgKCgoiEh+d3yNednZ2tqp3jz76qN6HhkqOdFaC+P1+\nfZ6/5TXLfL/xxhsB+5wXuectWrTQ5L54VGA4E2olQfeFF17QvTMnJ0f3wKysrLhVJjhDH1VVVbrW\n3G53xOmm9c1FudczZsxQReaoo46KW88OQJOu8/PzVRn6LSShU/bLuqe/NgaMcmEwGAwGgyGq/O2V\nC7H8TznlFO3iN3HiRE22cyoX4r2lpaVpKeqpp56qSXDJycmN5uQ58RjEIwY7GU9OKRw+fDhgX+un\nn34K2Ba+xMjffPPN3yxbjRWVlZXMmDEDsGP4khA5duxYjR02FtUCIpULyVmRXgS//PKLeiAFBQUc\nfvjhgN0tVTyLWHijLpdLk3uHDRumc7ywsJCPP/4YsEuAJWekrKxMS6xbtWql8WtnJ0ZJkmzbtq16\n3k2aNFHPqaKiQnMadgdZX927d9eW2BJXDgaDOu+DwaBeY69evRg8eDBgdyqUDo2xKOV0u92qYKWn\np+vzT0hIiOh5I16x3+/n3nvvBWDBggW/upbHHntMxx/LvcY5PrAP/Xr++ecB+/nL/O3fv7+WKsez\nI6Qz/8mpmhUVFel8z83NjehdBLaSK/O3oqJCVa6MjIy4J82CvRZFXa6bc+N8RtLRU9S9Pn36RCRd\nNwZc8ZSCosAfHnw4HFZpr6ioSB+Us2+FMxQSz+OmdwW/388xxxwD2GcpyGSTcVdXV+t1tWjRQs8k\n6dGjR1ybrchmsHbtWu3ZsXTpUs2CPuGEExpdO9u6OOVVgMsuu0yN1iFDhnDTTTcBdj19rI1S57kn\n0jDuk08+Ufl4xYoVrFy5ErA3KOk10r17d21OJutFkp4Bhg4dqid4OsNViYmJUXlezjCmHF8vVTmh\nUEg34TZt2mjjqbPPPjuuZ6PIva6oqND+Ij6fTxNm3W63vrRfffVVbY0vFWupqak88sgjgH1iaywa\n8dUdvzhWMpcffPBBPYXT7XbrcfWPP/64hlzjvTc6Q9tSRbRjxw6911lZWfoMZC5blqVhts2bN+v/\n79mzZ9wqXyzL0iTq5557TtfiKaecovPa2afpxhtv1GcjBsXbb7+tfWn+BA1ywSYsYjAYDAaDIars\nMcrF3xXLstQbPeOMM/SEQPFEMjIytDXyzTffrL0B4t0iVsY3c+ZMlYcDgYCWDor3/FdAFIyCggL9\ne1ZWlib3xbvTorM3h5R1btu2TT04v9+vHlBGRoZ6pqKCFRcXawjNqcA4T7eN9vUFAgFtWy49O9LS\n0vS04k6dOqlXF28PWgiFQuohV1ZWajfRsrIyPYzsqaeeUlle5vjVV1/NzTffDMTm+IC6hMNhVdwk\n/DVy5Ei9/wMGDGDkyJGArRjFe+8QnEnhzrJUUQFKSkr0c1E5ampqdI2mpaVpJ+GMjIy4XpfMm88+\n+0xb1geDQS23XrdunYb1/X6/hiBFBTv22GN3Rx1tkAVkjIu/AbJY5syZo22cZaL17dtX80fiWTfv\nxLIsbeA0b968iE1MwlSNYZyGWpz7hHk29eOsONuxY4f2nxk/frwaF9u3b9eX2JlnngnYL/L6zgKK\nFeFwWEMJUln04YcfqvF44YUXqlHXGEOVzsZZwWBQ95ZZs2ZpSHDVqlUAdO7cWav/mjRpojlrTqMu\nnsfb+/1+bSQ4duxYrWaprKzUudWrVy89E0WqoXbTMDJhEYPBYDAYDI0fo1wY4oKoLaFQSKXLpKSk\nRiO5Ggx/FOdBbs5+IeIt/+c//9HKF0D7WEgIsGXLlnGvQnOGzsBWXhpLWO+P4FQupDPuokWLNGQs\nlVHl5eUaGh42bJieEuzcd+J9rbI3lpaWsmjRIgDef/999t9/f8BOYJa+M1HaL01YpB7+0oM3ROdk\nSoOhseGU6svKyjT/Yu7cufTr1w+oLWH+K7y8/yo4K18sy2Ly5MlA7b3OzMzUst/U1NQIoy4aTeD+\nopiwiMFgMBgMhsaPUS4MBoPBsMcj78I9UEVqkAtufKm/BoPBYDDEmD3QqGhQTFjEYDAYDAZDVDHG\nhcFgMBgMhqhijAuDwWAwGAxRxRgXBoPBYDAYoooxLgwGg8FgMEQVUy1iMBgMUUI6XDorD1wul6lE\nMOxxGOXCYDAYDAZDVDHKxU5wHucrbWGh8Z0KGAgEKC8vB+zjhKXXvPTMNx5TfAiHw+rFBoNBPXXR\n4/GYZ/I3IhgM6nNOTEzU9efcN+qe//BXef4y/mAwqPueUWHiR30NLxvzszDKhcFgMBgMhqjSuNzw\nRkIgENDT9GbPnk11dTUA3bp1Y9999wVsBUMOuNnZyXROSzOaFqZlWWzcuBGAG264gUmTJum4U1NT\nAbjooosAuOuuu/D5fFH77mgQDof15L+ampqIg4ZkrC6X63e9/cZitctzrqqq4r///S8A7777rh5W\nlZGRwXk1MbFkAAAgAElEQVTnnQfAUUcdpdcV6/E7D9MKhUL6/X8VNSUQCLBp0yYAJkyYwFdffQVA\nUVERbdq0AeCSSy6hb9++ACQnJzfYWOQ+VldX6z5Q9xA+574gczwUCqkK4FQD4kkoFNLxFRcXA3DP\nPffw/vvvA/a8Ftq0aUPbtm0BOP7447nkkksAW7UxNBzhcJiamhrAPi21tLQUqJ3jrVq1anQHrhnj\nwoFsGNu3b+epp54CYMWKFeTk5OjPJCUlAdC6desICTSWL79QKMSIESMA+Oyzz9T4AaisrARg1KhR\nABQWFvLII48A6GmA8UIMioULF6rxtnbtWrZv3w5AixYtdCNLTEzU8bZr144WLVoAteGe5OTkuG7O\nlmXpYl+2bBkAzz77LPPmzQPA7/erXJ6ens7SpUsBGDRokBoXDT0++VPmxIYNG3TMwWCQtLQ0APLz\n8/nyyy8B+/jva665Bojacc5/GqfRBjB9+nRGjhwJ2HNIriUcDjNz5kzANjquuOIKAO67774Gu9cy\n58SYd45XkPnu9/v1GPCysjJyc3MBdF9JTEyM+4tB5shzzz0HwDfffKPXk5iYqCGSDRs26FHyU6dO\n5aeffgJg9OjRcTeS/gxyjZWVlZSUlAC2MyD7fDzC4PWt3Xnz5qnjsmTJEn0evXv3BuDMM8+kZcuW\nQPyNVcGERQwGg8FgMEQVcyqqA/E0p06dykMPPQTYcpPIrD179lQPIycnRz1rn8+nn8fCatyyZQt9\n+vQBYPPmzeohOUMJIlOmpKRoiOTOO++MqyUuiafPPfccb7/9NmDLyuJBZ2Rk6P0rLS3V5wG2UgRw\n+OGHA7Yk2759e8BWk2LhZVuWpWPavHkzL774IoB6FOFwmIMPPhiArl27qnTpcrn08wEDBsQ0TGVZ\nlkre1dXVESc/ylwpLCxk6NChgP2MRIkR7y1eiCL34IMPAvDiiy9SVFQEEJFkXRdRt7755hsOOOAA\nID4hKFFWfvnlF9566y0AFixYQPPmzQEYPHgwAIcccoiGGnw+X1zG6kw+lj9lTSUkJOi8KS4u5vzz\nzwdgypQptGrVCoBVq1bFXX2pD5kn8mcoFNJw5aOPPsrHH38MoGsV4P/+7/+4/fbbAVt1jAVyf2tq\nali3bh0AL7/8MuPHjwdQVQXsOSJrWsIieXl5+rOyn+4C5lTUhkY2g6+++orMzEzA3gD69+8P2BNz\n6tSpgC2X9evXD7ClZHlhNKSRIRNw6dKlEfHyrKwsAI488kidWF988QUAO3bs0Nj0VVddRdOmTaM+\nrj+KUyaWRdG8eXPdbJs2baovj9TUVLZs2QJARUUFK1asAGoXmd/v102uTZs2akw1xH13Zs2vX78e\ngLFjx2qoQ15gp512mhp9LpeL/Px8wJbzRQqPtXHnNCKc3+28T7m5ufozJSUlcQ+HgH3PJXQmxpvk\nA4C9qcoaTUhIoLCwELANEplDL774Ik8++SRAzF98zvuelpZGSkoKYBtvixYtAtBxzps3T3NyOnTo\nEJc5UjcPZGfk5uZqaGr69Ol6XY1Fiofa9VpVVaX7xqeffgrY9/qHH34A7PC3GFUej0eN6eTk5JjM\nF9nPA4EAc+fOBeDhhx/WfLri4mINCXbo0IHDDjsMsENrkydP1p8B27jbvHkzAB07dmzwsf8R4r+L\nGAwGg8Fg+FthlAsHGzZsAOywSJcuXQBbhhevMxQKqWc6ZcoUlflbtGjxK8t9Z0meu4NTOisrKwNs\n6VqSem699Vb1OiV7/pVXXtEErPvuu4/HH38ciK0HLeN2VtdIeKNz58706NEDsD2OBQsWAHaip4zR\n5/OpYiEKUefOnVXxSEhIaDDPybIsVVzWr1+vKlBpaSlHHHEEAGeccQYQGdapqanhm2++AeDzzz9X\n9SseqoCMyeVyRYRFBLfbrR5QKBRqFPJ2IBBg+vTpAOrJud1uDUUOHDiQf/7zn4CtbN1www2AvYbF\nc/3+++/12cXjmuRZZ2ZmMnDgQAC+++47Xbuigs6fP1+Tmh955BGV4huDglSXcDjMnXfeCdjKgCiQ\njUW5CIVCzJkzB4DHH3+cGTNmAPYcAVtFEkUpNTVVlYucnBxOO+00AK699toGqzSS9RcOh3Ue3HXX\nXYwbNw6wVQlRn7t06cKJJ54IwKmnnkp2djYABQUFzJo1C0DVUbfb/ZuhwnhgjIv/YVkW3333HQDb\ntm1Tg6Jp06b6knO73fr5XnvtpZJgfRtXQ4ZFFi9erPFor9er8doWLVroy/fss88GbCNo9uzZALz3\n3nuaf9GlS5eYbwiyqAcMGKDf3bp1a11MpaWlHHLIIYAdQ+zcuTNgbwJ1N9pgMKjX2tCbsCzalStX\nqny5zz77cOSRRwK1MU6Xy6WhtRkzZvDZZ58BtpQsOSONZRN25lqVl5er8dayZctG8VILBAIaFnPO\nleuvvx6wN1t5CW/dupWMjIxf/Y7NmzerrBzPcuykpCTy8vIAe96KUyKGT0VFhYZ+Kioq/kzMvMGR\n+fLuu+8ybdo0wH4u7733nv49nsj41q9fr0Znfn6+hktlXzn33HPZsWMHYK9RWa8XXHABAwYMAKKT\n9+LMdfJ6vbpfS6ivsrKSJ554AoBJkybpOHr16sVll10GwNChQ/Ud41yT2dnZv3IOs7OztaKusRD/\nXcRgMBgMBsPfCqNcOBBPyOVyqaVbt05e6tV9Pp/KVE5LNxYW/ObNmyMaT4mHV1JSopny4v107dpV\nlYvt27fz8MMPA/D666/HpN8C1N4T+b6WLVuq9+ByufD7/XotYql36tRJewjUd08bMhRSF/Ea0tLS\nVMVYt24d7777LmA3xgLbs54yZQoAX3/9tXqmgwYNinvlheAMi8i1FBUV6X0//fTT4+qFyvgqKip0\nrclaHDJkiIagnErEsmXLVB52hiMbIjT5Z9m2bRtgJ97JvHBK5DI/nBVSjQlJmL3hhht0/H369GGf\nffaJ57AA+z7KHnLFFVfoXEhNTeWCCy4A4KabbgJsFUF6c0yfPl0Vxb59+0Y08NtdqqurtSKue/fu\nWskhe0lGRobO79zcXG1G5gx/1G1uJ/f9iSee0MRyUTCOOOKIRqd4GePCgeQmtG/fXmVMqJXFa2pq\nVL7q1KmTNsGJlYwsEy0QCERUMMjCX7t2rU5M2aQ6duyoEzAUCqn8WlBQwF577RXxext63EJaWpou\nrKSkJA01hcNhNfCSk5N/c1yxemk4M+m7du1Kr169ALtpkGRsjxkzBrCvS6pxUlJS1NA75JBDGsVL\nztmh01l+OGnSJN1Y+/XrV29eRizHCHYIau3atUDt+mrdurUa1R6PR0sKX3vtNQ01ODn00EPjFg5x\nni3i9/vVqHfmhMj9TU5O5oQTTgDsMtrGEJZyEgqF9CUtFS4AZ511VryGFEEoFNLwzJw5czQU8vDD\nD+sYJXRdXl6u4cqioiKOOeYYIPphM6/Xq8bAzJkzNXQte+6+++6r1YZZWVm0a9cOsJ0m5/N35tlJ\nXtHLL7+sc0jGPXTo0EY3bxrXaAwGg8FgMPzlMcqFg0MPPRSANWvWaIKYUz4OhUJqeaalpcWldh7s\ndtgyPo/HQ9euXYHIUILQr18//aysrEyt6QULFmgCUEOHR5zyL0RWJGRnZ+v4nOe1NAZPX3Bm/UtC\n7NatW1WxEG+upKREs8wrKipUkdm6detOT8iMNZK46ewH8f777+u4V61apYpGPBquyXfPmDFDmwnJ\nnHjnnXd0TC1atFAV7ttvv9V/53K5tP/FyJEjY34NMtediuIDDzzAO++8A9SeMQK19/eYY47h1ltv\nBeLTRGtnyLU88cQTEY3iJITz1ltvMXz4cMD2yOM17srKSlUjAoGA7ocnnHCCzh3Z90aPHq2Nszp1\n6qSVdtHeA71er87DQYMG6eey1yUmJqqa4QzfVVZWamULoAnkV155pYZCoHYfkeopSX5vTBjj4n+4\nXC6NH/bp00c3gZqamog4qMj28Tx6/cQTT9QzLDp06KANeHJzc+vNIpbr2rp1q17L8uXLtdohVsg9\n/f777zW81LJly4j8EVlkv/cSjsdG5nK5VIYcMWKESsXSqXPq1KmaS1JSUqJS/YQJE7SEOZ7zJhAI\ncMcddwB2/o3kASxfvlyfx8qVK7VETvIwnEeJNzQyP7OysnRM8pJevXq1brYDBw7UzbakpETnQ2Ji\nItdddx0A++23X8wdAHkhl5aWcu+99wLwwQcfaNWKs0pH5sKAAQPiOi/qw7IsfvzxRwBeeOEFNT4z\nMzM17GBZFq+++ipgl2N36NABiN0cl3u5ZcsWDbO6XC4Nl02ZMkU747722msAPPnkk7rf7LfffjrX\na2pqIkrlo3ENxx13HBDpNDkPeHM20ZJy69GjR/PJJ58AsGnTJu0cKsaRXKPMd9ljpk2bpodqNhbj\n1IRFDAaDwWAwRJXGZS7HGQk1DB48WC1CafEMkZ5QvDxnsBu+iFqxzz77aHjD6/VGZMqD7WlIk6p5\n8+ap/Ldlyxb1EmOVVS+NgpYsWaIWeW5urnoP69ev1/NCnBUlP//8s45VGld17tw5rhUYCQkJ2qjs\nlltuAeyEQzmmOjs7W72N4uJi9b733nvvuHkWv/zyizZuSk5OVg/PGUpITk7WRmtyps7RRx8dM+VC\nPLvDDz9cvU3pDWBZljb7+vjjj7VCIBAI6Fzo0qWLnooaj5CIPPM1a9aouug80wVq17EoQ0uXLlVF\npnv37jGr4votampqmDRpEmCvOenTcMUVV+hcKS0t1VNUn3jiCR544AGAmCe6N2/eXNfiwoULVQWQ\neQC1VX6WZem8WL58OR988AFgh3sOPPBAwA5NRSOpWeak83k6f5/zO6Tib9KkSRoOrKioiPgZuZ9J\nSUkaZpX31KOPPso555wT8b3xxigXBoPBYDAYoopRLhxIPN2yLLVo16xZo7H1uiVC8fJA3W63JgZl\nZmbWq6aIxet2u7V8cv78+fz888+AnYQonl9DtbqtO5bVq1cDqGUO9oFCMqYVK1boNWRmZqq3tH37\ndv27HJp08skn8/rrrwPxT5KU7z/yyCM1j2Xz5s16EubChQu1lXW7du3iNm+aNm2q3pllWVq23LNn\nT/28VatWfP/990CtYhBL5N60atVKT2oVT3jjxo16OJMzSRZq126fPn30ecRjjcr3ZWRk6Kmna9eu\n1ZwLQD3/nj17AvZ6kJNfL7roIoYMGQLEtpdLXSzLonv37oBd5rj33nsDRHRCramp0byC6dOnM2HC\nBKC2RLWh16Xcm9TUVG1Jnp+fr319iouLdY44FQA5yfWkk07SQwfT0tLU44/WuJ35PvU9R/nM6/XS\nqVMnwN5DRHmprq7Wff7oo4/WzqNer5fLL78cQMu1CwsLVRlu2bJlVMa/uxjj4n9YlqUS0wMPPMDn\nn38ORJ5hAbU96n0+X0Rb8Fji7L1QWVmpm5UTZ4WGTLbc3FydgM6WtA29CUti3kcffQTYL1upusnM\nzNSFlZeXpxnUgwcPVtl4zZo1PPbYYwCaZPbFF1/o86rv+mOBJIbJyYs5OTkaomrWrJn+/3Xr1mnL\nZJEu40F6ejqPPvooYM8boVWrVjqHq6qq9H7K/Y2H8ZaYmKgvLrmnK1as0CZlZWVlERUi8vL+8ccf\n1Yjt2rVrg56WK9QX8mjRooVWIrz55psRBo9UM4hxsW3bNg0Tvvjii/r7Bg8eHPUX3u8hL+PCwkL2\n339/wA731efA+Hw+7RMxffp0dRLkucQqvON2u7VaYuLEiXq2yKhRo9Tgkb2ua9euWpnTvXt3PVHa\n2V/CmTD5ZwkGg78bQnd+LkbbSSedxMqVKwE7THzKKacAcOmll+p+GAqFOP300wH4z3/+A9h7kZyE\nLRVt8caERQwGg8FgMEQVo1z8j5qaGm3X+uGHH6ql27ZtW/WcAoGAJsR5vV5NLox1+2/LsjTR9Ouv\nv+b//u//AFsedJY3gZ10JTJhfn6+Wr/p6ema0Ne0adMG9fAkMVNKH8vLy9Vj+Ne//lXvAWXOcRx4\n4IEqMYvKUVZWpiceDhkyJObycTgcZvny5UBth84RI0aotxYIBPT+VldX/0qejQcej0cT35ylbR6P\nR8cXCAT0GuJx2Jrcn/Lycu2YK3N969atEYnK4iHv2LFDx7948WLtZPjoo4+qtN+QyZ3OZyp/9/v9\n6kE7E/M8Ho8mWEvCbOfOnTVR9YcffmDhwoX6u0T9aNKkSUyUUlFmX375Za699lod885kfQlHJSQk\nRJR1QmwTC2V8SUlJqgg1adJEQ7+iDIwZM0bXgFOhqPv3WOPspSPt7k844QTtkeHsWOzxePSU3aee\negqw77nMm8bCHm9cyKY0ffp0zXx2u936gO+44w7t2V5WVqZZvQsWLNAW4S1atPjNczCiTTAYVAls\n4sSJjB07FoB7771XY4jSLGnZsmUq2ycnJ+sGsHnzZv28oRuwyL0Rw2DhwoUaV+7WrZtumr8lH4rE\nKPH3mpoafXnHEnlJVFZWah5Ft27dADv7X64hGAzqSzEYDEacWxNPnH1EnJupbG7BYJANGzYAduv4\nWCP3d+nSpRrCkTWXmJjIfvvtB9iNp6SHwcyZMzW/pbi4WCsvRo4cqc2rsrOzo3Lv61YROA2LYDCo\nL9a5c+dqCDIpKSmieZYYb126dAHsEJrM65KSEpXyv/32W31BH3jggdpa3uv1Rt3AkH3w66+/Buw2\n5c7Q2c4QJ2zq1KmNosoFao2bt956Sw3QCy+8ELCdxd/LhYgG4XD4d5+RsxW/3Ovi4mKd182aNav3\ntF/nGUxOo1mM1caCCYsYDAaDwWCIKnu8ciEe/quvvqrWYM+ePTXjuUWLFhGHhIml2LFjR03Ckuzj\nWJGcnKzjKC8vZ8mSJQCcffbZ6t2IgtGsWTMNR1RXV7Ns2TLAVi6kPfWQIUPUQ2oIS17GKgpQRUWF\nWupOD7ouzhCPqDPy73r27Mk//vGPBhtzfYTDYU0c/PDDDzVxULwil8ulXtPYsWNVkvX5fBrWiXdl\ni1BdXa2hMI/Ho/e6sLBQPVAJB8YSeZYlJSVaVSRhNbfbrX1Qhg4dqopbz549NWv+66+/1vv+3Xff\naWL2WWedtduhEefBb3U/l7HLd+y///4RSdfSdRFq1TdnqEy86aysLA23HXDAAcyfPx+wKwDkILyG\nmO+iQIgC1KRJkwjPur7vDAaDjBs3DrAri6QLbbwOiwN7rGeffTZg742iGEpn2lj1PvH7/fpdiYmJ\nv+oUW1VVpVUhs2fPVhW0oqJCk92HDRumP1+3ElBUW7nXPp+v0bUA32ONC1nYkn2ekpKizZB69Oih\n4YMtW7boi7e6ulpLDVNSUiL6xMfypeH1ejn11FMBu7WwvNAqKyt1wkrJnjPuGQ6HVSasrq5WuVn+\nfUOOF9DN0e/38+abbwJ26ZW0J3e73REGhRyN/Mwzz2gIR/Jc7r777t16+cl92JX20DU1NXzzzTcA\nPP/8879azMXFxSxYsACwM9XlWrp3764x0niHRWTeV1RURGSzi2yfn5+vxsXvhasaAvmu7t27a4m0\nhBfC4bDmD40fP15LaYuLi/WY7ZqaGr3vVVVVagA6HYPdHZvz75Zl6dp3u916T9u1a6fGu2VZrFmz\nBrDzjmRvkbXq9Xo11+Gjjz7Sdel2u9VxSUtL03W6sxyIP4tlWZofInH7Ll266NkzztLGcDisc2X0\n6NHaOCscDmsllLPFdaz59ttvmThxImA/IzEqxHGMFW63W59XUVGR5jhJM725c+fy8ssvA7ZhJkZC\nx44d9Zm3atVqp20GxACRnKLc3Fxt/91YaBxulMFgMBgMhr8Ne6xyIZ6rWJQZGRnaz6KkpEQ9pJKS\nEq37P+WUU/QUup21dI0FLpdLFZTzzjuP0aNHA5FthuW6RCJ2/luwvR8Jl1RWVkZIoHV/NhrjhdqD\nfJ577jmVvP/5z39y8sknA/YJrps2bQJsZUBUAJfLxWGHHQbYbYYhMnnyz7ArSpPck+rqapWB165d\nq16IeKBQ6/lVV1erd/HMM8+oPBtvRLnw+XwRB8ZJEueaNWs00TAeCXryTHNzczn//PMBePrppwG7\n8klUjGeeeUb7poTDYR1/IBDQ55WYmKgJcdGSw+vOud/6b1FeunbtqiG8devW6R4iLamXL1+uql5x\ncbEqG6mpqeqNZmRk6PNwhmeisUadCeISJl61apV61gMGDFAldNu2bdpgsKCgQPeZAQMGcOKJJ0Zt\nTLuKhHX+9a9/6Zjy8vK49NJLgdiHI9PT0yNOQhYVVk5vXblypSpUwWBQ58Shhx7KscceC9hqVX3j\ndrvdWkwgyozH42k0bb8FVzxL46LAnxq88wwAkQM///xzlSYTExNVEszMzNQY3kEHHdToTi8MBAJ8\n9913gL0JS6msXItTQnX21c/NzeWkk04C4Pbbb9d8COcR886zSqKBzLVVq1Zx2WWXAfb9d4Yo5IXn\n8/k07HHeeedx9dVXA/Htmx8IBHRjvfHGG/Uey1yyLEvH17t3b55//nnAzlCPdzhEkJyRyspKzV/5\n5JNPNNzTvn17bTIkG168xi4v2bvvvhuAl156SV8idXEaxzLHDznkEL766isADaHEg1AopPe6srJS\nn4EYRD6fT+fSkiVLdMxer5drrrkGsF+Ukk/lzJGJxrOpqqpi1KhRQKQhJ98RCoUiToYWo6l3797q\nGJxxxhkaeoj1fLEsS5tJ3XnnnTruESNG6FyO9em4Mi5BnDxxnt544w01Ftu1a6clp82bN484nbW+\n0nzLsnSPltBLMBjUMP2fMKQa5IGZsIjBYDAYDIaoskcqF1ArD4snNHPmTM3M9vv9etZCt27dNKEp\nHtbvH8HptYnnLxnnGzdu1KS2tWvXarb98OHDVRlwqhuxkA8ty9KQwjPPPMO7774L2JKshKauvPJK\nbXGbk5PTaKosRKVYtWoVDz/8MICeftmmTRuVYQcNGhTzJLI/gng6y5YtU9n7008/VZVi5MiRKss2\ntr4FM2fO5J577gHs8JMz5CdrMyUlheHDhwN2a2Tx5hoLzv3W2bhM1IzS0lLtj+JsT21ZVoMlSjqP\nPpA98KWXXlIve+PGjTrvmzRpoqeNnnLKKap4xnN9VlVVce655wJ22EGqcV544QWdy/EYn1Ndkmct\nf/r9flU5PR5PvQrFrnxHMBiMSCzexd/TIMrFHmtc1PvLGiDfwPDbOCVjj8ejL7RoZ8T/ndkViVzk\n7eLiYg3bLFq0SF/Ip556aqOL3e6pmP3ot5H7s3btWj3jJD8/X5vavfPOO5r39He/f85wocfj2dXw\nvQmLGAwGg8FgaPwY5cJgMBgMfzkkxLBlyxauu+46wA5X3nbbbYDdhKqxhPYaOUa5MBgMBoPB0Pgx\nyoXBYDAYDHsuRrkwGAwGg8HQ+DHGhcFgMBgMhqhijAuDwWAwGAxRxRgXBoPBYDAYoooxLgwGg8Fg\nMEQVY1wYDAaDwWCIKsa4MBgMBoPBEFWMcWEwGAwGgyGqGOPCYDAYDAYHlmXxF28wGXd26ei0vzvS\nq96yLD1Fz7KsRnvUuiG2OOeFwWD4+2LW+e5jlAuDwWAwGAxRZY9XLnbs2AHAHXfcwQcffKCfBQIB\nALKysth3330BOPfcczn77LMBSEpKMtbt3xinJCqKVmlpKStXrgQgISGBjh07ApCamgqA221s9d2h\nurqa7du3A1BQUMCVV14JwF577QXAgw8+qH93u92qKMZzHQaDQWpqanRMTmTeeDwe3U88Ho/OrWAw\nCIDX6yUxMVH/f2PZV0KhEADl5eV89tlnAGzdulWvd9u2bXTr1g2AwYMHk5mZCaDX8lfCsiyqq6sB\n+7n5fD4Ao1rvBnuscSEL/OuvvwZg/PjxbNu2DbAXlddr35rS0lJ++uknAFauXKkL67LLLms0m4Bg\nWZZuCM6wjmx6Lper0Y05GoRCIX2ebrc7qi/56upqNSheeOEF5syZA9gbaJs2bQA48MADATjrrLNo\n3bo1YDalXUGeXXV1NUuWLAHg2WefZdGiRQD8/PPPAGRkZNCnTx8ADjvsMDU0fD6fPvNYGXgrVqwA\n4Mknn2TBggUAdO3aldWrVwOwbt06dUrKy8t1XD6fT+eGODaWZel13XjjjeTm5gLxWa9Ow0eexcMP\nP8z3338P2M+ovpfwgQceyNChQwE477zzAMjOzm70+40YgNOmTePll18GYNCgQfzzn/8EzDreHYyr\nZTAYDAaDIarsscqFKBDiHXXo0IEhQ4YAcNttt5GdnQ3Ali1buPfeewHb6zjhhBOAxiWBi1qxadMm\nnn32WQC+/PJLwJYxxRO68MIL1atIS0tr9F5FfTg9qw0bNgAwb948evbsCUCrVq2i8mzEo9mwYYOq\nW6tXr1bJt1WrVjRr1gyAWbNmATBjxgyuuOIKAI444ghVv+JBfZnuzgz4cDisz9/pIe/KnIjW/Kmq\nqgJgzZo1vP766wDMnTtX/39WVhYAGzduZMqUKQB8++23pKenA9CzZ08GDhwI2M+lIb1N2Teuvvpq\nAPLz8ykrKwNstaK8vByAQCDAxo0bgUjvNykpSderKACBQICFCxfqz8h+k5KS0mDXURfnvAD44Ycf\nuPXWWwFYsmSJPiOofe4JCQn4/X7AnvuyDt58800A3nnnHfbff/+If9OYCAQC3HjjjQC89tpreu3t\n2rVrVPu74FzT8neZS8FgUOeZx+NpFIpL47uDBoPBYDAY/tK4/uK1vH968OI1SIx0zpw5DBs2DLBj\nu2Jph8Nh1qxZA0BhYSG9e/cGbKu9MRAMBvnmm28A25sqKioC0ASypKQk9YA6d+7MddddB8DAgQP1\nGqLtVTjn1O78bqdKIc9LEv4ee+wxPvzwQ8C+xmuvvRaAiy++WOPAu4N4aq+//rrG13v37s2hhx4K\nQHp6uo5P4tH33HOPfvfrr79Ohw4dgNh5bTIey7I0WdCyLCoqKgDYvHmz3r/k5GRNwMvMzNSkVGdi\nm0WeEVgAACAASURBVMwhy7I0PyA5OZkmTZoA0UtqLigoAOCaa65Rxc3tdtO2bVsA/vWvfwG2gjF7\n9mzAVgmaNm0KQKdOnejevTsAvXr10uvyer1Rv/eiqMh8S0lJUbWipKREPcbq6mqSk5MBew+Rv6em\nplJZWQmguTylpaX6+1u1asUrr7wC2Gs0VuqXeO0y78877zy++uorwJ4HMg6fz8c+++wDwL777qvq\nzM8//6z3QX62X79+mggaSxXm95Brzc/Pp3///oCt+ubk5AAwYcIEDjroIKBh165zvcpaq6io4Jdf\nfgFg6dKlbN68WX9GcmA2btyoa0bWamJior6brr/+es33SUxM/CPX0CAXuceGReTBfvfdd4C9mcni\nELkVbBl069atgJ201FiMClkgX3zxhYY6qqqqdEO++OKLATvcI5LrnDlzeOONNwBo3bo1++23HxC9\nDHXnYnEmWAr1fUfdqgznJjdp0iTADvHIy01emt9++y3FxcWAvYDy8/MBorYZy6a5ZcsW/vGPfwD2\nZp+Wlvar65JwWlJSEo8//jgADz30EI8++iiAvoxjRSgUUiOzpKREX9g//vhjhPwtm+lhhx2m83rR\nokUsXrwYQI1qj8ej99Xlcmlo8KqrrtK1sjvzR17ICxcu1E02JyeHc845B7DDeQCVlZVanVBUVKRz\nPTs7W+eN3+/Xa0xNTdXfnZCQEBWpWwzGBx54AIA2bdpQWFgI2HNT7tOSJUto3749AIsXL9YQWkVF\nhb60161bB9j3TuTtiooKZs6cCdiJhbHAuV7FkJ45c6Y+i9TUVPbee28A+vTpwwUXXABAy5Yt9eU3\nYsQIpk2bBtSGjhYuXMiyZcsA9Ln92fHBH59jzn2o7r+xLEvHt3btWjWmLMvi4IMPBiAvL6/BjAqZ\np6FQSN83S5YsYfTo0QBMmTJFHYCampp6w5ter1eNCvn/CQkJarQec8wxOvfiGY4yYRGDwWAwGAxR\nZY9VLsTDkHLC+fPn8+OPPwK2hyqeREFBAc2bNwfscEljwLIsLc+77LLLNKmqf//+qkyIpxQIBLTE\ndtSoUSqRB4NBnnzySQBatGgRVQvXqUCEQqGIUlgZv1jcNTU1qkoUFhby8ccfA3ZSmMjFKSkpWnYo\nCZWBQECl/H322YerrroKiE7pWDgc1u9p3rw5nTp1AiI9YScyl/r27ctJJ50EwNixY3n44YcBuP/+\n+2OqeMm9B1i/fr2WSs6ZMyci4VBUh8mTJ+u9rqqq0uckz83ZpyEtLY2lS5cC9vNyqnx/FvEey8rK\nIsZ++OGH63eCPQ9EBfJ6vRFKhHM+yVh/+uknJk6cCNjqh8j5uzPXZSz9+vUD7HvdqlUr/b2irLVp\n00a9y+zsbFUpZsyYoYqQ0/uU+datWzdOPfVUILZJ484wINjlsxLi69mzJ6effjoAxx13nO6Dbrdb\n12D//v01oVOUgcrKSlVNDzrooD+9Nv/I85J549xPUlJS9L7KvXSqGWVlZfr3jIwMLrroIsBWIKON\njE/mZlFRkaoVH3zwAevXrweISJx1u936DFq0aKHzt6CgQBUh+X0ZGRncdtttgK2kRiM0vLvsscaF\nTPQePXoAdtz8v//9L2BPQJE6MzIyOPnkk4HGk/FcWVnJPffcA9gLRKTa9957TzPrZaxr1qzhrrvu\nAtDwDthSXN2eGNEiHA7rItmxY4cucFlgZWVl2i9ixowZLF++HLBlYudiP/LIIwE47bTT9HmNGzcO\nsKs4WrRoAcC///1v/Xs0cLlc+tLs37+/bqY7u0/yeUpKCmeccQZgx6BFpiwpKdH8gFjEcMPhsBoz\nTZo00b/7/X59obnd7og+C85NTZ6XxMlramr0s7S0NDVWJ0+erBve7oSjJARVXFyscyQ1NZW8vDyA\niCx4YWf30RljXr16NePHjwfsdfDYY48BaC+SP0Pd73VetzN2vmnTJg1/TJs2Tef4jh07fpXpn5WV\nxYABAwA73BJrSduyLM09k1CkZVn6gurSpQvHHHMMYBtKzhe1XH9eXp6uGTFgw+GwGqINgdy/QCCg\nYUBphAgwdOjQXzklLpdL59ibb76pIYi8vDyOOuoo/ZmGQhzBL774gsmTJwP2vBeDpkuXLpr716dP\nH839gNrcnLvuukvDV7JGzzjjDG3w6PP5GsW7yoRFDAaDwWAwRJU9VrkQy04yy/fZZx+tuvjuu+/U\nEk5KSlJJMN6Ix7N69Wr1/DMyMlRek2sBO1kJ4Mgjj1QVxnkIW//+/TXcEy0rV36Ps846JSVFQzGb\nNm0CYOLEiUyYMAGwVRjxpvv06cNxxx0HwFFHHaUSdDgcVu9WfrZp06aqbPTq1SuqErLL5VKPIC0t\nTb2bpk2b/qa063K5VCa+6qqreOqppwBYtmwZffv2BaKXcLqz7wfbS5PqhI4dO2pipNvtrrcufuvW\nrXqve/TooT+zZcsWAH755RdKSkqAyMqSn3/+WXs87E7SqnTAFa8fbA+5bsjlt+aps0+HKDUDBgzQ\ntTFp0iSVjV988UUAvUe7wm/1A6mpqVHP+e6779YkY2cHWa/Xq4qA9NI599xz9Rm1bds2Zj0KnJ1R\nJRwizzwQCKgK2q1bN91b3G53xLXLWLOzs/W+O9eiyPfV1dVRrRhxqkQFBQWMHTsWsBOSRQVq2rRp\nhGIh/0729ilTpmgY6/DDD2/Qihbn2pSxSUJ1YWGhJrzeeeedGqp3qnCBQEDViilTpqj6Isr7rbfe\nqvOqMagWsAcbF4IshGbNmqncu2bNGn2JDRw4sN7TUuOBjGPhwoU6vt69e6uRUFlZybfffgvA+eef\nD9iSm7NyQ+T5m266qcHOAHC5XBEvUfkeeUGtWLFC5cH27dtzxBFHADB8+HANb3i93gj5+K233gLQ\nTPq+fftqOWBDxBflu7du3arlh82aNdNclrqbrOAMkUjY4b333qNz586A/RJuqFi6M1dCXpw+n49e\nvXoBtuQqLwOndOr1evXF4MymF6l82rRpGv5ITU3V3z158uSo5JJIDoIzF+fQQw+NqFDZFeT+tmrV\nSit5Vq5cqc3Qnn/+eQCuvPLKXZ47dSsXnGNetWqV5tnk5+dHlAMLoVBIQ1CyHsaNG6ehkJycHH1G\nDVFK60SMyI0bN/Lpp58CtbK9y+XScsY+ffrs9BwXecF/88036sTI7/V6vbrfOEtzo3VN4rSMHz9e\nK1WGDh2q1UzOlgLOa7755psBO8wgc+WSSy5p0Hstv1vmW/fu3TUMX1RUpHN9woQJms/Ttm1b3fMn\nTZqkDb/Kysr0Xt5///2AvTftrBovXu8sExYxGAwGg8EQVfZ45UIoLy9X+dvr9arVXlFRoe1sTz/9\n9N9N7mtIxIrdunVrxAmEImkWFhYydepUoFZ+b968uXoXfr9f2yS3bt26QWu5xSNISEhQGV36Egwc\nOFBl9EsvvVQ/T0xMVA/Jsiy93nHjxvH000/r9QBcfvnlKuU3xHWI1/nhhx+qvL18+XLtKdKyZUtN\nwnKeoCheaiAQ0PbkeXl5EdcVC5wVOs5x/t4BX85W4DLmvLw8DjjgAMD2rOUaMjMzo3I9kkj3zDPP\n6GfRyNhPTEzklFNOAWDMmDFasSGJlieeeKIqMn+U+uaaeOoZGRkqUxcUFGjIKBQKRYRTZF6L5716\n9Wouv/xywD6dWcJ9TzzxhErnDYFTGawb7vH5fKo4rl69Wo8QcCp2VVVV2kPl3XffVcXLOX+irVYI\n4XBYE9R//PFH3U/69etX7x4tz+irr77S529ZFu3atQPs6p5Y7Omy7nJycrSvRlVVlVbafP7557qf\nNG/eXOdIYWGhqkper5djjz0WqD000eVy7bTlf7yUC2Nc/I/y8nLNLO7Xr5++XCorK7W8c8KECZx7\n7rkADBs2LOYNtZzNgWSxb9u2TUsNExIStIOkdDVctGiRxv5ramq0uVZDhERkfJWVlbpAnEdjS5b+\n0UcfzYknnghEvvCcG28oFFJZ/sYbb9SFJY2/OnTo0KClejKOyspKPX8mNTVVDY3q6mqVt+VlMGTI\nEI3brl69mpYtW+rnDbXJ1odzo3F+385COU7C4bCGc6Q0e++999YN2+Px6Eadk5MTlXkk98n5/KXU\neneRMtFjjjmGUaNGAbU5HgsWLNhl46IuzhyP1q1b63dcd911Wg0gjfrAPgdHwq8yf8LhcET+wNtv\nvw3YoQbp4tkQ+QAyF/x+f0QYTf6U8X333XfqIOTl5akRMWbMGG0at3XrVp0Xsi7T0tI0HBjt/SYY\nDGoIOCkpSY26vLw8daxCoZCOVYygyy67TKtZXC6Xjm9naybayL1JSkpi+PDhgO10yb5SWFio66+s\nrEzfQ/In2Pkt11xzDVDrRDqdMWeo7s+eGxQNTFjEYDAYDAZDVDHKxf/Yd999NfmoS5cudOzYEbA9\nVPn8k08+0ZbOGzduVCkzVgqGfM/w4cPVkk1LS1OvomnTphE95cGu5Rar+KCDDlILvyFkSvEI1q1b\np/fPmZTmzJR21sqLxR0MBtWDW7hwIVdeeSVgq0oSBjrttNMifldDIZb/tm3b9LqysrLUw6uqqlI1\nRc5P+OSTT1RmDQaDKttWV1fHpCGSswGVk13xXCorKzV5VjzDG2+8MeIZiTcI0al+kX4gTubNm6ef\n76rX7pSHZZ4cdNBB6lnLc3vuuecYOnQoEJ3rcDaVOvjgg7VHwTXXXKP3rLS0VM+FkOq0OXPmaLKp\ns5FYQUGBngX07LPPRn2fkfmQlZXFYYcdBqDtvNPS0lS5WLx4sX7u9XpVBaibtFp3nuXk5OjvkHu/\nu8i9WbJkid6/7du3a+Mxv9+viaVjxozhnXfe0bGCvW5lfvh8Pk2M37x5s4bi/uB5HLuFx+PRaqG+\nfftqeGP9+vXMmzcPsNUW+XthYaEqwCeeeKKGgZz7qDyLiooKva7MzMyIxO1YsscbFzJZ33rrLX1x\n7LXXXloVkJCQoC+05s2b64SeP3++loP27NkzJuVjzkUrYY+6Z3fUlZU/+ugj3diGDBmim1+0cbvd\nKvd+/vnnnHnmmYCdxezMfpefdS5e5yFxUq565ZVXahmbz+fT7nkSa2zIhWJZlp5b4vF4tCRvwIAB\nHH/88Tpm2eyl2qG4uFhjpKNGjdKue506dYr4d7/33bu6scnG7Sw5ls0nMTHxDxk2Mm9++uknlfal\ngiE5OVmNvlAopBtXSkpKVOZ9165dgUgJt6SkRA9wkg6pu2qgOZ9R37599e/OF56zsiHaODd1+f3J\nycmaN9SlSxcdhzy7E044QcNw4XBYDb1jjz1W51C0qamp0bGIQRQOh5k/fz5gN7qTl3N5eXmEoeDM\nr3DmWcmfso9G6+wieYFOmTJF88tcLpcaEePGjYuowpB5K+GexMREHX9KSopeb1JSkn4eizwFl8ul\n9ys5OVkN6JycHM1v2r59ux7U5/F4NHx40kknabjPWcUj409OTo7oJh2v49dNWMRgMBgMBkNU2eOV\nC2ka8/HHH6tleMstt0RUIojVe8IJJ2gN8j333MMXX3wB2Alv4o3EQv72eDwR2fT1Hd0rHo/f79eM\n8/PPP79Bxyff3b9/f23i5XK5NKzk7KXgHLv8d3l5ubZoXr16tf78ueeey+233w7UVhE0dE26JMwO\nGjRIm9ocddRRmgCYkpKiY5D265ZlaYXAPffcox7+rFmzVD36vcZNNTU1u9x7Qe77q6++CtjJdVdc\ncQVgh/v+iOcioZ+rrrpKFSg5xrx58+Y6poSEBPWyotWHQdS04447TttPJycn6xHToqD8kVBY3Yx5\nGV9paamqM/JZQUFBg4fX6qNuNY7H41FPdMyYMVrRtWXLFlVZnnvuOU0AjNbcF0935cqVeq+lcdmO\nHTtU/Vy3bp0qcs776/V69f7l5uaqSiHrZe+99+boo48G7HUbjXGLcrFmzZqIk5KlKZ/X69VwWmJi\noioy0kp+/vz5ukYPP/xwrr/+eiAyVNsQe8vOKjnkT/nOUCikKtHbb7+tFYyZmZncd999gN00sL4E\ncWcTLWeIyvS5MBgMBoPB8Ldgj1cuxDMIBAIRJVT1WbEej0c7SPp8Pq2Xzs3N5YILLgAapmSsLjuz\nRC3L0pyF9957Tz8fNGgQgHrjDYUkkzrbTAcCAfWsRYkIh8MRMW7xKF966SVVXMLhsHbufOihh/5U\nq+bdQRSSYcOGaalvXdXHeWooRB72lJeXp8lYLVq0UGXA2cvDWf5WXxnvHyEcDuv45JCuzMxMjTtn\nZWVp/kXdhEC579u3b2fEiBGA3TZe4rXyWfPmzSPWQ7Q9Ifl977zzDnfccQdgx/mlM6r0wUhISNjl\n75ZrnDt37v+z954BUhXZ+/DT3dP05EzOcQgDSBRUgoKSRdSVXRHDrqg/xPQ3sqZdc1xXUdcARkyI\nICaSggIiSBQk5zyJGYbJHe59P9z3PFQPM5I64dTzxbbp6a5bdarqnOek4wIOMzIyQtocTMZTOfDW\nZjvWTVWNE1Ch7qlAxIeYpknLf968eYxvEni9XrKPahCkzWaj/CYmJrJs9e23344uXboAOMZ+uFwu\nylwg/P5q5d82bdqQEVUrVmZmZrJ54GWXXca5/uabbwBYLIycg+PGjfOruBsKyDyqqaV2u53/X15e\nTrnPysri85533nmMN4uNja1yvDLHhmH4yXUoUmyrQo1XLoRCi4+PpyD+0UUmC5iamkqXyueff85g\nq1DUM1BdCfL/gHWRS7dUKS3sdDrZLS/Y0cIyN/Xr12cwXnx8vF+gFGBtJplrwzDYlvm5556j+yA9\nPR1vv/02gNPrAXGmkM2bnJzM1t/qIWsYRpWFsUQxuPfee/Gf//wHgBUkJzUjevXq5deyWuZBDe46\nFZSXlzPoSy6D5ORk1nIoKipiQGRZWRldEB6Ph2v022+/URlxuVwsmR2IjqengpiYGHYg3rlzJ7NV\nbrnlFgCW++RE+6ryoSr7+7///S/XSaL0X3rppVMe4+kc1GqX4KysrOMu2q1btzJbZNq0aaTtTdOk\nXLRp0yagF6DP52OPje3bt5OKF/dHdHT0cReg/FeUh86dO+OBBx4AYFH1YlgF070gysqECRN4rpWX\nl3NNXS6X3/zK84gCfvToUZ7Vge5JdCpQ974a3Akcyz4rKyvjXF999dV01Z9OYHM4oN0iGhoaGhoa\nGgFFjWcuVMtR0sF27tzJvOPKnxXrUBr9AFbVSKnDEGot0TAMBg7OmTOHFpDgnHPOoVskVFp6TEwM\nXSRqJ9aqalsUFBTgtttuA2Bp6mIl33rrrX6VG8MFlQZWXV5VpauZpsnx9+3bl8zWjh07sGnTJgCW\ni0Ro2bi4OFomYhGfai0Dl8vFAMAhQ4YAsBgMqQop6XqAFdyrWt4SjKe6qTp16sRg0FAzRna7nTR7\n69atGTAttWUeeeSR4/L7q4LqYpJumQcOHODf3n777QCOlaM/WVTFFlYnm4ZhsJ7GJ598wnXIysri\nfpXv2LBhAy1s1WXicDj8Gg0Gch/YbDZ22Txw4IBfN1SBsIgqzR4VFcUgyXvuuQe9evUCYMlKKM4X\ndRzi0qgOhmHwTJe02piYGPTt2xdA6OoTAcfLi91u9zsP1bo64k612+0M9O3bt+9Ju5bCxcZURo1X\nLoT6dblcpCMvu+wy+pv79etHwZw5cyZL+no8Hiog99xzz3HtoYMNtV3yggULAABPPvkk6VcRysmT\nJwela2hVqCrnvTqonSSlQE9MTAwzFO69996I2SSCqmpzVPeZWrVq8WLYs2cPD7mFCxdS2WvWrJlf\n/QgAp1yHxOFwMBL+1VdfBWD57cUtsmzZMkb9x8TEkOpWs40KCgoY4X/VVVex7kQ45l/GNXr0aHz6\n6acALLcjYNXxeOaZZwBYrjc1hkfmr6ioiBT41KlTsWjRIgDW/pbS96JcnA69rLrFqvoOia159NFH\n8eWXXwKwsj5UpUEtca/+v/yGKH0tWrTAU089BcBSSgOpXJimyQy3vLw8P1eMPJeqTMlzJiUlYezY\nsQCACy64gOsVaXtVIAXLJL6kdevWlO9Q1n9QFQnAvyaRx+NBYWEhAEtupPhedHQ0u1unpKT84fob\nhuG3dpGAyBiFhoaGhoaGxp8GNZ65EMZhypQpbEqWm5vL/GdVe4+OjiZ9fOONN+Luu+8GYFWhDDV1\nLxbv3r17cf/99wMA9u3bR8v3X//6FwArgyMcboUT/aZo14Zh0P3RtGlTBnEGoitmqFE55/zcc88F\nYFG4YkHn5+fTSszOzuazC7vk8/lO2aJSg4wBy8qR2ht//etf/SxjobqPHDnCpli5ubl0EXTo0CEs\ntR8EIhddunTBPffcAwDsiPv777+zeqhqyauVXbdu3crXxcXF/L4RI0bQ/XYmTF5lpkn9roqKCmbs\nfPLJJ6xR4PP5qm0kB1j0vLigevbsyeq2gwYNIvsVaCvb4XCQtT3//PPp5pXnUzsbq1Vq//a3v2H0\n6NEArLMznC7LE8E0TQaqSpbf6NGj+SzhgMyXeq+UlZUxu2/u3LlkFzt06EBZCFeVzTNBjVcuZLEH\nDhzIrIU333yTroY9e/bQr3jNNdeQtk9LSwvrgssl8eSTT7KVNGC5cQCrRCwQ+UKZmJjIttjNmzdn\nZ9pIPrROBjabjcXLOnfuzMtDLT9cXl7OCHCh+APx3JXTRVUZkN+JjY1l7IfX6+VFEqjCWGeKqKgo\nui9E6f/555/57zt27GCxr7y8PHbQjYqKoosnLS2NFPgNN9zgVxjvdGAYBku6y29Uhih1Q4YM8ctQ\nUPtZSHyWdDBu3749+vfvD8Df3aOuWzDSf0U+b7rpJr8OqIAVn6MWkpMuxhdeeCENskiQkz+C2+2m\nO1KMlTZt2oS8x0ZVUOPODh06hGnTpgGwzgQZ63nnnXfSc11V6unJ/F0wod0iGhoaGhoaGgGFraqy\npGcRzurBny4Mw6DVNmDAANYr6NWrF4tQCZ16spprqAutqM22hHlRs24inXE5FZim6dcVU9wXan67\nSpeG2tqIFEvnRFADNw3DYJDe4cOHWQSqb9++dOtERUX5FXALBDsk1qbMmSqn6vgqKiqYFQIcc5+o\nVrPqFgnnvKvyKQGQOTk5rI8SHR3NOY1k+aiMsrIyBjYvXboUAHD99deTkQnns6i1T2bPnk0Xe0VF\nBTPt3njjDbRt2xbAiWvNeDwevzNEPn+SzxiUidDKxVkIr9dLf+7999/PlKz77rvP7+KKVKiUoMfj\n8euyGcr0MA0LaipcOHsRBBPhUNo0wgv1Ahcl0Ol0hiWbonIhwfLycqYqT5s2DbNmzQJgZflJNtmo\nUaNO2i2iZot4PB6/PiMngaBsDO0W0dDQ0NDQ0AgoNHOhERZU7qkB/Hmt5kiHylyoRc80NDQCg8rn\nndvtJnu7ZcsWvt+oUSO6oFQm92TOxTNwbWu3SBU4qwevoaGhoaERZmi3iIaGhoaGhkbkQysXGhoa\nGhoaGgGFVi40NDQ0NDQ0AgqtXGhoaGhoaGgEFFq50NDQ0NDQ0AgotHKhoaGhoaGhEVCEv4OLhoaG\nhoZGDcXZUn7/VKGVC42IhGmaKCwsBGD1YDhbOjFqaGhonAxycnIAAAcPHkTDhg0BAHFxceyKGo4y\n5YHE2T16DQ0NDQ0NjYiDZi40IhKlpaX47bffAADt27cnc6GhoaFxsqhcdlttmqh2Gg0HI5qcnMwx\n/ec//wEA7Nu3D5dffjkA4LzzzmMjSgAB6eobSujy338CVNWnQ6AKotpDAsBx7b7DDdM02e31iSee\nwMGDBwEADz74IDp06AAgMqlCtcOrdD9UWyDbbDYeZE6n84Ttk0MB0zTh9XoBWOOTfiKRIgtnC8rL\ny/0O/erkU+1YKfIic+5wOEIu1z6fDzt27AAANGvWjO/LmNTxnA0yIfPr8/ko10VFRezTUVxcDADY\nvHkz3RG1atVCixYtAAAZGRnsJGq324P+zKZpYu3atQCARx55BGvWrAEAHD16lArFueeei969ewMA\nhg0bxu7XiYmJgZYXXf5bQ0NDQ0NDI/IRfhPqTwDTNGmxlpeXA7C035iYGACB0fwNw6BGnp+fj6ys\nLP7e/v37AQDbt2/nOERTr1+/PtLS0gBYGq9YTV6vF507dwYANG3alNZXOOF2u/Hxxx8DAD799FPE\nxsYCAAoKCiLOejJNE2VlZQCAadOmAQC+++47unLcbjfi4+MBWN0NU1JSAAAdOnTA3XffDQCoW7du\nyDuQigwVFxcjPz8fAFC7dm2uv9PpjDhGqyqoLJw6zlCP2TAMP+ZQfS3z6PP5uB8XLVqEWbNmAQBd\nfX369MGgQYMAAPHx8SF5hr1792LRokUAgDfffBObNm0CYAUUAkBmZib69esHAGjevDnq1q0LwJIP\nGV+ksIher5f77qeffqKF36FDB7Rp0waAdfYBQLt27VBSUgIAmD9/PpYuXQoAuPvuu/mMoYBhGJg/\nfz4AYOnSpTh69Cj/7fDhwwCAZcuWobS0FACwbds2dOvWDQDQqVMnnHPOOQAQESxodYjckQUZJ3IH\nVbfBVfpNBGLr1q18X2j9evXqoXXr1gDO7MBQDytRHCoqKvDrr78CsDaTvJ+amspLQt7Lz89Heno6\nACA6OhobN24EAGzYsAEtW7YEAEyZMsXPtxdqyFh//vlnvPXWWwCsy0/G3apVq7CNrTJkPfLy8nD7\n7bcDAGbPng0AKCsr42avV68elaOsrCzO+6+//sqD/J133uGBFqwLRfUxu91urFy5EgDw7bffoqio\nCABw0UUX4fzzzwdgUcVut5uvASAmJoavw92OXebfMAx4PB4A1n6QQzgxMZHR9qEYa0xMjN+ZIHPn\n8/k4vwcPHsScOXMAAHPmzMGWLVv4twCwatUqrv+ll14aVEVf9lpubi5effVVAMChQ4c4VlE+Z86c\niSeffBIA4HK5qBy3atUK3bt3B2ApRXLGNW/enDISKgVPnuXLL7/EAw88AMCS2ccffxyApUiIFazd\nUgAAIABJREFUq0PGFBcXhwsvvBCAtRdFwc7NzUWdOnVCMm7AkmM5f+Pi4ii/ycnJGDp0KACgTp06\nvGMcDgdWr14NADhw4ACfq23btgAsJSPSjIHIUD01NDQ0NDQ0/jSokcyFaZp0X+zatQuA5V4QOi0h\nIcHPUhNtfs+ePbRAduzYgb179wIA9u/fT4tVAg+vuOIKWqUul4vfd6pQKUixaBo2bIgxY8YAAK66\n6irSk1VRZHa7nf9eUlKCzz//HACwevVq0oPhhGEYWLVqFQAriFPGetlll+Gyyy4DYGnzkaCVG4aB\nQ4cOAQBuv/12/PDDD3wfsNwcN910EwBg9OjRtEwPHDiAF198EQCwYMECLFu2DADwv//9Dw8//DCA\n4NGbKnORnZ2NDz/8EADwww8/oH79+gCAHj160HLKysqijIvrzeVy4W9/+xsAf2swHJBnKSwspCU3\ndepUWrHnnXceRowYAQBo0KBB0Kl7VS7tdjvZEqfTyTG53W5UVFQAsCxrOQvkXNm2bRu+/vprAMCg\nQYMoC8GQeRnHTz/9xMDG0tJSzpOsreqG9Xq9yMvLA2AFSa5fvx4A8PHHH7M+w8UXX0wZkSDJ6Ojo\noO1bwzDw/fffA7BcGmLhDxo0CMOGDeOzVPX7cu5t3ryZzEWog5rtdjvOO+88AMAbb7zBtejevTuZ\nWsMwWOtnx44dmDJlCgDLDSvunJtvvhkAMGTIkJAzRydCjVIuhL6sqKigj27ixIkALOVCNlhSUhIX\nav/+/cjNzQUAv8u4fv363FgZGRn07YlfrFu3bqQSo6KiqvQPnwrUjAP5zjPB0aNHkZSUBACk70MJ\nmY9Vq1bhjjvuAGAdyE888QQAoGvXrn4K3pnOXyDGevToUVx77bUALH+o+Mx79OgBAHj22Wd5sNaq\nVYtjrVevHpWLPn36UJ5mzZqFhx56KOjjF/fBjBkzMG/ePADWhSeXR1ZWFi+MLVu28LX8Nzs7m66f\nDz74AJ06dQIQOp+7zL/b7WZU/ccff0zXYE5ODvfDhg0bcODAAQDAQw89RBdJMKEaACKzhmEw5iYj\nI4MuPuCY0padnQ3Aeq59+/bx30SGgiHr8p3NmjVDkyZNAFhnmRhF4stfs2YNL16v10vlMzY2lkpT\nVlYWtm7dCgD48MMPGS911113AQD+8Y9/BK0gVHFxMT744AMAlnIkcWVPPfXUH16yXq8Xzz77LADg\nl19+wYABAwAAbdq0CenZYrPZaHxecskllHGHw+E3DlH24uPjebfMmTMHu3fvBgCsW7cOADB58mRc\ndNFFfn8Tbmi3iIaGhoaGhkZAUaOYC4Hb7WakrgRXVVRUkDJUc6VN06QmGR0dzQCmBx98EBdccAEA\nKyBHpUOB43OlI4GqOnz4MD766CP+/9ixYwGER9MVmvXf//43KcFnn30Wffr04ZjUegDhnD9hrMaN\nG0c60m63k6164YUXAFjBblVlWkRFRTFgNjY2lrIlzFGwIZby5MmTGYmuslVJSUlk4fLy8tCoUSMA\nwM6dOwFY+2X79u0ArOBfKfgTbOZC1l9cmIsXL8aXX34JwLLYxILu1q0bAxLXr19PxsXtdh8X0BdM\nqL+hsm1JSUm04Lt06ULXlDBKtWrVYq2JYO9FseqHDh2Knj17AgDS09P5uzKnQ4cOrbLnhbrmeXl5\neOSRRwBY1rQwckLfd+3aFV26dAFgBa8GYg1kTBs3buS+jIqKotVev37942r7AGCg7dSpU/Htt98C\nsBglYRRDwXCpsNlsfnVwqpsbuVdiYmK4L8vLy7kn5M765JNPWBNDzegJ57lZo5QLNdJcfHSy2QoL\nC3noA/BTKGTjT5w4kXEA1fkTI0GJUGGaJovlDB06lD68oUOHMv0t1GP2+XxU7n7//Xf6SAcPHsxD\nLi8vz6/wVLjg8/nw5ptvArAyLOTCy8zMxNtvvw0AjPr+o+wEueD37dtHOUxOTg76BW2aJn788UcA\nFg0vMh4VFcWDf+DAgXTxpKWl8XlE6du8eTMvQlEygjVWgcfjoQL6zjvvALAMARlHt27d8Je//AWA\nNe+vvfYaAOuwlTgqdT+HEw6HgzJcUlLCS1GN1RkyZAgAa/6DuR9FRmNjY+kWqRw3Ahy/51S3pIzb\n5XLxc6pbWdxBtWvX5nuBas6lFiMTt02TJk0oCz6fjwqSYRh0kX322WcArFRgcWPecMMNrJIZTpzM\nfBiGwf3o8/k4DzL/derU4feoBnE4od0iGhoaGhoaGgFFjWIuRJsrLS1lPQqhlex2O63mhIQEBgh1\n796d1F/jxo0jvsCQWBVHjhwBADz88MOYPHky/y0zMxMAcOutt552BsuZwuPxYPHixQCsugSSlx4b\nG0v6Misri1ZWeXm5XxEfIPjzL5ZBdnY23n//fb4vGUWvvfYaGa0T1VNwu9245pprAFiyJ/PeqVOn\nkMiRZLj4fD6OtWvXrqwNULt2bb8MBdkHYhkKBSs405YBhmH4MTZqLRdhJnJycjBp0iQAFrsFWPMl\nAXg9e/bkOEtLS/32tgQfVldoK1SoihEoLy/nmSNo0aIF92V1ezLQ1mh131WVS8Hj8ZBt8fl8zJJ7\n66238M033wCwnq9jx44AQBdF5TLVgXSL1K5dm/svOjqacuN2u8lKr127lkGfUmiwbt26HKfdbmch\nvEisE6HCNE2yn16vl2MV1+rgwYOZnVaZDQ3XHtDMhYaGhoaGhkZAUaOYC1UTlxQqCQQzDINBd/Xq\n1aP2fd111zGNzDCMk9L4Qw3xLe/fv5/BhVOnTgVgPZ+q7f/zn/8EcKweRzhQUlLCpmQ33HADS/MC\nx4Lctm7dyvTH8vJyliUeN24cACv9M5gVGIUBmjFjBi21qKgoBk01btz4uM+qQbxqRdXnnnsOK1as\n4PtSX2LChAlBj7lQS08DYBXCp59+mkGcalqzWkp5wYIFACxrUKyiiRMnnnEadHXP7PF4GHA6Z84c\nBmY2b94cgGWdSXVINQDPNE0yYaZpsgJjXFxcRFijqizs2LGD8iJQ0xKrszrV18F8JlV+hUWcOXMm\nXn75ZQBWOrYwA0eOHOG+HD58OP7+978DANsKxMfHB1y+VZZL2J569eqhffv2AKyzRSz8xYsX82zM\nyMgAYFUxliq1s2fPZlD+ww8/HPKgzpNZT/lMUVERPvnkEwDWfhQG9x//+AcAi8lT2cdAxbicCWqU\nciGTXFRUxMtNFkGN0q5fvz7Lqh45csRvg8jF4HQ6SWGqhapCvZClpaW47rrrAFiXgShLcpjZ7XZe\nKJ9//jkvxXAeusXFxTy4unbtyrkzDAPbtm0DADz//PN+JYlF6ZACVPPnz0eDBg2CNkb5vU2bNvG1\n3W7n623btjFzSFwlKg3v8XjY9fDFF1/0y3yRoGC19kGwYLfbqUSoufIZGRlVKgklJSW48cYb+Vq+\nQwr+dO/ePeAXhrgJDh06RKW/oKDguE64cXFxlGu11Pr69ev5HdHR0VQAgRN3Cg4VxFWzd+9eP2UU\nsBQ+udgqZw6Ec59KMOSDDz5I1xpwrFV4//79WQSuY8eOfs+g/jdQUPdXUlISAzOTk5Op/DqdTtYJ\nGT9+PD8v85+Tk0PDa+nSpXT9jR07lgpIMOdcTSoQ/FEXVpH3Z555hsHUpmny7JOaIgkJCVXKTTiD\nO7VbRENDQ0NDQyOgqFHMhWiNubm5tMqqSj/dt28fOwZu3LiRwUD5+fm0/Nu3b08LqU2bNtTmQ9Wl\nTjTfRYsWsXy2pIABYEDq8OHDmaaVlJSEDRs2ALAa3kjedKjGLPO/adMmshJNmzblv5eXl7PKX05O\nDi6++GIAVsCbvC+Wxvbt24PKXMj8bty4keOOjo5mAFVZWRnlRdgM1UI4dOgQRo8eDQBM/wWs55UA\n4VA01rLZbKxnMGzYMDZbq7zm8ryvvvoqaWVBUlISG8qJhRgoeL1elvFeu3YtKw+6XC5ay1K98ttv\nvyVb2LZtWzZbmzp1ql+DJ0m9jY6OZrplRkYG106ePZQVRqVmyKJFi2iNyjgGDBgQtuDq6uDz+VgT\nJycnhzKemJiIe++9F4DFDKjVMENhIctejI+P96uALG4CtRttdHQ02RQ5D1NSUniurFu3jgz2iy++\niOeeew5AcNsNiIty48aNlNkOHTrw/lBZDJvNRvbzjTfe4B6NioriWGUOKo/3TIOuA4EaqVw4nU7m\n9cslBxw7OOvUqUMlIi0tjcKakJBAqnDBggWsf3DzzTfj1ltvBYCQ1qcHLMVmwoQJHJ9QgmqXS3nu\n3bt3Y/ny5QCsglpXXHEFgNC37V2zZg2zdbKysrjxn3zySSoRY8aMwaOPPgrA36cufxfsAlRyAZxz\nzjmMuejbty/LxdevX58Hl0pxyvgmTpzoRyWLbD3yyCN+MSbBht1uZ8zCLbfcUmVbadM0sXnzZgDA\no48+epyr8N13362yJsKZQNxiOTk5eOqppwBYSqeUn87Pz+eFLAplRUUFZfXgwYM8ePfu3Uu3iMPh\nwHfffQfA2qNS6Kxp06Ys3S5GQa1atUKiYJimyayF7Oxszq8864gRI8LebVYgcj916lT873//A+Dv\n4x88eDCuv/56AP7Fmk6EQNDzhmH4dXUVxSAlJYXnclxcnF+9jcq/GRcXx/o+DocD//rXvwBYnVVl\nv7711lt0Jas9R84EYsQ+9thjACxFWfZXnz59qPAAx/pdrV27lkXjKioq+PmHH36Y/XNOJL/aLaKh\noaGhoaHxp0GNYi5Ey+vatSvuu+8+AGDzo/j4eAaQde7cmZ3poqOj/WpHiJW1ZMkSaqMzZ87ELbfc\nEroHwbFnadKkCQPwbDYbLeSq2IhGjRoxI8br9R4XtR5siAbdunVrWprz5s1jR8gPPviAkeYTJ06k\npr5lyxa6Fi655BIAVnfOYEIstdGjRzNDYdiwYWS8VBpYDeKcO3cuAOCrr74ilRwTE0MW5q9//WvI\n6HiBjLlFixZ8LrXK36pVq8h0qTUohP4eNmxYwMcs1uCBAwdYr6CkpIQ0b+vWrWnZi0UZExODwYMH\nA7Ci/qUr5iuvvMLvTU1NxeWXXw7AovClzLnb7SaDE2p20TAMVsk1DIO/L5H+knFxslADGwO1LlLL\nRJi5qVOncs/FxcVxD9x9993clx6Px6/dQXVjBfwz7c5kzMJ4ud1uykVubi5l3Ofz0U2ckpLC80Qy\nSxISEvwYanFTeL1e/PLLLwCAO++8k/PQrl27gIxbXH/vvvsuAH9Zz8nJYWZWdnY2fv75ZwAWO6cG\n5suzjB8//g/ZZsMwePaUlZWRKQ31uVOjlAsRktjYWKZNSaZF5WIvVR08drudwl1eXs7LWS1OFGoK\nyuFwMNbiRH5Pp9PJy3nlypUUwFBTZ82bN+emWbFiBfbs2QPA8nVK22afz8fMkLvuuovFbiQlLtg+\napGHjIwMKjIq5aqme8mzLF68GLfddhsA/3LI9913H6O6w1HKXA4i0zR5YZSUlPAA/eKLLyjLNpuN\nayAKUTAoe7WYl8yvGv+klpaWeVT90T6fj5fc5MmT6bP+5Zdf/Fw4arpfuOhhr9dLd6RhGLzcpLts\ndeOqKg01kFAzKe68804AoEupvLycl9+1117LFHbV1WAYxh+mypqmyX3r8Xio6J3uJacaQ7GxsczS\natmyJd2kPp+PWVzr1q3jWGQc8fHxfL1z505+R2lpKWXy999/p5slMzMzIHOvFvmScQq6du3Kf58z\nZw5j/NTn9fl87IP15ptv0o0iz6eWWs/Pz6cyu2/fPu5nMSxDBe0W0dDQ0NDQ0AgoahRzoUK1hk4E\n0VyPHj2KOXPmALA0SbEIb7/99oBY0mK5ncrYTtUik+Y3VRXzCRWaNGnC396wYQML4Fx44YXMDLjx\nxhsZZAiAJbhD0UnUNE2/WgQSuFm5ZLVYGxKk9dJLLzGf3m63s5jTxIkTw9p8TSwy0zQ5vieeeAIz\nZ84E4E9Zjx49Gu+9957f3wUTTqfTL9JfcCKZttlslAm73Y7/+7//A2DJVqBLTp8pCgoK/AqZyfMK\nMwCcmKVQ5U1lP04XKqswadIk0vKCwYMH45577gFguYnVrJCqxurxePwKXAEWxS8dSA8cOMCsNXF3\nncpYAX9LPi8vj8G6jRo18tujwlxlZmYe11nX4XD4lbWXAHiXy4V69eoBsBglybBS2ZnT3Q+maTL7\nUM34kDEvXbqUzFZOTk6157K0dJg4cSJZR5Fvh8NB91psbCzvpvr16/P96667LqT7ocYqFycLwzB4\nMFxyySV+GQByKY4YMSIg/ixpWSwUb3WdV08XFRUV9Ofl5eUdV/QmVEhLS8PIkSMBALNmzeKm3rVr\nF9atWwfAqtApNN67777LzR4KGIbBDV5aWup3aMrF4PV68eCDDwIAe2C43W7OZefOnTFr1iwA4e3q\nWtk1IJTrjz/+SGrWZrMxxujdd98NefaQOtaThdfrxZIlSwBYF9vAgQNP+TtChY0bN/oVYpP0a3Hr\nqHEuajVPtQOpuo6BaMteVlbGzKzp06fzfUlPHz58ON0Lqg9fPefcbjfdbFu3bqUiIZ/9+eefeabZ\n7XYWcDtd5cLn85Hu93q9TEWPiYmp8uKPioo6zlXj8/moaCQlJfn1+RFXQ4MGDejCcTqdATnbRTGQ\nuQGO9TvZu3evn5tEhYyjTp067BKsnkmqcqEqn9LZODEx0S8TJZTQbhENDQ0NDQ2NgKLGMhd/FKSj\narfz5s1jHQk1Rz01NZVU4qlGe1cH+Z5NmzYBsGpYCLvgcDiq7FgInNhak8yMlStX8vWECRPCVrjH\nbrcz68YwDNYaycvLo6XTuHFjfqZ79+4htUhtNhstsm+//ZbBXRkZGbQkJ0+ezNob4s6y2Wzo06cP\nAKtvgVDX4bamZcz79u3D66+/DsBiydR6AAsXLgQQGKs4FCgqKmIhvMTERDIv4Z5rFbJHpScEYO1j\nKTMte9Hj8fgFhKtWuBocqXYEPtPn9Hg8XPPS0lKePeKqWb9+Peu72O12Bl07nU5+pqysDNnZ2QCs\njCOp8SLnSn5+Pl+npKQwG+9UA8jVbA3JCsnLyzslRkGdU6mrY7fb+Sy9e/fm+OrVqxfQvWuz2dgT\nSeqdHDlyhOeGOh9RUVHMnlq6dCnZW5vNxmDQoqIirp3sgebNm/P74uLiyH6npKQwaDXUe6NGKRcq\nvS0LIQtmGAZdHt9++y19ZD/99BPdIjabjf0gFi5cyAUM1KKJoEhchDrOpk2bciOom6pyXfrK/skj\nR45g3rx5ACzfr2SLNG3aNKyFe+Qwe+yxx0infvHFFzx4R48ejb59+wIIje9fhUpBv/fee1T27HY7\n16OoqOi4HhFt27ZltH1MTExEXHSqb3369OlM3/T5fExRGz9+PItrRcKYTwYFBQV037Ru3TqkhclO\nFcXFxX7GgJwz0iDO4XDwAlCVu2A2oIqPj2e11vfff5+xTvJf9fLLz89nNUnTNKkwJCYmks4vLCw8\nbj/Url2bF/mIESNI8Z8q5LlVY+jQoUNUANq0acN5s9vtVcaEqG4E+Z6OHTvyDG/SpAn3gMvlCniv\nKEmDFoPk9ddfp2KWmZmJq6++GoAVI6EqkSrkGV0uF+NXqkLl5w91CqpAu0U0NDQ0NDQ0Aooaw1yo\n2pzH46HWKFb9xo0bWURl165dpMUNw6CVPWjQILz66qsArI6WgbbyRKOWse3atYv0V0pKCrXf888/\nnwxKbGwsNd2cnBwGCYl15HA4qOEPHDiQkdThDDIEjmnlKSkpuPTSSwFYeefCzgwePDhsbhubzcY1\nd7lcpK/Ly8v95EjmdejQoQAsC/B0rbNgwTRNBj5+9913fJb4+HiO+4EHHghbEOfpwuFwkK3o379/\nRI8/KSmJ7JvD4WBQolixmZmZ1Wa4BItJcjgcLIyVmZnJ805qKfz+++9kUHft2sXzpKCggIGUSUlJ\nnHeXy8W9IVkcI0eOpLUdFxd3xmeOaZp+DJswyllZWdx3aqZEVd2qo6Ki+NkWLVr41URRgyMDPe+y\n/nKGP//88349RAL5e5HSVVczFxoaGhoaGhoBReSq+0GGVLWUtLBly5bRx1irVi0G0jRv3hwPPfQQ\nAKsOQygC3sSSX7JkCTp27AjASleShjb5+fno0qULAIuhkCZrhYWFtKYlFaldu3ZkA9S0qkjxrdts\nNq7FuHHj+DohISGsY5R57NixI8sJq+Np0KABHn74YQBWkzUAEcdaAFZshQTM5uTkoH79+gCsZknS\nnTUSx10dxDpOTExk/NAVV1wRMY2/VIi8XHPNNfjmm28AWPEXYn0PGzYMgBVgGI4Kv/KbcXFxZOpE\nPvr37+9Xb0Feq6XHTdP0q/4q8RfVlQQ/02dTa840bNjQL31XHZ8wnpXj0WQM8p5aY0X+NhDj/CNE\nopwGC7ZIaM16BjjtwYswCrW2efNmUn92u521F5o2bcqgpFAJhqyJ2+32CyCU19HR0QzSS0xM9HMf\nyHPJpjmVzoXhRjg7+FWHnJwc1rPYsWMHc/8ff/zxkMvF6cDj8dDdN2XKFBYNuvzyy9G2bVsA4XeR\nnQ48Hg/bs6t1CSJNfgArm0g6vK5evZoZReLarFWrVkSOW6PGICjCp90iGhoaGhoaGgFFjWUujvui\nauZBWxQaZzPUdObCwkLS37Vq1YpoxuVEME2TwalOp/OsfhYNjTAjKJecVi40NDTOOgSj7biGRg2F\ndotoaGhoaGhoRD40c6GhoaGhoVFzoZkLDQ0NDQ0NjciHVi40NDQ0NDQ0AgqtXGhoaGhoaGgEFFq5\n0NDQ0NDQ0AgotHKhoaGhoaGhEVBo5UJDQ0NDQ0MjoKixjcs0IhNutxsAsG7dOnzyyScAgEceeQRJ\nSUnhHJaGhsafEGopBl2NObDQzIWGhoaGhoZGQKGZi5OA1m5DA7fbjYULFwIAHnroITRo0ACAnnMN\nDY3AQXrtFBcXY//+/QCsM0Y6HqtdpjVOH5q50NDQ0NDQ0AgoNHPx/8M0Tfr79+/fj7lz5wIAduzY\ngQ0bNgAAsrKy+Jlhw4bh3//+NwAgNjY2DCP+80CYof379+Pll18GAFRUVGDAgAEAgJiYmLCNTSMy\nYZomfD4f/1/2pd1uh9Pp5GvNetVsyNlSWloKAFiwYAGWLFkCwGIwsrOzAVgdg0eNGgUAGDNmDFwu\nVxhG++dCjVcuRPhWrlyJ1157DQCQnZ1Niqxx48bo168fAGDDhg2YP38+AOD777/HOeecAwC46qqr\neKBFOgzDAAD8/PPPeP/99wEA9erVw0MPPQQAiI6ODvmYPB4PAOCjjz7Cjh07AADp6ekYPnw4AJxV\n7bTD6UKTy7a8vJzreCYXrMiK2+3m65iYmLBe2PKMFRUVyM/PB2BdHOqz16tXDwCQlJREitvhcFQ5\nblkvrYT8+WCaJs+TiRMnAgDWrFmDLl26AADOPfdcdO7cGQDgdDrRoUMHAJYshFIufD4ff8cwDO41\nm83Gs08dk2EY7AQsfxeJ8qvdIhoaGhoaGhoBRY3tiirPvWLFCgDAqFGjkJaWBgB4+OGHaTVHR0dT\nKzRNE8XFxQCA+++/Hzk5OQCAyZMnIzk5+XSHEhSoWm5FRQUAYN++fZg5cyYA4N1338WhQ4cAAOPG\njcOzzz4LAIiKCi2ZZZomdu3aBQC47LLL+P6jjz6KIUOGAAi/tVwdZI7Fai4sLMTmzZv5761atQIA\npKamcl6D8RwyDlnnLVu2UK579OiBdu3aAbCssxP9vs/nY5CbrMfevXvRuHFjAJbciLUn1lOo4PP5\nsH37dgDAzJkzsXz5cgBWYF7Dhg0BAAkJCWjUqBEAoFOnTsjIyAAANGrU6Lg1ME0zoi2/UEF1Mcl/\nDx8+TGbIZrPx/YYNGzItXGXFInH+8vPzyTrLWT1s2DC88MILACxZkecS9hSwzptAn4OyRyWY1Gaz\n8S6ZPn06GfGSkhLEx8cDADIyMujui4mJwbZt2wAAW7duRWJiIgBQ1nv37o2//vWvfK5TXI+gLF6N\ndYuID+7FF18EAMTFxeG5554DAAwcOLBK4bLZbFz4iy66CE8++SQAK1YgEpQL0zR5weTm5gKwhFme\nxW63Iz09HQDQsmVLCnz//v3D5nrweDyMXSkuLsbo0aMBWIdAJERtm6bJA0F1DxiGQeVszpw5ACxX\nmchVnTp1qKxedNFFGDRoEAArPieQl7IaKzR58mQAwA8//EC/cqNGjfDwww8DAHr16oXU1FQAlqIh\n62+32/m6oKAAt99+OwBg/fr1fFY5CD/44APuk1ApFzLn27Ztwz333AMAOHLkCFq0aAEAGDlyJBWo\n7OxsKnjLly/npREbG0vZV6nmYME0TT/lU71UKlPadrud76kKj3rpqxe5+v7p7hHTNFFeXg4AOHTo\nEOVl1qxZAIBFixbx39XxOxwOKnJXXnklrr/+egBA8+bNIyZOQfbDuHHjkJeXBwC4+OKLAQD/+9//\nOGc2m83vbJS9G2iDW/0+kT2fz8c13LhxI1atWgXAiusTF/v333/Pefd4PFwP9TtFJj777DMqKFOn\nTo2ItdBuEQ0NDQ0NDY2AokYyF6ZpkvLbu3cvAMuqu+iiiwCcnGsgKiqKWuWmTZv8goFCDbHstm7d\nyqBUocvGjh1Li021fmbPnk3avlOnTqEeMjXv5557Dl9//TUAK8DqwQcfBAA/zTvUQZKmaXJOs7Ky\n8PHHHwOwLGGhhJs1a8Yx7t69G4AVGFtWVgbAkg9x93z44YeUlUGDBpHSDITlX1ZWRlkWN8GcOXNo\nFblcLrJzo0aNoruvYcOGtODkWQGLlk1JSQHg7z6Q8cszhRJHjhwBYFVqFUyaNAnt27cHAL9gaq/X\ny4DOL774gixGZmamn/UtCIQ8iXyapkmm5MCBA7RGp02bxoyz8vJyskeyRwsLC/lefHw8x+fxeFBQ\nUAAAaNeuHRmDli1b0jofMWIEn/9knkXG6na7MX36dAAW47Vu3ToAxxhdn8/nt+9ERjwxBfeGAAAg\nAElEQVQeD4Mkn3/+ebz99tsAgDvuuINBk+EMbvd4PPjHP/4BAJg/fz66d+8OwGIsABxn0avsUbCC\nONXvU1/LOXDTTTdh69atACzmUNhnt9vNdVb3qArVJbplyxa/98KNGqlcVFRUUKkQavWBBx44pU1R\nr149HgKHDx8O/CBPEoZhMG32mmuuQZ06dQBYNBkA1K1bl5eYYRg4cOAAAGuTCaXZsGHDkCtFhYWF\nAIB33nmH8/j8888jLi4OgP8mVC83+WwwKHnVLypxB9dffz19nT6fD926dQMAjB8/noe9jKW4uJg0\n7I4dO7B06VIAVgzEpk2bAABNmjRBjx49AvYMUVFRnDNxzamZEV6vF0ePHgVgKUElJSX8W1Xe5fBK\nT09H7dq1jxufzPtf/vKXkMXliIL066+/ArAu1fHjxwOwLubKMiJjFgUvNzcXTZs2BeCfORIsWXe7\n3VQonn76aSxbtgyAdWGol4MoaPJZFaqLxGazcY1++eUXrnOvXr2YNun1ek/JNSLjyMnJoVwXFBRw\nTmRta9Wq5fe9QsmrrkHg2D5+6623cMkllwCwjAQZf6gg6//iiy/StRMXF4d33nkHAOjOrg6qiyQU\nUNc5IyODrQ5mzJiBb775BoB1buzbtw+AtRdEMVLdlKLMOhwOGseRkrmo3SIaGhoaGhoaAUWNYi5U\ny1Qs4a5duwKwApJORdPOy8ujhRTOYM78/HzccMMNACxK87rrrgMAUsaq9blz505Shu3atUOvXr0A\nhL6OhMfjofujrKwMt956KwCgbdu2VVqjJSUl2LhxIwCw/khKSkpALSM1AK+srAyvvvoqACuIUGjK\nxo0bk15V2R75u+TkZNStWxeAlSkic5+dnY2ffvoJgGWBiAstNjb2jK0lwzD8rHb5r1g58fHxDGZ7\n6KGHaMFFRUVVSdc6HA5S4wK73Y6EhAQA1n4JlWuqqKgIAOgaGDduXLUl4YXl+P333zFlyhQA1jNK\nxlFiYmLQxi3zf+TIEfz2228ALLeIuHOqo7RVqPOvsnPihktLSyPF3717dzRp0gTAqQd0iozUrl0b\nY8eOBQB07NiRTG7z5s0BWOyD/LbqRj58+DA+/fRTAFbwp2Rh5Ofn4+677wYAfPvttwAstihU7IW4\nBN5++21a7i+88ALn6UTjqMoFFCrYbDYWYhw5ciTXYP78+XyuxMREukiWL1/O5xEmsnbt2mT1Qp3F\nVR1qlHKhQg7ZCy+8EMDJb1IRwq+//pqv27ZtG4QR/jFkA7z22mukJkeMGIE777wTgL/CIIf0yJEj\nkZWVBcC6JEQpCtUBIPP11Vdf4eeffwYA9OnTh9kJDofDL8L+iy++AGC5rITal/iQTz75hL71QEBN\nt1u1ahWVGafTyd8ZM2YMlQe1qI16MMml7nA4qJSsWrWKa2Sz2XDjjTcCsGTuTDMXXC4XDxP5DVW5\nuPHGG3noxMfH+x086rhVRU79HnkWcbfFx8eHpMBQRUUF3X3iyvN4PH4pg2r1RXEDfv755/zMG2+8\nwRRa1VUUrBget9uNPXv2AABdDvLb6muZV4lt6datG42Bdu3aoX79+gCslEJVcRU5dDqdvIxOVTmV\n561VqxYvMdWwqpzJIhDlsmnTpiweeP/992PYsGEArOJUErchrohbb7016K4owDIWX3rpJQCWjIwY\nMQKAFWN0ootWztHCwkK6IJo3b35KcSxnCjX7p6ioiPEXderU4bxnZ2czDbuiooJF8mQNx44dy9eR\nkhYcGSqOhoaGhoaGxp8GNZK5MAyDmp8Er6l9CoDqNXihjNUcegm0CiXEuvziiy9YT+H555+nxq1G\nhYvbYffu3ezT8cwzz4S8YJYwKC+99BLnd8KECRxTaWkpI9gfffRR1v0XFxZwLLhv7NixpF8DUQ9D\ntWbr1q1Lyz81NZVWu9PppHVTUFBAqlio+qSkJLJB27dvJ/OSm5tLC0n97kBlKshcSvCiWkK4bdu2\ntHKrs+IMw6Dr4b333mMgqnyH0+nk/ti+fTvZm2DUIZHf/O2331jYTYIJP/vsM/5meno6x7FlyxaO\nr2nTpixl36pVqxO6/ALBwsh3ZGVlUX7dbjfHGhcXx8ysjIwMjBkzBgDQs2dPAFaBJFkbr9fLZ4mK\niiL7VatWLb9AvTOVHTWg8HT+FrDcejJWr9fr1yMICH7Wgnz/nj178MMPPwCw9qDKDFa1virjKEzu\ns88+y7PliSeeIKMd7FoogMVESE2Z6dOnY+fOnQCs+iNy34iLDbDYGQmevf/++wFYgeKREsgpqFHK\nhepTlwNL6HaXy8UYiqNHj1LpiI+P5yWsFk6KiYlh5oBcPqGCaZoskhUbG8uUNrWIljzfbbfdxuhp\nl8uFxx9/HIAVPxAq+kwujI8++ggAsHnzZs7v5s2bmS74wgsvkFY2DIOXcJ06dbjJJEp648aNvNwD\n4R6x2Wycj9TUVAwePBiAVUhKsoEWL16MadOmAbCi7eXzohzVrl0bHTt2BGAdBhJnUVFRwbSzgQMH\n8tIJlHInci3+fsMw6Iv98ssv0bdvXwDHaHj5G/HhTps2DY8++igAS8GTPSFKndfrpbxNnz6dsTpq\nj4NAQWRl4cKFVC7l+Q4dOsQx22w2Znq1bNmSmTvXX3892rRpA6D6WKJAy72Mb+PGjbwE1CyeLl26\nUMHv1auXX1YPYD2zWqhN5MLhcPhlT0UK3S3Pu2PHDrp/TNPkuPv37w8g+LFcMmdvvPEGz4cWLVow\n9b6iooJzJhevx+Oh22HDhg3MzFixYgUNthUrVrCyZyjiF8rKyniuLF68GAcPHgRg3UMi7263m8/Q\ntm1bTJgwAYAl++rzRRK0W0RDQ0NDQ0MjoKhRzIVYRTt37mRRm2bNmgGwtFV5Lz8/349qltLNjRo1\nImVVr149RuHHxcWFPKdbXAw+n48R25MmTfIr7AUAS5cu5bMMHDgQV199NYDQaroy1nnz5gGwWBWx\njmfMmEFKMC8vj9ZPmzZt8J///AeANe8SoS5UeUFBAZ87UIGdMk/p6em45pprAFgleKVeQVFREa0x\nh8NBZkXGnJiYyDHt27eP8hYVFcWAvR49ejBIKxAtwe12O6lpYVB8Ph8tno8++gg//vgjAKvQkYx/\n5cqV+PLLLwFYbNCJKGy1NLJ8dzBKDMuzDB48mOsq8tOqVSuyGTk5OWTs8vPzacG1b9/+pC1mtdT2\nmUC15GX8NpuNjGaHDh34OioqivMne0Clv2vVqkVZcTgc1bJbZzruyut9st9nmiZruYwfP55rY7PZ\n6Kbq3bs3gOAzFzKOzZs3kw0aMGAA98HBgwcZZCpsxZQpU+jOjo+Pp0tTRcOGDUOacREfH4//9//+\nHwDLbSZsyg8//EAGWs1my8/PJ1MXatf2qUAzFxoaGhoaGhoBReSqPQGG2tF01qxZtDDEb5ucnMw8\ncpvNRst/5cqV9IGNHDkSs2fPBmBpm+HqDgmAsQnJyclMm/zkk08Yc6EGqF566aUAgDfffJPBfaGC\nYRjMoZeAKZvNxhiEmJgYWjyNGzfGK6+8AsBKORXL2DRNxkAIm6FWngwU1CqF4useOnQofciGYZDx\nUVkHYSjKyspYGvmmm27i+9HR0YxTaNCgQcA7SYr1IhbvjBkz/BqsiayIdXQmv9GlSxdahsGIXZC9\n1Lp1a9YDqco6M02T679q1Squy6lacoEI6JRYrYMHD/rVq5A02FatWnF8+fn5jF/57rvvAFixMhJg\n3qhRI5bRVgMSg3HGqE3R1Nd/9NlffvkF//znPwFYQe0yPpfLhSuvvBLAsbLWwWRzDcMg05ybm8uq\noMImA8C6devw/PPPA4BfGqec8zfeeCMZo2nTpjHluU+fPiFhotV4EEk/vu666xgj9cwzz+Dzzz8H\nAL/aMxUVFfz/SInDqQo1Rrnwer1UDPbt28cLo0uXLgCswEih8UzTZEDPunXruIC//vorgyM7d+7s\nVzzrdGnG04WMv6SkhIdb48aNsXbtWgBWOV7AckFI99bY2NiQC6PX6+UhKnRwnTp1WMyrZ8+ePIxa\ntWpF5UcNYKtctlz+K3ndwYC68aWYkPq+Cln7mJgYHmK7du2iotewYUOWa05MTAz4RSFjko6h8+fP\nZ8ZH5Syo6v5eLr+EhARe0GpZewnAPe+884JWA0DNlHC5XCdUFNQW6hI8m5GRQWU1VBBXZE5Ojt8l\nLRdAeno6FYqlS5dyfJJVsXLlSspK3759GVgYExNTpfsyEPNuGAaVs+LiYhpQIptHjx6lSyEhIQEr\nVqwAYPWtkc+qynbnzp1ZRCuQWS3VwTRNnnXAMfd2Wloa5WLPnj080yWjrl+/fuwSnJyczIDUadOm\nUcZTU1PDdmk7nU4WCnzllVfoTnv11VcpI6oyGMnQbhENDQ0NDQ2NgKLGMBcVFRXUUtPS0jBw4EAA\nxyi8ysFH8n7Pnj2pJU6fPp3WXGlpaZUWaCg0XrvdzgqjV111lZ+FL3UYJNhJTdMLRyOh8vJyzpmk\niHXo0KHK7pxOp9NvjDLvhYWFDAYV+rh///4hsVBPZs7kMz6fjxVSS0pKKFPDhw9n2nLlstuBhLgr\nfvjhB5Ypf/nllxnAFhMTw7VwOp10CQ4dOhS33HIL3xc3irjTDh48yLlu0qRJ0FI5i4qKyAKIpVkd\n1BLNSUlJZMV27NhBa6+6cap1DgLBIsn8qumYTqeTqb9bt27lcx0+fJjjEwYgPz+fz7N69WqydGpz\ntkAFnwoMw2Da7Ny5c/H7778DACtuJicnk5nds2cPa8qoNVvsdjvrQXzwwQfHddNVq9hWfn2m8Pl8\nrOvSpUsXP3eUWPjR0dF0gUjX2bvuuov7pKKignNtGAYuuOACAAi567gy1Cqqsu9URjEjI4P7IxTV\nck8Xf3rlQi1rLHUR6tSpw41T3eEi77tcLl5yrVu39ivioyLUi6sWy1LLIEstCYkyHjhwYFgjim02\nGy9Z2eCmafr5EGX8avEntabIiy++SFpWDpSnnnoq5D1RqoOMf+vWrVi9ejXfl6yQxx57LKTZOU6n\nkyXVJ0yYwDlVMz28Xi9lPDo62m8uhR4Wd9D+/fsZrxSM55AxLVq0iBfDkCFDGPeiFrRTYxBE0WzZ\nsiXHrJaQr7wng+W6FENk0KBBjH8Cjs2f1+ulq6NZs2ZUKiRLyuPxcCwNGjRgeW2XyxXw+ByBw+Hg\nBZWUlOTXZ0R+TzJw5s6d6xfLJWNJT0/HpEmTAPi7Eqoqi6/+v2EYZ7x3bTYbY942b97M8R05coS/\nExUVRdep9JhxuVz89wMHDjBjKiEhgcp0JJ0r4rqqW7cu5enqq68OS+HGU4V2i2hoaGhoaGgEFH96\n5kLVpkUj3bFjB90H1blF1L8Xy6lfv34s2ZuSkkKLK9hlbqsbF2BZATKOFStWMKBQrGYJ8gsXoqOj\nGRwrc3748GFaDP369aMFWrduXVog69atw4cffgjAWi9hPaQJV7169SKGChRma+LEiX61LaSBUzhp\n1splntVqsyoLIJ8xTZNBs1IS3DAMlkl2u91cr0BADcybMWMGfychIYFBy9X9nqy/1+tlQzun0+lH\n21f1+UBD5uvaa68lI7d69WqeLcnJyejRowcAS/bFxacGzMp33Hvvvfy7YGah2Ww2Wr+jRo3ycysA\nFpsiHTlfeOEF1rNQXbKTJ09m11H5GxVqQy6bzUbWVz2LTxcOh4MZFhdeeCHn0uPxkF1LTU1l2XW1\nSaOwut9//z2DVlu2bMlsq0DVPjnd75E52717N+csIyODc33OOeeE5c45VfzplQtBWloa/YMff/wx\n3n//fQBWx03AEsQTbeaUlBTSbGlpaaSKwwE1pkFSPeUyA6yofuAYxR1qqGmdkoYpZb6nT5+OJUuW\nALB6o4gilJ6eTn/ooUOHOL+NGjXC5ZdfDsBKBwYio62wrIFcwsuWLeNzN2/enPX/I0UJAvzHUrkT\npkAOZ/Gh79+/n0rfoUOHePkFGg0bNuSF9u6771Ipk4wutQeHqlTv2bOH7ojOnTv7uYFCFQMFWArR\nTTfdBMAq46z2LZLLr6ioiB2BVeVOKP6LL744ZLKttnaXC03Kxu/fv58pp9u3b+eYGjVqhDfffBOA\nFfekKqUCieUoKSkhlZ+UlBRQ96yq5Jx77rkcd0VFBfdjq1atjiu17/V6aYAtX76c/37ttdcG1Aio\nqpdJ5fergsfj4fjLysqodCYkJPD9NWvWsBN3MArZBQrhP6E1NDQ0NDQ0/lSoMcyF0+lkGe86deqQ\nil21ahUAq9y0BDhVDqSSOhJTpkxhKeUOHTqElZpSo88lkruwsJCauER9h9vCt9lsZCZuuOEGAMD5\n55/PmiMrVqygVZydnc1yvD169CDtOWzYMAaXRaKmvnv3bgCWpSFW/5VXXhmRY1WD6uR1ZYpaZEYt\nhiRsgDBLgYLNZmNEf+3atdGuXTsAVmGpr7/+GsCxrIoePXrQRVJWVkaWY+bMmZz3rl27hk3m7XY7\n5+yCCy4g85afn889IO0DAPg15pNiT2KNBxvVWdBSj+O5557jGenz+eiWfOmll1jkSZUb1e2h1paQ\n9XI4HAFnkdRaNDLv5eXlXP/i4mIWQxR31f79+/Huu+8CAPbu3ctn6du3b9Ca8B08eJCl7OU31N/y\n+Xx0O82YMYNn4IIFC+i2iYqKYhJB7969g1bILpCoMcoFcIzu7d69OyOyJWVz6dKldC+4XC5uCp/P\nx3SlDz/8kJ9v27YtN1yoDzPTNKnwfPXVVyyYVVhYSB+jVMuLBOGTw0Yo0u7duzM1U80QAeDn+62K\nwo8kyOUshdUMwyD9OmTIkLArdpWhdt8sKSnh5aamAKsFtW6++WYAlgIoa6gWjgsU5LubNWvGg3Xw\n4MHs0yJZQytWrGBq3oEDB+gKOXr0KPdzqDsUV4Yq66IoJCQk8Aw5cuQIL2F5luuvv54KVrhkpnL3\n0H379tFVUqtWLaaOX3zxxX7Fy1TIfpDviI2N9dvbwYTMW0xMDOfX5XJxLKKIzp49m4pGly5dmIId\n6AKDpmkyQ+ixxx5j7Ju48nw+H8cxffp0jk8t2qhmcQ0YMIB9oVq3bh0xGS1/hMg6/TQ0NDQ0NDTO\netQo5kJgt9up3Qr1ftlll5Ga2rNnD5mBvXv3sqhQ+/btSRs+8MADDNgKtVXtdruxcuVKABZNKcFi\nXq8XmZmZAI4F40Ui1NLekWbdnw7EArHb7WSz2rZtG3Fsi9frZa2XnJwcWvnR0dF+WVVigQoNm5qa\nyv1w5MiRoAV0qrUrXC4Xf1/YuIqKCjIv7du3p6zb7XbuxbS0tIiQKYfD4edekvlt0aIFa7UIczRm\nzJiwutDUNRfLulmzZuyT07p1a9xxxx0Aji90p6JyxsnGjRvZuTYhISEkz2iz2cisJCQkMFNN9mjj\nxo05jkaNGpE9CobLRli+f/7zn3TFiJtm7dq1vEvUDBaRb8Caa5HrDh06sAZJrVq1Iu5sqQo1UrlQ\noWY1yIWsXsy9e/fG6NGjwzK26mCaJubOnQvgWKYCAHTr1g1PP/00gMhuxftng8QJuFwuVr0M1gV8\nqjBNk9R0YWEhPv74YwDATz/9xCye0aNH0yccFRXFlEJRRAYMGMBiPl9++SXTgYMpY6oCKgpH5ZRU\nGfMffYeK6lJUgwVVgZaxd+7cGcuWLQMAVoRs3LhxWC8Lm812XOOyBg0aoE+fPgCAzMxMVvk90feo\naN++PeOpgPDMvxiR8l+JsQjV7wOWq07cL1988QUAa/+JQqG6jpxOJ+encePG/Lu//e1vzGaJBOX5\nZHB2jFJDQ0NDQ0PjrIHtbCjG8Qc4qwcfCJxKDrVGcCCBwIMHD2bL55deeiliLAyRkYqKCrz++usA\nrEh6yZ5KTEwkVZySkkI5kuBl0zRZ9OyWW245rix3JCMSei+Ypkm6e/fu3cwYEZq7bt26YQnQk7nx\n+XxkLoStmjlzJlve161bl/V9oqKi/MqxVwVhvhwOB59bdVfUtHOqrKyMwZ0yN6+//jrrbaSmpiI7\nOxuA5aYfN24cAIv5EcZFZfKCgKB8ceSfDhoaGhoaGhpnFTRzoaFxhhCrb+fOnQxgC3Q9iHBDM2Rn\nBvGr5+bmsm6H1G6Ji4sLa2qhurbVNR1TU5X1+p851CDa6uY0VBVmESTmQisXGhoaGiGC2plWpbw1\nNMII7RbR0NDQ0NDQiHxo5kJDQ0NDQ6PmQjMXGhoaGhoaGpEPrVxoaGhoaGhoBBRaudDQ0NDQ0NAI\nKHSNaA0NDY0wQs0gqaioiJjS8RoaZwLNXGhoaGhoaGgEFJq50NDQOCsQwqJCIYXX60VWVhYAq8lV\nqBt8aWgEA1q50NDQiGioVSPVi/dsVzSk78aSJUv4Wm2zrZULjbMZWno1NDQ0NDQ0AgrNXJzlcLvd\n2LhxIwCgqKgIaWlpAICGDRsCsHpcSN+CyjXsz3bLL9IRCR05z2ZIzxbpHjl37ly+161bN3Tv3h2A\nJeNn0xyLXBw4cAAAMGPGDGRkZACwnku6h0YqZPxer5dMkmEYcDqdAPwZF3UPaCYmcKiu90skIbKl\nWKNKmKaJ0tJSAMDjjz+OJUuWALCaIknDrIsuuggAMHr0aCocPp+PGzw2NpYCmZKSwj4HoaKbVapb\nIuUdDgcVoeoOIvk7wzD42uFwhHxzqY2HysvLAQDZ2dn4+OOPAQDr169n1P/NN9/M9toyz5EGeRZZ\nCwC85MLhgjBNk3EIzz77LABg+fLllI+SkhJ069aNn1XHd7JVh8N1IMv4Pv/8cwDAb7/9xvbmiYmJ\nEXlRqM35rr32WgBAVlYWGjduDADo3LkzBgwYAAC44IILAFgN2UT5iImJ0cr2SUI942Q/Llq0CP/9\n738BAKtXr+b70vzu1VdfpbIdKcqpViU1NDQ0NDQ0AorIUHE0TgkejwfLly8HAMyaNQs5OTkALOtC\nqMl9+/YBAH755Re0aNECAFBYWIhffvkFgGWBiFVx2WWXYcSIEQCsaHWXyxWSZwAsa3/Xrl0AgObN\nm6N+/foAjmcuVCoWAEpLS2lNxcbGcsymaQaEfq0qiNDj8SA3NxcAsG7dOsybNw8AMGfOHADAwYMH\nUVFRcdw4Vq9eTaujd+/eQW+vfTKWu/oZwzDIhC1atIjP27lzZwBAvXr1Qt7B0+v14tNPPwUAfPnl\nlwCA4uJiREdHAwDWrl2LoqIiABYbpFrFIiNer5cWXnR0NOdddROGAyIjP/30E8cmVn8kug4Mw8A7\n77wDAPjnP/+JwsJCANa+S0pKAmCxFMLUyZwnJibyPAIim7FQ90M4x2kYBoqLiwEAkydPxssvvwzA\nOlvkvAOOjXHnzp0AgMWLF6Nr164hHu0fo8YoF4ZhVBlpfqqCJBsnNzcXixcvBgAsXboUcXFxAIB7\n770XAJCcnBzwg0LGn5eXh7fffhsAUFBQQBosPT0d/fr1AwA/ilJcJYWFhTwYDh06hDVr1gAA3nrr\nLRw6dAgAMGHCBCQnJwNA0JQMwzAYJ/Lyyy+jadOmAIArr7ySykVlyOY/fPgwAEt5koOrbdu2/Nzp\nHgyqEgEcW+fdu3dj0qRJACwl4eDBgwCAo0ePoqysDMAxylh1lchzApaLRNare/fuAVcu5Dfl0ios\nLERJSQkAS9ZjY2MBWNS0yGRhYSEV0DVr1qCgoICvZV5FEb366qvRpk0bANZFHszDV57l4MGDeP/9\n9wGAz1L5ApDn9Xg83ANqWufcuXOxYsUKANZFOGzYMABAnz59AFiXejgo5NWrVwOwZAsAzj33XNSp\nUwdAZF7AS5YswV133QXAkrH4+HgAwKBBgzB+/HgA1lkhci1zWlFRwTMkEDJf2YUqc2W326tVDqqa\nT9XtcPToUQCW3FS1T0IFGVNBQQHvkM8//7xa2ZfxJSQkAAD69u0bcbITeWqyhoaGhoaGxlmNPz1z\nIRpfWVkZLTGv18vobLGUVYuscnCYUPhTpkzBY489BgDIz88n/QocozOFNn/uueeQkpJy3PedCcRS\nnjZtGhYuXMhn6dWrFwDgzjvvRJcuXQCATIpKSyYmJuKGG24AYDEajz76KABg27ZtDAodM2YMmYtg\nobi4GE8//TQAa74kAKxJkyZVWgymadJKfeONNwBYbpErr7wSwPFZMKcD0zS5nm63G5s2bQIAPPnk\nk2SoiouL/ej3yvLidDrJVni9Xn7W4/Hgm2++AWAxBmLNBUou5Hckq+KZZ54hQ5WUlETrpkePHpzH\nPXv2YO3atQCAtLQ0yolpmtixYwcAkM1o0KABLevU1FQ/mQo0ZK999tln2LZtG4BjrjC73c7fTkxM\n5PiSk5P5XLt27cLSpUsBAJ9++innBABWrVoFwKL2AeDCCy+kFR4qeL1evPTSSwCOWfM33ngj3T2R\nhLy8PADApZdeSrdZamoqPvroIwBAv379yFK43W4cOXIEwDEmL1AMnXxfWVkZfv/9dwDWPDZo0ACA\nJb8SVB0dHU0ZUeVU9ojP56OsHDx4kHdCXFwc+vfvD8BiYULNXAhD8X//93/4+uuvARwLFAf8z5uo\nqCikpqYCAP7+978DAFq1ahV0d+upQjMXGhoaGhoaGgHFn565EG3P4XDQOs/JyWEwmKSzZWZm0opZ\ns2YNNfVJkyZh//79ACytV7VWVStW/laCatRUz0DAMAz8+uuvAIBXXnmF42vZsiXuvPNOAMB5551H\nC6g6zVu023bt2uGKK64AALz55puMW3A6nfzbQJdbFqt+5syZ+PnnnwFYMQgSJxIfH1/tuGUNJICp\nSZMmTN8LRHqnGofjdrvJDIl1DFhzI/ErqampaNSokd93JCQk0N+/fft2WnJq6vCvv/6K4cOHn/F4\nVci4JS3QMAxs2bKF/96+fXsAwObNmzm+/fv3kxGoV68eevToAcCSjx9++AHAsb3RsWNH+qYlrTkY\nME0TixYtAgA8//zz9K+rFluTJk0AAH/5y1+YhudyuShbrVu35hotXboUe/fuBTRje2wAABflSURB\nVGBZvVL7RQKcQxG4XBlHjhxhvJHsv44dO5IBiJQS5z6fj0HeR48e5fgWL17Ms6JybIOcgep6CXtw\nJs+ksg7CDKenp5MZjo6O5rmnnstVfYdhGFz39PR0niHR0dFk+ELNAPh8Pnz11VcAgNmzZ5NZAY6d\n46mpqYx76tu3Ly6++GIAYPppoO+bQOBPr1wIoqKikJmZCQD47rvveEnJBfDBBx8wa8HtdvtFnItg\n2u12RkcnJCQgPz+f33/++ecDADdkoOsZlJaWYvbs2QAsCk1cF1dddRV69+4NwNogJ6LzVGVLDtmG\nDRsymCktLY3fEWjaXhSEJ598kvMzZswYXorVwefz0a0gcz527NiAuxfke1wuFxWegwcPcm6Sk5MZ\nKNulSxf+vsjS/v37qTTl5OTwQvb5fKR2N23ahKFDhwII/CEm42zbti1dCrVr12bNk0aNGjEgNSYm\nhgGdrVu35kF9+PBhdOrUCcAxpSQxMZEXRzDp4uLiYjz11FMAQLeO+pu1a9fGHXfcAQA455xzqEQ4\nHA7KWK1atah03H777XSz7Nixg65QkftQXiKqYi0X2siRIzlm+feq3G3hwMGDB+k2s9vtuOmmmwBY\nslXVuMrKyihbosQ5nc6APIOsYVlZGZX97t27Mzvlj+qwVFY01MwiwzCorLpcrpBnRKlBnOKqEyME\nsORTZPmGG27A1VdfDcBSNGRPiPIWjlo/J4J2i2hoaGhoaGgEFDWGuXA4HKSju3fvTmvzt99+A2Dl\nnEt+cXx8PCkyl8vF4MjbbruN9NukSZMwf/58AFZtiIEDBwJAtXUaThei3WZlZWHBggUArBQvsSpH\njRpFC+5Uf1Os0f79+5ORCTQta5omA5Pee+89ANZ8CdMzZMgQUq6VXU3yurS0lIFXMrY2bdoE3PKU\n73O5XLTe4+PjGQiYlJSE9PR0ABYbsW7dOgBgAKHb7aYF6nQ6+VyGYVBufD6fXw2NQM61fFf//v3J\nwo0aNYrMhWmadAdmZmYiOzsbgLUeYsEZhkHrUCy5Jk2aHEd5BxIyH7/99hvTo9U6IfLbo0aNIqNU\nOdVbHZesY9u2bSln8fHx6Nmzp9/vhtLSk3n3+Xy4/PLLAViuScDat7JHoqKiwhqYJ2tx7733kr1N\nSkrC/fffX+XnxXW1cOFCnpPNmzcHcObzq6YlA9YcyhmekJBwUvNUeQyV69YIc12nTh0/d3B1fx9I\nyH0zduxYsojqmZCWlkb3x/Dhw8kuqokEkVrxF6hByoXNZuMB37RpU1J34mseM2YMF0otQ61GHJeX\nlzNSetmyZRTC4cOHY9y4cQACX3pV6PSlS5diz549/I1zzz0XAEgNnizUMs9yGPTu3ZsCrQpuIODz\n+ZjPL1T3yJEjSbNGR0fzt9WS3moxpH379nE9xE9du3btgG989bKS182aNWMNkGnTpmH9+vUArEh6\nifAWuWnfvj3ntKKiwq/Uuigr/fv391MuBGfyLJX/NiYmhnRqr169/NxH8tkePXr4ZbkIDMNgXIUo\nGVFRUQF3lamQC+q///0vXxuGcdweLC8vp2stKSmJY7Lb7X7jk8ujqKiISlPHjh2pQIfa7eDxeJhx\nlJaWhgsvvBDAMblRFc5QjqsqyGUrBeIAKxNBvYRlXbxeL+uITJ06FRMnTgQQ+AtPMiOWL1/O9TxV\nBUytBSNxG2VlZX4xIaFsde/1eln7ZsGCBX5uMZm/OnXqMHPrscce495IT0+nkiXKdlJSknaLaGho\naGhoaPy5UWOYCxU2m40aqzANcXFx1Vo0wh6sX78eDz/8MACL0qpbty4AKzdZgukCDaltsWTJEgY2\nRUVFkSrcvn07NXuXy/WH2qtKCRqGwaDQlJQUUnSB0tplzkpLS2nhjx07FoAVWCjzpdaX8Hg8/Kwa\nSFtSUsJgpr59+/JZq3KhBEJ7Vyv+RUVF0cXw008/kb6UtQCOWWqHDh3iWqjylJmZSbnp1KlTlfSr\nIBAMRklJCWVTzYgwTdOvBkBV8q5G06sVEINlzZmmyXoKBw4cqLLCqcjmjBkzsHXrVgAWbS/sTFpa\nGuUpKiqKa2MYBjN66tSpQ1Yp1AF769at417r2bMnXa4i9zk5OZT7Vq1ahWRslSFz/d1333Fswoo2\naNAAjz/+OABLnqQsfNOmTVmTISEhgYGygQ6wlvnKyMjwyxwRVNWFVV6LLIj7e968eWQ/+vXrxwDK\nkpISP/eyfKdhGEGR/YqKCkyZMgWA/1kCHGPqioqKmD3ldrv9GjXKOkndogcffDBod9DpokYqFypO\nRJGapskMhXHjxpEqjo2NxS233ALA2mTBOrBEsNu1a+enEEkxmaeeeorU2G233faHB6i62crKyvwy\nAERwjx49etpFtNSCZeJDnjp1KudPXBpOp5PjKCgoILW6b98+ptvm5uaiXr16AKzDRQrmiHuhcpqb\nGusQiMNAdRlUlV5cVeGu3NxcXsx169blxXbrrbeyuFlUVBT/Ts0ikTGfiVtNvuvHH3+k+yYjI4Pp\namqJ5uoKj9lsNn5m8+bNAKzLT8Yf6PLfqo85PT29yv0ol/DRo0cpK+PHj6cC1bRpU1x11VV8LcrK\nwoULeeBeeumlIS9UJRfXp59+ivvuuw//X3vnH1Jn+f7x63g8uuNx5q+ps7Z06VzOmTA3VkS5qOZY\nAyGCghj7Y9BWQRGyAiEiFgRBjSiKkfRHEGtra2tE2Q8JGmthtIXosEXuj/n7R6nnpEc9Pp8/Hq63\n93H6Tbf7HM++e7/+srX53M/z3Pf9XPf1432JuLkiej+nT58WEZGOjg4YzVpuGG90TLr+CgoKsP78\nfj96GYXDYeQ/FRQUoNT32WefjZkgmc7HjIwMGPerV6/G+o9EIlj/kUgEYcz29nZUmWl7+9LSUnn0\n0UdFxN2nNNQ8MzODNgR33nlnVIg8FoRCIbQyMA2i5ORkCNaNj4/jvsx1Mj09jX5SH3/8sYi4e6uu\n0UQJjzAsQgghhBCr3PKei/9icnISNcidnZ041dfV1aGZTywTgPR3V1ZWwjItLCyE1dva2gqX3/nz\n5yGPbTZCMjuKqkfBPIF6vV64BMPh8HXL92r1QSgUgl7Bl19+iXtoamoSEfckpJnPfX198GxEIpFr\nmoGJuIlcOj49idx///2o0LnnnnvgsbGdUJuUlITT5MGDByHE1tnZCaEqrQSYmZlBwmFKSgr0O8zk\nQ5O5jc5uFPVWHD58GNfzer3y/PPPi4jrkv+vWn6Px4NnqO/g559/RggiOzvb+mlOvVimnobP57tG\n5Mrv92NsqampGN/AwABEz4aHh7Eeent74bno7++HXLwmQ8dSRMtxHGlraxMRkY0bN8K17zgOqo9O\nnjwpIq4QnoYaluvUaXYpFnE9XqoLsmnTJryXnp4eCLFduXIF3pn77rsvZvugKRKn1/vnn3+iJN/V\n85KWlgaNl/7+fjz3p59+WkREHnnkEXhBw+EwwlFmKNFsXGb7fZgaG6aXTtdUeXm5PPnkkyLihrzV\nM5SUlITnfvnyZdy7fgcaGxvl3XffFZH4i4AtBI2LBdBJ0NnZCfW0SCSCEtC33347LmVAOslXr16N\nEjatdBER+eSTT1AKefbsWWyc2q+jtrY2qhpA46IFBQUwQCKRyLyCT0tFjYSuri5soMFgEAtfP8J+\nvx/GQlZWFsI6O3bswP0WFBTgHTQ2NqI9tW5+XV1dcA2mp6fD8LKFWdGh7vTdu3fLjh07RMTdmPT6\nn332mYi46nr6Z0NDQxAhamtrg6Hh9/uj8l5sbl5qOEYiEWzCly9fjjIoFvMB0M1Jy+BycnKwYVdU\nVGDDtjV2fb6PP/44NlDHcSA2pvM0Pz8fRqkZFpuYmEBW/YkTJzAPg8Eg5tyJEycw53SNxFINc2pq\nCh+Ahx9+GNfp6+uTI0eOiMhsNc66deuue83ZwHEcVHKtWrVKRNxn/sQTT4iIa2ioOGAoFJJPP/1U\nRNweNlrBEI9ckdTUVBiEmZmZuHZxcXFUiFSrcURmjSYz7KjvIiUlBeXAjuPMW1UVK9LS0lDWPjw8\nHHVo1fCx2ccnHA7DWK2vr8fPurf+9NNPmOux7g21WBgWIYQQQohV6LlYAE0cqq2tRY13RkaGHDp0\nSETc03I8XJime66qqkpEXO0FPe1t2LABJ+f3338fyUzHjx8XETeMoCGDtWvXylNPPSUirqtWXcY+\nnw/Je8FgEBLXSz3ZaQLj2NgYJMmzsrJwKtq0aRPGoUlVVVVVGMdcUST1HmzcuBGnPb2v3NxcJJyV\nlZXBXW7DNWtW1ZjJYitWrMBYPR4Prl9aWioiIrt27ZJXX31VRET++OMPnKCPHTsG7Yjq6mqMNRwO\n4/fZcGXq/LjrrrvgaUhLS4Pc9GLDAObJTsR1R6uAm8fjgTaMjRCUx+NBVcKWLVuQJJ2Xlwcv4UI9\nH8x3rX/X5/NBiGtwcDBK80K9SjYri+ai1xsaGsL7r6iowBqor69HQqQKfK1duxbu7YyMjGXpbaFa\nNOpt2bNnD0I1poc2PT0dCc4TExPYh+JxWp7by2QxCbpz9wPzdyQlJeF3mFUhsQxzm6Fo3Q+7urpQ\nwbJz507so2bo2u/3I5l9+/btqGDTSqpQKARvZaJ4LmhczMPg4CDamPf398OdevDgQYQb4tWSV916\nIyMjWOTmR664uBiNyxzHQdxNN9DU1NSoEis1mrKyshBHDYfDcEebH6Clbr7q2q2oqJBXXnlFRNwP\nkIoXmWWQi3l+ev1AICD79u0TEYlSztTs7szMTKvCSI7jRJXS6gcjLS0NG785fn0X27ZtgwppQ0MD\nhIguXrwo7733noiI7N+/H7Hs5ORkvBsb6DjeeustfKSzsrKu+5noPa5cuRIGbDAYRN7LUgXcFnMd\ns4GaGhUL5Ynof5sltmvWrIlqH684jgPXfSxbxqub+vPPP0f+TXl5OaoVRkdHMW59fqOjozBUYzm2\nhTCrD3R+l5WV4bmbc93r9cr3338vIu561w/Zcox7Mfxfc38+Vdd44fP5pLa2VkTccLaqmhYUFMxb\n0WUaQnl5edcY9uvXr8chMlFgWIQQQgghVqHnwkATrDZv3gxLPiUlBdUiL730Utw8Fop5UtaTkCka\nZFq0+/btk507d4rIrBfh77//RkZ0OByGq7avrw8CLd3d3dBCePDBB6/7FKInTZHZpFNb3fp0TJrc\nNzU1heQ+2wlYZlVNc3MzrlNWVobrzzcPkpKS8HdLSkpwwuvu7ob2gs/nQ6JkXV2d1ROTPoPt27fL\nm2++KSJuQrK6S01hr6XQ09MD/YNAIGD9lKpzdXJyEuEDn8+H+WRWvqgXSWT2fqenp5FA/MYbb2Cd\nmCQnJ0NLwnZFkYl6LlQXQsTVjtD5NDY2Bk+eur83b94c1eEz3oyPj6MqQSuw/H7/vHNlZmYGLRAi\nkQhCJ4lSobAQC4XCTO9XvFHPcXFxMSpYRkdHkehpdnt1HAffpytXriABX9dDRkZGwuhbKPRcEEII\nIcQqt7znQk8a7e3tUldXJyLuSVM9A4cOHYJOwHKcKjTuuX79epwYW1paUE6XmZmJk18gEEA8XzGT\nEwcGBqA18d1336F8LzU1FUl/lZWV132yM/+d7aQ5vUc92XZ0dCCJzzZerxcnzcHBQeQYpKenI/ej\nqKgI+Sk6tmAwKL/99puIuCqZmghsliV2dHSg9DYtLS0m9fRerxfx3HPnzqFRU3Z29pLerb7Dd955\nB94vc8w2MBs1+Xw+xI2bmppwgtO52dbWhhP22NgYvEGXLl1C2W8wGIyae3q/lZWVyNeJ5Qlvbp6H\niJvcqXNh1apVSJjWvJicnJxl2VuUFStW4Lmr8u+9994blbys99LR0YFycK/XK/X19SKyPHvjYliK\nRyKWib5zMSXGe3p6sEb//PNP5NBt3boVHqGBgQFoBx0/fhxluKY+BuW/EwQ1KjQB7+WXX8aHKzc3\nFyGD0tLSZXU3mUmNOhmbm5tR51xVVYUkzdzcXNSpqyEyMzMDYaqLFy/iQ3n16lU8g+LiYrjqzevY\nGLcNHMeRv/76S0RmNSUqKipi5sr0er1IVKuurkalREtLC+ZFYWEh3PbqrhwZGYGc8PDwcFSHWX0f\n69atQ8WO2RHWJh6PBxtNSUkJDMqMjAyEqxbzjvVD3tTUhE3sxRdftB4W0d+dnp4OQ83j8cg333wj\nIgKdmV9++QXPemJiAs9ucnIy6lmb3WhVAO2jjz7CO43letYE3cOHD8ulS5euGevdd9+NZHGtIFru\nD3Nqaiqquz744AMRcWWln3vuORFxw53qhq+rq8O6y8vLu6aNfSIwX9fh/3rG5pyIpf6JOR41mn0+\nH/aNvr4+HAZramqwjr/44gtUGelBRcStFhRxWz/EMtx3PSSmuUkIIYSQmxbPciSyWOS6Bj81NSVf\nf/21iMy6JoeGhuAabGlpsd7h70ZxHAceiB9++EFOnTolIm5yjyawTU5O4uSnyUKBQAAubY/Hg9Pb\n6OgorN6Ghgb87PP5Ei45a2pqSr766isRcZUWRVzJbw1XxaIhlSmZrjLfDQ0N8uuvv4qI+/z0tKzP\n3JTzNhMPzRr1kydPouwwHnMrEolgzEeOHEHC3q5duxZU2tRT6oEDB0TEnW865rNnz8asQZXI7HML\nh8PQXlDPUXNzMyTOe3t759UiSU5Ohprk7t278fPKlSvjupbNuTDXq2KWGiYKWvKuiahnzpzBflNd\nXY0S99bWVngBzpw5gxBPotyLWUYuIkt61jqH4qHQae7ne/fuhUd5amoqSuJe34uGQUSiWxLov1uz\nZs2NeMBicrO3lHGh9/r777/LY489JiKCWJfX64XB8cADDyTMYjHR8YfDYWTvHz16VM6fPy8ibn6A\nTkbNBzA/clu3boVwT35+PrLn8/Pz/7Nb5nISCoWQoa5jKyoqgsxvrN2B+gxDoRBEvBobG6W9vV1E\nZrtfms86KSkJFQDbtm1Dz5fCwsK4P1+dExcuXIChUVNTI0VFRSLiPj8d+7lz52BUqLDThg0b0Fpb\nq2HiwVxDLRwOwyU8MDAg3d3dIuKGHTRHprCwEIZcSkrKsoccbjbUEOrp6cGH69tvv4VL/t9//4VB\ncfTo0Zj2Z1kKpiGnxsVSD0rxzLkQmT2U9PX1IVTa0tICQ8L8Nnu9XjzrmpoaiDmqENcNzvOY3DBX\nHiGEEEKsckt6Lvbu3SvHjh2L+n9btmxBZ8VECwvMh95LJBJB2GNkZAQnDz3Nm416cnJyou5trtx2\nojI5OYmTkzYAy8vLi8pmj8f4zcqb8fFxJOxpp9T29nacoEtKSuCSLy8vj0noZqmY45+amoJn4tSp\nU2h+9+OPP+LvaBfR119/Pa4eC7L8mOGF4eFhJDJPT0+jEilRZKYdx8G+19vbi7mampqKfXAx+4N6\nxeLtjYlEIliLr732GpR9RWb36DvuuENeeOEFERF56KGHEMK35JljWGQeljR4nTy33347ss51Il64\ncAEfLpJ46Oahi8lxnIQzAueupUQ22BzHQbjkww8/RC+SQCAg2dnZIiLyzDPPiEhiCvSQWxszp0Vk\nNsTQ2tqKfLlAILAk42LuHjP35//HMCxCCCGEkMQnsQpjY4h50j1w4ADquffs2SMigmx4kpgkWg33\nfNxMp3uPxwO9iv3790c1SNImdolY1UBubcyKDsXj8SA0nJ6ejgRrr9cLgbbFhDpMTyjn/I2T+Du2\nJTweD9rTXr16VU6fPi0iAqXLm+HjRYhN5rZWV1Roi5BEY6E8Mc3/uO22267bMKBBYReGRQghhBBi\nlVsqoZMQQgghUTChkxBCCCGJD40LQgghhFjlZs9iZAYOIYQQkmDQc0EIIYQQq9C4IIQQQohVaFwQ\nQgghxCo0LgghhBBiFRoXhBBCCLEKjQtCCCGEWIXGBSGEEEKsQuOCEEIIIVahcUEIIYQQq9C4IIQQ\nQohVaFwQQgghxCo0LgghhBBiFRoXhBBCCLEKjQtCCCGEWIXGBSGEEEKsQuOCEEIIIVahcUEIIYQQ\nq9C4IIQQQohVaFwQQgghxCo0LgghhBBiFRoXhBBCCLEKjQtCCCGEWIXGBSGEEEKsQuOCEEIIIVah\ncUEIIYQQq9C4IIQQQohVaFwQQgghxCo0LgghhBBiFRoXhBBCCLEKjQtCCCGEWIXGBSGEEEKsQuOC\nEEIIIVahcUEIIYQQq/wPqxSZ+465rkYAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n", + "save_fig(\"generated_digits_plot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "n_rows = 6\n", + "n_cols = 10\n", + "n_digits = n_rows * n_cols\n", + "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " outputs_val = outputs.eval(feed_dict={codings: codings_rnd})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpolate digits" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAACsCAYAAADBhXmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAGe1JREFUeJzt3XmsnUX9P/BBCwValkKBltLKIkKLBaQUKVtdiyzBCBij\nsaJCYkgICUZIjAkhIhADEYyiJXGFCE00bBESTFDABSgipWxtpa0FWrrQhZaytcjvzx/zmfGeQ7/3\ntPfeeb3+ex9mzj29z5zneYb7fGZ2ePfddxMAAABt+cD2/gAAAABseyaDAAAADTIZBAAAaJDJIAAA\nQINMBgEAABpkMggAANAgk0EAAIAGmQwCAAA0yGQQAACgQSaDAAAADTIZBAAAaJDJIAAAQINMBgEA\nABpkMggAANAgk0EAAIAGmQwCAAA0aNj2/gD/w7vb+wOwTezQ4/c3jtrRy7FkHLXDOKI/GEf0B+OI\n/tBxHPnLIAAAQINMBgEAABpkMggAANAgk0EAAIAGmQwCAAA0yGQQAACgQSaDAAAADTIZBAAAaJDJ\nIAAAQINMBgEAABpkMggAANCgYdv7AwAAbXr33XeL13bYYYc+28T/DjCQdXOe2578ZRAAAKBBJoMA\nAAANMhkEAABokJpBAKBPtZqXd955J8uvvfZalufPn1/0ufXWW7O8ePHios2yZcuyvNtuu2X54osv\nLvqcddZZWd5pp52KNmx/tXG0cePGLN9+++1ZfuaZZ4o+48ePz/KZZ57Zsc2wYfkt70Cq2WJoqY3z\n99tmW45PfxkEAABokMkgAABAg0wGAQAAGmQyCAAA0CALyAwinYpNYzF/7bUdd9wxyx/4gP8fMJj8\n97//7TPXjmcsQlY0P7TE88KWLVuyXDsvxDEQzwvdjJHa+cj5ZOiIx/ett94q2jzxxBNZvuOOO7L8\n1FNPFX3WrVuX5Q0bNhRtli5dmuW33347yzvvvHPR53Of+1yWLSAzMMRx9OqrrxZtrrnmmizffffd\nWd68eXPRZ9ddd83y/fffX7SZMWNGlmfOnJnluDBRSq6PdBbHdO1aGK/D8V4tpZQ++MEP9vlzav89\nvm+8dqe0dWPYlRsAAKBBJoMAAAANMhkEAABokJrB96g999vNs8Gdns/dmj61+oznnnsuy3Fj1vvu\nu6/oM3LkyCz/4Ac/yPLxxx9f9On0HDO9EcfJm2++WbRZtGhRlh966KEsx817U0rp1FNPzfLBBx9c\ntIn1F7H2qzZe1Vb0XhwTtfq/FStWZHnu3LlZjhuBp1SeF4455pgs12qy4s+u1dsMHz48y8bI4BVr\nUx555JGizaxZs7K8fv36LF966aVFnylTpmS5VtsXNxm/6qqrslwbn8bawBTH0W9+85uiTbx32XPP\nPbN88sknF33iNevRRx8t2vzyl7/M8qpVq7J82WWXdXxf42rw6HS/3s29eK1NrPeL9+dxXKWU0pw5\nc7J8yCGHFG3GjRuX5XhN3bRpU9Fnjz32yHJ/1Ub7yyAAAECDTAYBAAAaZDIIAADQIJNBAACABllA\n5j1qhaOxIP7pp58u2sQCzriIQu19Y5+4WEjt5/zoRz/K8n/+858sv/HGG0WfWGgfFyCZOnVq0aeb\nxUPof3Fj5auvvrpo86tf/SrLtQ18o5///OdZ/uxnP1u0+fznP5/lCRMmZHns2LFFn1jIXFt4yFj6\nv4nnjlpB+d///vcsL1y4MMu1hV4OPPDALK9evTrLtY3A58+fn+UzzjijaFMbJwwOcazFMXHjjTcW\nfeICVpdffnmWTzjhhKJPN4sMTZo0KctHH310lmvnvdrmy2x/cVGzeL5KqTx2cWGiQw89tOgTF/Wo\nLaB3xRVXZPmWW27J8ic+8YmiT+01tr+4gFltgb1XXnklyytXrsxybeP3MWPGZLl2Porv+9e//jXL\ntcWL4pg+7rjjijZf+cpX+uwTF1L6X5+vP/jLIAAAQINMBgEAABpkMggAANAgNYPvUdvQ+YUXXshy\nbePd2C/W20ycOLHoM2LEiCzHDSpnz55d9Hn55ZezHJ9/rj1LHOsz4maZ3WzCSW/EGsErr7wyy9df\nf33RJz4nH4/VLrvsUvSJda+33XZb0eb3v/99lmOtadyIN6WUvvjFL2b5ggsuKNocdNBBWbaZ/fsT\nfxdxzKRU1g7vt99+Wa5tdnvEEUdkOY6bBx98sOhz7bXXZvnJJ58s2vzkJz8pXmNwiNeCu+66K8ux\nTialskb07LPPznK8/qTU3fc7XttiTc7o0aOLPrVrGdtePA7x2MV1C1IqryWHHXZYlmvjKI6ReE5L\nKaW99tory7H2K9bgp5TSxz/+8SzXrn30Xjy+y5Yty/K9995b9Bk2LJ/S7LvvvlmOayHU+tRqEeOY\nnjdvXpaXLFlS9Jk5c2aWTz/99KJNHFuxZjDeL6WkZhAAAIB+ZDIIAADQIJNBAACABqkZfI/47HBK\n5X5HsR4npbK+ZsGCBVkeOXJk0WfdunVZXr58eZY3b95c9InPCsfnmGv7iX31q1/N8pe+9KUsx/0O\n6Y3a8fzOd76T5ZtuuqljnzhG99577yzvs88+RZ9Vq1ZlubZHV3xOPu4NVXtO/eabb85ybY+5888/\nP8uxNq32THykhvD/q9WvHHPMMVmOdcEf/vCHiz6xliaeS+J7pFTWJv7lL3/p87MyuGzZsiXLd999\nd5bXrl1b9Il7lsa9R7f2uxvrheIeprvvvnvRR83gwBCP3cMPP5zlWr3n5MmTs9zpXqf2c+J1ria+\nb+18Gq+FtTp816T+VTu+8Rr0j3/8I8txDY2UUjr33HOzfPDBB2e5dr8bj2U8D6ZU3sMff/zxWa7t\nuTtjxowsx3VCUuru/mdbGTifBAAAgG3GZBAAAKBBJoMAAAANMhkEAABokAVk3qNWFBwLTmsLyJx1\n1llZjpvQv/HGG0WfNWvWZPnFF1/M8p///OeiTyyyjZ+3toDHJZdckuW4yIxC6N6Ixe133nln0eYX\nv/hFluOG4rXi4rig0SmnnJLl2vh84IEHsjx37tyizeuvv57lWLxdGydxXD///PNFm/hvipvZ07f4\ne48LaaSU0qhRo/rsE/977X3iuaW26ExUK7Rn8Irf+bhgUO27GzcH79WCCHFBK2Nv8Dj00EOzvHTp\n0qLN+PHj+3yPeE+VUkrPPPNMlmfNmlW0efLJJ7Mcx3BcuKbWht6L90splQtWxfPR9OnTiz7xuhXv\n32vnp3jtq11jV6xYkeUDDjig42eJixMN9HttfxkEAABokMkgAABAg0wGAQAAGqRm8H2qPXMcNyWN\nzyB3s2lprNnasGFD0Sc+Vx2fh545c2bRZ9y4cX3+XP7vahumLliwIMuxdjOlsuYuHpva5rxnn312\nlj/96U9nec6cOUWfz3zmM1mOYyKlcgPxWNNa+zfGZ+Ljxqy1NsOG5aec2vsao//bjjvuWLwWaxzi\nuaS20W6n2q7apvPx58SN61PqXNfMwLV+/fosxzFQO96xHqxWbxN1MyY6jc999tmneK2bn03vxeM7\nZsyYLMe695RSGj58eJbjtfGxxx4r+syePTvLjz/+eNEmjolYI3jaaacVfWrnS/pXvE7UaoCXLVuW\n5Xj+OfDAA4s+8f4ijsXa/Ua8r960aVPRJo6/qVOnZrmbe/yBzl8GAQAAGmQyCAAA0CCTQQAAgAap\nGewHnZ5Lrj07HPfNeeqpp7Ic635q7xPrxS688MKiT3yGmv735ptvFq99+9vfzvLy5cs7vk88VjNm\nzCjafOELX8hyrC2N+9+kVO49WKv9iTWCDz/8cJZr/8Y4hvfdd9+iTe0Z/ffqZt+fwfbsfS/Vfl8H\nHXRQllevXp3lVatWFX3233//Pn/On/70p+K1eLxrY6LT8WbgevXVV7Mca2lq15I99tijJ58ljqM9\n99wzy7VzgvPEwBCPXazvrNV6xfNa3FPutttuK/osWrQoy7XzUaxFjJ+lVpcf6wyNq+0jHruTTjop\ny7Ua4bivcTdrFMTr2rPPPlu0iTWsH/rQh7Lcq/1Vt6XB/y8AAADgfTMZBAAAaJDJIAAAQINMBgEA\nABpkdZEeiBto1jbU/Ne//pXlBx98MMuxeD+lclPvG264Icu9KuYnF4uQ582bV7T529/+1mefmrFj\nx2b5vPPOK9qMGDEiy7H4Of73lFJauHBhlp944omizUsvvZTlWJxd21R17733zvJuu+1WtIm6+T0o\n2P/far+b3XffPcsrV67M8nXXXVf0mT59epbjglU33XRT0Wfz5s19/tyUynNdLKx3bAeG2vVlzpw5\nWY6LU8WFqFJKaeedd85yXMChtshDNxtBx0WQYp/jjjuu6DMUFnEYCuLxfOutt7JcO05xPO64445Z\nrm0OHxfdqy1wNH/+/CzH66MxM3DF+4t4rxPPPSmV4ygu/BLPTymV55p4L55SSscee2yWh+K9tm8C\nAABAg0wGAQAAGmQyCAAA0CA1g/0gPiMfNz999NFHiz4XXHBBluOzzTXTpk3Lcty8VT3OthHrom6+\n+eaiTTfHMz7zfv7552d5woQJRZ/4fPtzzz2X5SeffLLo89hjj2U5buaaUjlmY/1FbWzFeqAxY8YU\nbei9WPcSN5Sv1cXceeedWY41oxs3biz6xDFQ2+Q5btgbayvGjx9f9In1QfRerL9JKaWHHnooy5s2\nbcpy7TjFusJ43qv1ieOxVr/4z3/+M8tr167N8kEHHVT0cf0bGOJ54emnn87yiSeeWPQZN25cluOY\niJt8p5TS5MmT++yTUkrPP/98luO9Wm18GkfbXq22ONYMRrVjF8dArBGM908plfdHtXUXjj766CwP\nxVrTofcvAgAAoCOTQQAAgAaZDAIAADRIzWA/iM8pL1++PMvXX3990SfuBRbV9nX72c9+luXac9b0\n3po1a7J81113FW1ibUI3+8PFuohXXnml6HPfffdleenSpVn+97//XfR5+eWXs7zvvvsWbaZOnZrl\nuE9irT4s1k4+/vjjRZuTTjopy7VxTf+K+5FedNFFRZs//vGPWY7jM+7hlVJZo1Hbs+nee+/Nchw3\nl1xySdFn1KhRfX4W+t9rr71WvPbCCy9kOe4reOqppxZ9Jk6cmOVYj1w7b0S14z137tw++3Szpym9\nV9sjcsmSJVmOdaS1mvVYgxXPNbV7nQMOOCDLtfNR/FmHHnpox89C78VxU6v3jMc8jpFa3V48l8Rc\nGyPHHHNMlmvnlvhaN/d3vdKrn+0vgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZAGZ96lWML158+b3\n3af22nvFDclTqm/YTO/FY7Vo0aIsx42XUyoLomtFvnEBlt/+9rdZjgvMpFQu8hA/W9wsPqVy4YeP\nfvSjRZu48Ex839om5HHcr1+/vmiz0047Fa/RW3Gs1Tbo/sY3vpHlWET/+uuvF33iAkFxcaCUUjrk\nkEOyvP/++2c5Lm7DthG/z88++2zRJi708alPfSrLF198cdFn9OjRWY7ntLh4VUrlGKgtZnPPPfcU\nr3Xqsz0XdWhVPN4ppXTLLbdkefHixVk+8sgjiz5xg/G4sEs3m3zHzcNTKhch+eY3v5llY6T3aovD\nxO/vq6++WrTZZ599+nzfd955p3gtLlj10ksvZbm2gF281167dm3RJi64NmPGjI7vG8fs1oy1Wp9O\nc4et/Vn+MggAANAgk0EAAIAGmQwCAAA0SM1gB/F551p9YNzIMm6YWquTic/0xueAaxtf2mR+YIjP\nstfq9OLz4rXnvONz8/PmzctyrW40bhh/yimnZPnwww8v+px22mnFa9ENN9yQ5XXr1mW5tlnrrrvu\nmuUxY8YUbWq/m/dSs9F7tfNGrEf9yEc+kuVjjz226BPHa63NCSeckOU4XuO5MSVjYFuI9TVPPfVU\nxz6xjjTWB6ZUHs94nuvmnPDII48UbWKtYX/U39D/Nm3aVLwWawbjPdOUKVOKPrGOsJsawVhnFmv5\nUyo3mR81alTH96V/1dZU+P73v5/l2vlov/32y3K876rdk8Q1FSZNmpTlc845p+gTz42XX3550SZ+\nvtNPPz3L3/ve94o+sV6+dh3udJ9Y+x70ah7gL4MAAAANMhkEAABokMkgAABAg9QMdhBrE2p7m8Q9\nueJz9LFuJqWybiI+V197Vrib/UXovREjRmR5+vTpRZv7778/y3H/m5TKPfhi/U1tH7cLL7wwy0cd\ndVSW4/5MKZVjac2aNUWbRx99NMvxefzaXkFxXE+dOrVoE8e5Wp+BIR6HWEvz4osvFn3imKjVp8Yx\nEce44799xOvWihUrijbxHDV27Ngsd1MbHffbqtU9R7GuK6Vyz8NYQ1+7Phpb296SJUuK1+L1JZ4D\nxo0bV/SJ9efd3HfNmjUry7V9bqdNm5Zl6y70XrxPjfcWKaU0e/bsLK9atapoE495HBNxb8qUyjr2\nuMfuAQccUPSJ9cm1PUzjde2MM87Icm1Mx/u5bs5Psc22PKf5yyAAAECDTAYBAAAaZDIIAADQIJNB\nAACABllApoNYwFkriI8LxsSi6sWLFxd9YjF+XKBjwoQJRR8LyGwfcQzEYuIbb7yx6HP77bdn+fHH\nHy/axA2Zjz/++Cx/7GMf69in06buKZXjplb0H8doNwsaxU2pjzvuuKKNgv3BIR7fvfbaq2gTC/pr\n58I4HrvZPJrei9/DuOhZSuUCMuvWrcvyli1bOr5v1M1CaLVzWBxbcYGRbhamoffmz59fvBbHSVxI\n47DDDiv6xDbRxo0bi9d+97vfZbm2yMy3vvWtLFtkaNubPHly8doRRxyR5dqiLfG8EDed//rXv170\nOfPMM7McF3aJ55GUynPY1VdfXbSJ4zPe69TG72Aba67UAAAADTIZBAAAaJDJIAAAQIPUDL5PtRqI\nuKlqrMmKNYUplc8px816jz766KKPmsGBIT4fHjdnTqncHL5WzxCPZxwTtWfOO21KWhsjsf7v5ptv\nLtrEDXvj+8TxmVJK5513XpZrz+MzOMTzWm1Mxw18a23UCA5M8dxSq7d54YUXsrxgwYIsL1y4sOhz\n+OGHZ7lT7VdKZX187X3ffvvtLMda6d13373oM9hqdIaCkSNHFq91uq7F2q9anzhGbrnllqLPsmXL\nslyrTattBk5vxe9h/O6mlNIdd9yR5Xj/kVJ5Dthvv/2yHO+7UyqvP3Fc1e6PdttttyxPmTKlaBN/\n1tZsKD/QuXIDAAA0yGQQAACgQSaDAAAADTIZBAAAaJAFZPpBXFzjwAMPzHKtODaKha+1wuxYVM3A\nUCsejkXztc2ZYzFzN4vBdGpTGyMbNmzI8pw5c4o2sV/8OVOnTi36TJo0Kcs2mB+84vGeOHFi0Wbt\n2rVZtvH34BGPb23j78suuyzLP/3pT7P84x//uOhzyimnZDmeJ2qLDD3yyCNZjhuDp1RuMh7HY20B\nGba9E088sXht1KhRWY73NosWLSr6xHGydOnSLP/6178u+sRNya+99tqijQWttr/aMYj3t90sRLQ1\n4nvUFvKLn2/48OFFm6G4YEzkmwIAANAgk0EAAIAGmQwCAAA0SM1gP4jPrsfamtWrVxd9Yo1WrLeq\nbaipJmto6fTceTfPpXezqeqWLVs6toljOG4MfNFFFxV9Ro8e3fHzMTi99tprxWuxlqJWf8HgULuW\nxE27p02bluXvfve7RZ/Zs2dnOY6RWm3fpk2bshxrmlNKacKECVm+4oorsuxaODDEDbtTSunLX/5y\nlu+5554sX3rppUWfk08+OctPPPFEltetW1f0ufLKK7M8ffr0vj8sg0ov6vK6OW8MxXrAbvjLIAAA\nQINMBgEAABpkMggAANAgNYP9oNMeXbUarc2bN2c51gjGuoqUyr2X9tprrz4/B0NfPOa1PX3i2Ir7\nA6aU0ooVK7L8yU9+Msu1/aTU7Qwdb7/9dpYfeOCBos1bb72V5XgOY3CLe2mdf/75WT7jjDOKPldd\ndVWWX3rppSwvW7as6BPPP7U9D2fOnJnl/fffP8uudQNDHDMppfTDH/6wzzb33ntv0ecPf/hDlseN\nG5fla665puhzzjnnZNmegkTdnCe6GTctnG98ewAAABpkMggAANAgk0EAAIAGmQwCAAA0yAIy/SAW\nlx511FFZPvfcc4s+t956a5bj5q3Lly8v+qxcubLPPrWN6hnaullAZsSIEVn+2te+VrSJYykuGLPz\nzjtv7UdkEIiLw2zZsqVos3bt2iy/+eabRZsWCu1bEc8lcRGXlFK68cYbt9XHYZAYOXJklq+77ros\nX3311UWfuIDVLrvskuXaQjUWjOH9ql2f4kJ4tQUfW7iu+TYBAAA0yGQQAACgQSaDAAAADdqh9nzs\nADAgP9TW2rBhQ/FarLUYO3ZslmOdV0opnXTSSVkePXp0locNK0tAB/izzr3+cENqHHWj9n2O9V+r\nVq0q2sQNxDdt2pTl8ePHF31ibch2ruHo5Vga8uPonXfeyfLGjRuLNgsXLszykUceWbQZPnx4lgf4\n+afGOKI/GEf0B+Oox7ZmDjQUr2v+MggAANAgk0EAAIAGmQwCAAA0SM3gdhJ/74PwGeT+oGZwG4hj\nbWv20emmz3Yew2or+lE314Uhes4yjugPxhH9wTjqsa25PxqE1z41gwAAAJRMBgEAABpkMggAANAg\nk0EAAIAGWUCG7ckCMvQXhfb0B+OI/mAc0R+MI/qDBWQAAAAomQwCAAA0yGQQAACgQQO1ZhAAAIAe\n8pdBAACABpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACA\nBpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAA\nNMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAANMhkEAAA\noEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABv0/PzWtg7b77FkAAAAASUVO\nRK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAACsCAYAAADBhXmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAGlVJREFUeJzt3WmsnUX9B/ABgdqFbnSlQBeRgqXQCMSyLw0Q2iKKggpi\nGmJ8JSFgSIgalxCNSIwhEJcYXIKBKAYpUMAgBCjgxi5LW5ZCoay1pQtbW+T/xhed38y/5wL39p7e\n+XzefU9n7jntM+d5nul9fjM7vPfeewkAAIC27NjfHwAAAIBtz2QQAACgQSaDAAAADTIZBAAAaJDJ\nIAAAQINMBgEAABpkMggAANAgk0EAAIAGmQwCAAA0yGQQAACgQSaDAAAADTIZBAAAaJDJIAAAQINM\nBgEAABpkMggAANAgk0EAAIAG7dTfH+D/8V5/fwC2iR36+OcbR+3oy7FkHLXDOKI3GEf0BuOI3tBx\nHPnNIAAAQINMBgEAABpkMggAANAgk0EAAIAGmQwCAAA0yGQQAACgQSaDAAAADTIZBAAAaJDJIAAA\nQINMBgEAABpkMggAANAgk0EAAIAGmQwCAAA0yGQQAACgQSaDAAAADdqpvz8AANvWf//73yzvsMMO\nW83w3nvvFa+98847WV6yZEmWr7rqqqLPfffdl+WxY8cWbaZOnZrlBQsWZHnvvfcu+uy0k9uZ7UFt\nHG3evDnLa9euzfLq1auLPhs2bMjybrvtVrSJr330ox/N8kc+8pGij3MfvSFeY2u66brrN4MAAAAN\nMhkEAABokMkgAABAg0wGAQAAGrRDrZi3C3Tlh+p2H+RY9nOxdF+/+XY1juLxqxUgx9diAfyOOzb7\n/zt9OZa6ehzFcfP2229nec2aNR37jBo1Ksu1cdRpkYeUOi/YsB1odhx18uabbxav3XTTTVm+7rrr\nsrx06dKiz5AhQ7JcW/jl2WefzfKECROyfMUVVxR9pk+fnuUBfG3brsdRXHQopZQef/zxLD/yyCNZ\nfuaZZ4o+8do3ceLEos3o0aOzfMghh2Q5jquUUtp5552zbBwRxetnvDamlNIbb7zR8ed0OhfWrsPx\nvWvjs9Km4zhq9s4RAACgZSaDAAAADTIZBAAAaJCawQ5ijda7775btImvxeeHaxubxmeD4yO9tXqx\n+D6rVq3K8s0331z0iT9n9uzZWZ4xY8b7/my9qOmawfjdizU5d955Z9Hnsccey/JRRx2V5QMOOKDo\nM2jQoCzXjucA2Gi32dqKTZs2Zfmee+7Zak4ppb322ivL8+fPz3Ksm0kppVdffTXLt99+e9Hm5JNP\nzvL48eMrn7irNTuOoni9WbRoUdHm0ksvzfLgwYOz/P3vf7/oEzeM37hxY9Hmxz/+cZb//Oc/Z/nE\nE0/s+Fn6eRN64+h/4v1Q7bp29dVXZ3nYsGFZrl3XJk2alOVaXWG8Xo4YMSLLCxYsKPpMmTIly7Vz\n4TZkHP1PvF/qydylt+5r4rkwXgvj+SmllP7xj39k+fTTTy/aHHbYYVnedddds9yTc1gP/45qBgEA\nACiZDAIAADTIZBAAAKBBJoMAAAAN6tcK625TW7Qlbqz8q1/9qmizZMmSLMfi0trmk5MnT85y3PQ5\nFlCnlNKTTz6Z5X/9619ZfuGFF4o+sfj5zDPPzPLFF19c9Bk+fHiWB8DiIl0pFiXfeOONWT7//POL\nPnE8jh07NsvHHnts0eess87K8oEHHli0iZufxnFT2/zUuNj2akXz8Xxz1VVXZXnDhg1Fn3333TfL\ncXP42vF+/fXXs/ynP/2paDNy5Mgsf+5zn8uyMbP9WL9+fZb/8Ic/FG3ioh3XXHNNlmvnmrigWu26\nO3fu3K2+94oVK4o+tZ9D/1uzZk2W42IxKaX0xBNPZPlHP/pRlmfNmlX0ieNon332KdrEe6a//vWv\nWa4tCPi9730vy7VFPJzHelftuvbOO+9kOd5nr1u3rugT2wwdOjTLcfGqlMoFg2oLPj799NNZPvfc\nc7O8fPnyok9cpK22EFFccKsnizf21aKffjMIAADQIJNBAACABpkMAgAANEjN4Bbi5s0plRsr//GP\nfyzaxJqdWKOzyy67FH2WLl2a5VizEzcgT6l89j5u5lqrmYhtli1bluVaPWOsGaRvxBrPCy+8MMsv\nv/xy0SfWcr311ltZfu2114o+sUbjyiuvLNrEzU7jZqgnnXRSxz7qKPperV7gwQcfzHKsm9htt92K\nPrG2dNCgQR3fO9bX1Daz33PPPbMcawbpXnFsPfzww1l+4IEHij5HH310lmfOnJnlD7ppcryWrV69\nOssvvfRS0adW/8W2F8dRPB/F+quUyvPRQQcdlOVYW1V7n/HjxxdtPvGJT2R54cKFWV60aFHR55xz\nzsny7rvvXrThw4nHrvbdjfXIcWP3eH5KKaVp06Zt9X3efvvtok+8h6qNz0ceeSTL8T461jemlNL+\n+++f5Vr9dLzu9uQeqq/us/xmEAAAoEEmgwAAAA0yGQQAAGiQmsEt1PbWinuFxP0AUyprv+LPqe0Z\nGF+L+8W98sorRZ+479PGjRuLNlGs2Tj88MOzXHvOXu1X76vVo37961/Pcm3vrCiOxzlz5mR59OjR\nRZ9Yn/rQQw8VbWLda6yNrdVNxLFk3PSPeK548cUXszx79uyiT6d6z1pt4n333Zfl2v6FtdoJtg+x\nTu+6667LcqzbS6msi4l7dPVkT6xavVDsF2vqp06dWvRx/ukOcRzF/ZBHjBhR9Pn0pz+d5do6C53U\n7oeeffbZLMc1EuL1NCXjaEvxe9hX/za19S7i/e7KlSuzXLsniTXMsV5++vTpRZ94j1yrc77pppuy\nHMfwJz/5yaLPJZdcstXPklJ3jTW/GQQAAGiQySAAAECDTAYBAAAaZDIIAADQIAvIbCEWv6eU0t57\n753l448/vmgTC5fj4gzz588v+sQNKEeOHJnln/zkJ0Wfp556qnhtS7Vi1BkzZmT5vPPOy3Lt79xN\nRa3bq1gQ/eijjxZt7rrrrizHYu1x48YVfeLG9Mccc0yWa4sxxLFU25j+zTffzPLmzZuzHAvv6R+1\nBTlioX1sM2HChKLPB/mOx815a+eOMWPGvO+fS3eI3/nHHntsq3+eUnndimOvNl7jubF2zooLf8Tx\nGt83pfp4ZNuLx3ft2rVZri1aN2XKlCzH411bYCT+3N/+9rdFm1/+8pdZjmP4jDPOKPrExfxavh/q\nz797XAxv8uTJWd5vv/2KPkceeWSWhw4dmuXa4jBxTNQWInrwwQezHK+pl112WdEnLhjT7ePIbwYB\nAAAaZDIIAADQIJNBAACABqkZ3ELtmd642e2kSZOKNgcccECW4/PE8+bNK/rE54ljPc7y5cuLPp02\nma9t5nr11VdnedSoUVnu9ueYt1dx8+2f/vSnRZtYp7fzzjtn+bTTTiv6nHLKKVmOz8THZ9tTKp+T\n78kxj7U+gwYN6tiHvlc7di+//HKW16xZk+V169YVfWKdVqy3qtVxxdrEYcOGFW2mTZvW8fPSneL1\nJW4yP3jw4KLPjjvm/5+8adOmLNdqBmOf2hiJYy2qbRYefy79I9b3DR8+PMt77LFH0Sdeo+I4evHF\nF4s+t9xyS5avvfbaok28Ds+cOTPLp59+esfPQt+rfXdjLd/HP/7xLE+dOrXoE++H4j1VTVwP4de/\n/nXR5u23387yBRdckOWJEycWfba3a5+zJwAAQINMBgEAABpkMggAANAgD0dvofaMb3yWOe4hmFJZ\nb7Ny5cqt/nlK5TPxixcvzvLf//73ok+s44n1jF/84heLPrGGR11F76vVxSxdujTLN9xwQ9EmHs9Y\ng7XnnnsWfTZs2JDlJ554Isv3339/0Sc+ex/HTUplzeouu+ySZftRdq+4t1+s7/zd735X9Jk7d26W\n999//yzHetaUUlq2bFmWa7U1RxxxxNY/LF0r1gzG/dbiOSKlsq4w1qfGerGUyvFZO3/GutfYp7af\npfNRd4jHM54TYh1fSmWN6H/+858sL1y4sOjz/PPPZ7l2zop79Z500klZrtU9G0d9L/4b1+5L416i\ncc/veC+UUjm2OtU0p5TSzTffnOV4L55SSkcffXSW4/VzIIwZMwMAAIAGmQwCAAA0yGQQAACgQSaD\nAAAADbKATAdxkYS4WXxKKb322mtZfuSRR7J82223FX3iIh5xE8taMXQsUo2b3V988cVFHxuo9r3a\nBt3//ve/sxw3LU2pPJ4xP/zww0WfWGgfF5CJixelVG68WtuINb53bDN+/PiiD9terdB+wYIFWY6b\n5sYxklK52fIJJ5yQ5bhxdEr181gUi/rjz7GAVXeoLdrSaaP3uBhHSuV5IW7gXFugI54va9e65557\nLsvxejljxoytflb6T/yOT5gwIctxoaKUygXL4gIytXEUN/qePn160WbVqlVZPu6447I8EBb+GAhq\nx2HUqFFZjuPmscceK/pMnjw5y3ET+to9VbxvjuewlFL65je/meXaInzbO1dmAACABpkMAgAANMhk\nEAAAoEEKyjqItRXxmeSUylqKIUOGZDnW8KSU0qOPPprluMluTXxu/pJLLsnyrrvuWvTxTHzfq9Xf\nPP7441mOmyanVNZTxeMXN3ROKaWnn346y3Hc1J5378nGq/HvEMdNbYPX+PlrG9PT92Id8y9+8Yss\nf+UrXyn6xM1577jjjizX6mDXrl2b5dq55YYbbshyrGt2jupe8Zz15JNPZvnQQw8t+sQ6rbjJfK1G\nNNZPx/qwlFJavnx5luO5pla/aBxte7VrXzxWsdarduxijXo8/8yZM6foc99992U5bkKfUjm2avXy\n9L/adzfWkY4ZMybLxx9/fNEn3qc89dRTWa7di7/44otZnj9/ftFmv/32K14baPxmEAAAoEEmgwAA\nAA0yGQQAAGiQmsH3qVaTFesI4z6DDz30UNEn1nrFZ+9r+wPOmzcvy7GGQ81E/9i8eXPx2ltvvZXl\nnuztF2u/ascz1m3FcVN7n1jL15M9LOOeY7E+NaWUvvOd72R5n332KdrY57LvxWN3xBFHZPnWW28t\n+lx77bVZfvbZZ7P8wAMPFH1iDWtt3Mfaw1jn/I1vfKPoE/dsshdh36sdu5tvvjnLsf7vmGOOKfrE\n73w8lrX9KuP5p1YbHffujWM81g/RP2o1g/HYxWthT77fcd2FmmnTpmV58eLFRZu4z1zM7pm6Q20c\ndbonHjlyZNEn1iNff/31WX7wwQeLPqeeemqWL7rooqJNN9/H1P7tPsi4dtUFAABokMkgAABAg0wG\nAQAAGmQyCAAA0KDurYrsB7VCzFjsXlsM5plnnslyLIivbdhde68t1RYCOfroo7Os+Lk71BZjiEXy\ncQPVlFIaPHjwVn9OXNQjpXID3zgGxo8fX/SJG6bec889RZu4oFFc+OGWW24p+sSFAS699NKizV57\n7ZVli4P0vfhvPHXq1KLN2WefneW4gXNcnCGllO6+++4sP/HEE0Wb119/PctxQ99NmzYVfeKiI/S9\neFxSSum2227L8qhRo7J82GGHFX06HbvauTEuxvDCCy8UbV5++eUsH3zwwVmuLTDierjtxetRSuUi\nHfG4jB07tugT74dijgtRpZTS6NGjs7xixYqOn2/SpElb/WxsG/H41s4T8VrxzjvvZDnes6SU0uWX\nX57lRYsWZTkuMJNSSiNGjMhy7ZwWP283jZve+izuzAAAABpkMggAANAgk0EAAIAGqRncwrvvvlu8\ntnDhwixfeeWVRZtXX301y3HD7lq9WHxGurY5b7R06dKt/oy4uXhK3fVs80BV25B0//33z3KtvmrV\nqlVZjpvOx5rClMrn5mMNxEknnVT0iXWEK1euLNrEutc4bmp/x7333jvLcUPfmm5+9n6gqh27WG9z\n6KGHZnnmzJlFn7jBeKzPSKk8h86dOzfLtTFiDPS9+L2LtTQppfT8889nOR6XWt1WJ4MGDSpei/Xw\njz/+eNEmft4JEyZkuXZNpe/F47JkyZKizbe//e0s77rrrlk+7bTTij7xWhJrRGvHO36W2nVtypQp\nWVafvO3V7m3jsfrZz35WtLn33nuzHOs/a+sPPPXUU1ttU7tHjvdmv/nNb4o2CxYsyPLw4cM7fpYP\ncq/TG3162i/ym0EAAIAGmQwCAAA0yGQQAACgQWoGt1B7tjnuS7Ju3bqOPyfW1kyfPr1o889//jPL\na9euzXJtP64777wzyy+99FKW4/PxbBu1uphTTz01y7FuIqWyZjDuvxTrDlMqa2diXWHtmfhYZzhj\nxoyiTRxb8ed+5jOfKfp897vfzXKtpsi+gt0pjpNYA1EbR7EeMPZJqax7Pfzwwzv+XDWDfS/WldT2\ny33jjTeyHI9V7RwW61Hj9712bGOte+2aGvf3jTWu9I84ji677LKiTazBiteS2bNnF33mzZuX5ZEj\nR2a5No7+9re/ZTmu3ZBSSoccckiWa/XT9K14XkmprMFbvHhx0SbeA8cxMG7cuKLP/PnzsxxrRGv7\nYsZ9l6+//vqizQknnJDleC7stG/4/9emN6599hkEAADgAzMZBAAAaJDJIAAAQINMBgEAABqkmnYL\ncTPclFI666yzsnzkkUcWbe6///4sx8VgaotoxDaxoL+2gMzq1auzHBcG6asCVbaudnxHjBiR5c9+\n9rNFm3i8erL4Qnwt5toYiOO6tlBSLKyPi9l89atfLfrEDcQtFrP96jSuUirPc08++WTR5sADD8xy\nLLR3PuoO8dqRUnnuiAsGxcXUUurZ5sudPProo8VrcZyMGjVqq3/OthEX4IgL4aVULhAU86xZs4o+\nkydPznJcvKg29m655ZYsv/XWW0WbL3zhC1l2jdr2aouGnXjiiVmuLWi1YcOGLMdryZe+9KWiz9ln\nn53luHjNX/7yl6JPXJTv2GOPLdrExRl7cq/2QfTnec03AwAAoEEmgwAAAA0yGQQAAGiQmsEt1J7X\njZtWxg3lUyqfd48b5j7zzDNFn7vvvjvLcaPWWu1Xp/qbWi2YZ+T7Rzw2/bnZbaz9ee6554o2sa4j\n2n333YvXjK2Bq3ZsX3nllSyvX7++aBM3BzdGukM8H51yyilFm2uuuSbLsa5w+fLlRZ8xY8Zs9X1q\nYr38kiVLijaxzjnW8RhX/SMel4MPPrhoE4/n1KlTs3zccccVfWJdWbz/qdWV3n777VmOtV8ppXT4\n4YdnWa3ptjdkyJDitfPPPz/LX/7yl4s2y5Yty3I8vpMmTSr6xPH52muvdfx8e+21V5Zr6zvEecBA\nHEfOqAAAAA0yGQQAAGiQySAAAECDTAYBAAAaZAGZ96lWuB6LS2ObuPBLSuUmqnHxl0GDBhV99thj\njyzHRUkGYlErH14cW7Wi6jhmYyH2Lrvs0vsfjK7Rk3PH66+/nuXaubB23qL/xeM7d+7cos3JJ5+c\n5TvuuCPLP/zhD4s+3/rWt7L8sY99LMu1zcK/9rWvZbm2oFVc1OGQQw7Jsmtd/4gLvfzgBz8o2sQF\nZGoLu0RxkbPVq1dn+ZJLLin6bNy4McsXXnhh0cb56MOJC/n01vcu3rtOnDixaNNp3NQWWYzjKG5c\n/+qrrxZ9pk+fnuV4n90KvxkEAABokMkgAABAg0wGAQAAGqRmsBd0eo669mxzrMGK9Te1epwpU6Zk\neejQoR0/W1898832I24eXdtgPo6luDHw8OHDiz7G0sBVq/V6+umns/zmm28WbWL9hc3Bu1Osc08p\npcsvvzzLsa5w0aJFRZ/bbrsty7E+PtYrp1SOrWnTphVtbrzxxizXNq6m/40dO7Z47Zxzzsnyvffe\nm+Xf//73RZ/DDjssy1deeWWW47knpbL29IwzzijauEZ9ONvq36/2Ph/kvjq+NmzYsCx/6lOfKvqM\nHDkyy62uj+BKDQAA0CCTQQAAgAaZDAIAADRIzWAfiHu01fYZjPv1xFyrtVm3bl2W475fY8aMeV+f\nk4GnJ3vv7LfffkWbVatWZXnGjBlZjmOagSWOm1qNTjzfxL3gUmp3j6aBINZ/3XXXXVmONYUppfTz\nn/88y3EcjRo1qugzZ86cLJ933nlFm57sTUf/q10XPv/5z2d5n332yfKtt95a9LnmmmuyHGvUL7ro\noqJPHEdx7zoGtlpNYbxvjuefWNOeUjluanXOtfuqgcZvBgEAABpkMggAANAgk0EAAIAGmQwCAAA0\nSMVtH4hFrOPHjy/anHnmmVletmxZltesWVP0GTx4cJbjgg49YRP6ga12POOGzfPnzy/a7Lnnnlk+\n6qijshwXOGJgiws4pJTSpEmTsjxz5syizbhx47Ls/LL9itebCy64oGhTe422xU27DzrooCzPmjWr\n6LN58+Ysx/NGbXEY1ySieO8dz2H77rtv0Wf9+vVZ7slm9p1yTe1a2E3XR78ZBAAAaJDJIAAAQINM\nBgEAABqkZnAbqD3vfsQRR2T53HPPzfIVV1xR9ImbYcYNNTdt2lT0ic9Q1zazZ2CLtRXDhg0r2hxz\nzDFZjjWEtY1Y4/Pu3fT8O+9PPHZxo+iUUrrpppu22iellIYOHdq7HwzYrvWk/s+G8fSFeL87aNCg\nos2KFSuyvPPOOxdtYr84pmv3R3FMd/v9kZkBAABAg0wGAQAAGmQyCAAA0KAderI/Rj/oyg/Vm+Iz\nxvF54lr938aNG7Mcn2Puyb47XVbn1ddvPuDHUU/E7/i7775btInj4IPUlg7gsWQctcM4ojcYR/QG\n4+hD6Mn8Jt5Xxz0vUyrvteP9Ue19umwfzI7jyG8GAQAAGmQyCAAA0CCTQQAAgAaZDAIAADTIAjL0\nJwvI0FsU2tMbjCN6g3FEbzCO6A0WkAEAAKBkMggAANAgk0EAAIAGdWvNIAAAAH3IbwYBAAAaZDII\nAADQIJNBAACABpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNB\nAACABpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkM\nAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAANMhk\nEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAa9H/jW9O1wPwCqAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAACsCAYAAADBhXmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAGllJREFUeJzt3WuMnUX9B/BBAVt6oS0Fei8UWkGEXhJTtRJsImAFRUAT\nRBPEYIyJiCIimoCRqu+QYEI0hWoQQwDBxCjRcKfFCwUspUIpBaECBVtobemNm/xf/pnfjHu27W57\nuvP5vPsuM+ccembneSb7/Gb2efvttxMAAABtedee/gAAAADsfhaDAAAADbIYBAAAaJDFIAAAQIMs\nBgEAABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADbIYBAAAaJDFIAAAQIMsBgEAABpk\nMQgAANAgi0EAAIAGWQwCAAA0aN89/QH+h7f39Adgt9inn1/fOGpHf44l46gdxhF9wTiiLxhH9IWO\n48hfBgEAABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADbIYBAAAaFC3njMIAAC7zdtv\n58fv7bNPfx+HDHuevwwCAAA0yGIQAACgQRaDAAAADVIzCADAXivW+qW0c/V+sU/tdfvifaCb+Msg\nAABAgywGAQAAGmQxCAAA0CA1gwAD2H//+9/iZ5s2beoxr127tugzcuTILO+///5FmyFDhmR56NCh\nHfuwd6jVTsVxc/PNN2f5zjvvLPps3Lgxy8cff3zR5uyzz87yxIkTs7zvvm5dBpJOdXm1/x5/Vpvn\nYpudqf9797vfXbR517t2/e8ofVXjCH3BXwYBAAAaZDEIAADQIItBAACABlkMAgAANGif3hTU7gFd\n+aHoc/1dLd3V4yj+7r3xxhtZfvPNN4s+sZg9bsjRcAF6f/6Pd/U4it56660sr1mzpmizaNGiLMdN\nPQ444ICiz5QpU7L8+OOPF202b96c5dNOOy3LRxxxRNGnLzZj6EPG0f/w6quvFj+76qqrsnzbbbdl\n+ZVXXin6vOc978nywQcfXLQZM2ZMlr/yla9kec6cOUWfLttUxjj6H2obvcRrXbyOxWtjSuU8V3vd\nOB9t27Yty7V5Lo7PYcOGFW3iWOvNHLaTh9cbR10ojrXXX3+9Y5/a5mnx++7H+7eOL9xVV2EAAAB2\nD4tBAACABlkMAgAANKirHrLfW8Xn2eNz6rXar071Db2pf4jPF9eeSR80aNAOv0/DdWf9pvbdbN++\nPcv33HNPlm+//faiz6hRo7J83nnnZXn06NFFn1hnWPt+u6xui12wZcuWLF977bVFm2XLlmX5lFNO\nyfLpp59e9Im1NOPGjSva3HLLLVk2l+y94nXr1ltvLdr87ne/y/LIkSOz/L3vfa/oM3ny5Czffffd\nRZtf//rXWb7sssuyHGsVU0pp+vTpWTb2ukOs7avVka5cuTLLcezV6gHjtS7OTymltHbt2iwvX748\ny/EanFJKY8eOzfKnP/3pok28b4rX996MPePz/9Xuj/piP5Pe/BvH96m9b6w1vemmm7JcmxuPPvro\nLF966aVFm+HDh2d5T44Jd4AAAAANshgEAABokMUgAABAgywGAQAAGmQDmR302muvFT978MEHs/yt\nb30ry//617+KPkOGDMlyLJiOG4WklNKRRx6Z5UMPPTTL8eDo2ut89rOfzXIsuk+pPIjV5iK7rlaU\nvHTp0iz/7Gc/y3Lc5COlcjOYl156Kcsnn3xy0WfixIlZjgc6p5TSIYcckuUuO8CZHsTNFf7yl79k\nOW5MlFK56dWpp56a5aFDhxZ9YnF7rc0f//jHLMc5af78+UUf80t3ihuh3XHHHUWbOK/9/Oc/z/Kk\nSZOKPnEcTZgwoWjz5z//OcuLFy/OctzAIaWUjj322CzHuZLdI46JuGHMddddV/S56667shzvSz76\n0Y8WfeI1q7b5xqpVq7K8ZMmSLMcNZlJKad68eVmuHXgfN+azYcyOidesrVu3Fm3i/c+aNWuyXNsw\nKG70EjcDqm2wN2zYsCw/99xzRZuFCxdm+YYbbshy3CQppZRGjBhR/KybuQoDAAA0yGIQAACgQRaD\nAAAADVIY1EF8FjgejppSSt/85jezHJ91rtWLrV+/PsuxhrBW/xffO9ZE1J5bjrWIsU7tJz/5SdGn\nN4f3ev59x8TvIaWy7uWxxx7Lcu05+vjdTJ06Ncu18fmLX/wiy7VDf2+88cYs1+p46E5xbF1//fVZ\nXr16ddHnoosuyvLBBx+c5Vq9VZzHamP6qaeeyvK6deuy/IMf/KDoQ3eI32+cj55++umiz1lnnZXl\nWCNYqz2O7zN48OCiTRyPr7/+epb/8Y9/FH1im9rr0v/i9/vEE09k+fbbby/6xHFzzjnnZDnul5BS\nOUfV5qOjjjoqy88//3yWY01hSmX9dK02jV0TawZr+2rEe5J4LanNLYcffniWn3nmmY59YpsHHnig\naBPnvrh3SO16Gfdm2H///Ys23XQf7S+DAAAADbIYBAAAaJDFIAAAQIPUDL5DrbZvy5YtWa7VvMT6\nhVi7F88xSSmlcePGZXn8+PFZrj1D/fLLL2d506ZNPb5vTXxGvnbuCrsujqXt27cXbR5//PEsx7H2\n3ve+t+hzxRVXZDmeZXPfffcVfR566KEsb9iwoWgTz/GKtUDd9Gw7uTi2/vrXv2a5VvNy2mmnZXln\nzmSr1TXHWop4Hpez37pXnLPinBBr8lJK6f3vf3+W4zxRu6bGn8V5L6VyzMb6v3jG7v96L3a/eB8S\nz2EeOXJk0ef888/Pcrz29eYs0to1Kt4zxTkrnsGbUkpz587Ncq3Wy9mouyZeB3rzPdx///09vkZK\nZW1pnLNqZ+PGWtN4/UypnKPi/DRr1qyiz2WXXZbleH53tzGiAQAAGmQxCAAA0CCLQQAAgAZZDAIA\nADTIBjLvUNuAJR4gH4uhUyqLVGPB8emnn170OeOMM7I8atSoLC9YsKDoc9ttt/X4eWsF9J0KdWvF\n3LGPzUN23Ysvvlj8LG4SFAuMv/zlLxd9pk2bluX4Xc2bN6/oc+mll2a5tpnN3XffneW4gQzd69//\n/neW4/cbD79NaecO5I7zS9zAKqWyGD/ON+aS7hUPgl67dm2P/z2llIYMGdKxTbR58+Ys1w6QX7Jk\nSY+vUdtcq3agNLtfnAPidS4eBJ9SufFHbzZoie8TD5RPKaWLLrooyy+88EKWr7766qJPvMbaLKbv\nxetAbXOVj33sY1meOnVqlmsbow0fPrzj60ZLly7N8q9+9auiTZzX4jV14cKFRZ8DDzyw43t3E6Mc\nAACgQRaDAAAADbIYBAAAaJCH7N+hdvjt9ddfn+V4iGlK5TPlU6ZMyfI555xT9Jk+fXqWY31G7UDN\nbdu2Zbk3h+wefPDBWb7yyiuzHGs+UlLX0xfiM+Zbt24t2rzyyitZjs/AH3fccUWfWI8ax1489Dul\nlN54440s18bN6tWre2xjTHSH2ncXa3JeffXVLNfqJmKbgw46aIc/y4gRIzr+LI5h46h7xTkr1vbF\nuvZamzjP1epK161bl+VVq1YVbeL4jPU3J5xwQtFHzeDuV5uP4vf7n//8J8vHHHNM0Sfe78T9EOI1\nLKWyVvqqq64q2jz99NNZnj17dpY/8YlPFH2Mo92vVpc5aNCgLMf76lgzmlJZCx+vN3F/j5RSuvPO\nO7O8Zs2aok28N7vkkkuyHOtMa+/d7fxlEAAAoEEWgwAAAA2yGAQAAGhQ0w9Hx+fdn3jiiaLNjTfe\nmOXaGW377bdfluPZWpMmTSr6xFqKZ599tsf/nlL53Hx8Jrn2jPJJJ52U5VmzZmW5VpvIrovfRawJ\nTal85j2Oo5tvvrnoM378+CzHZ+QXL15c9Im1pjUzZszIcm/qUekO69evz3KsK421qSmV50p+4Qtf\n6PE1UirHRK2OJ47PE088sfKJ6UaxZvDwww/PcqwPTKms24rX0GeeeaboE+tTY71yzejRo7M8duzY\nos3eVqMzENSuE/G6dvzxx2c51n6llNKGDRuyHO9LatfPWIsY76FSKvdM+PrXv57l2ll17H61391O\ntZu1/RHiHBbzo48+WvS55pprsly7X/rMZz6T5S996UtZHghnUe79/wcAAADsMItBAACABlkMAgAA\nNMhiEAAAoEFNbyATi0vj4ZMplUWqvdlYIxZD33vvvUWbWCC9ZMmSLD/88MNFn/h542eJxdIppXT5\n5ZdnWcH07hELomsH7U6dOjXLK1euzPKCBQuKPitWrMhyPOQ7jqOUUtqyZUuW42GuNfFw1lofGzZ0\nh2OPPTbLsZg9HuCdUkrXXnttj33mzZtX9Inf9+233160ifNl3OjDmOle8foSf+fjxi8ppTRy5Mgs\nx3mjdk2KG2XVDo+OmxPFQ+drGxzRHeKYmDt3bpbjpmcplfclcXOY2qYecVOZ2oYj8ZD5D37wg1k2\nH3WvThsk1janitebuHnaN77xjaLPunXrsjxs2LCizRVXXJHl3txD7W38ZRAAAKBBFoMAAAANshgE\nAABokJrBd6gdWhpraWqHtB9wwAFZHjJkSJZ/+9vfFn1WrVqV5fj8e+2A3yh+lo9//ONFmwkTJmTZ\nM/K7R/x3PuSQQ4o2P/7xj7N88cUXZ7k2HhctWpTlWGuxffv2ok+sLa2N4SeffDLLsX62dnh4fG9j\nq//V/o0nTpyY5SOPPDLLjzzySNFnzZo1Wb7jjjuy/OKLLxZ9Yp3W73//+6JNrBnsTc21cdMdYp1W\nrE+eM2dO0WfGjBlZjvWAtRr1F154IcvxQPmUytrDOI5qNYPGUXeINYGxbrRWb9VpL4aNGzcWP4s1\nY/E+LKWyVl+t6d4j/j7He/Fabd/ixYuzfPXVV2d5+fLlHd/n3HPPLdrU7t8GGn8ZBAAAaJDFIAAA\nQIMsBgEAABrUdM1grK+KZxulVNZAxJxSSocffniW47PrtTMD4/knb731VpZ7U/8wfPjwLB966KFF\nm/gsfm/OSVR70fdq4+YjH/lIln/5y19mef78+UWfWP8Vz1aqfb+x/qZWN/Hggw9m+YEHHsjyd7/7\n3aLP2WefneXaOWTGUv+LYyuOmxtuuKHoM2rUqCzHswqfe+65ok+sa4411ymV81qs2bjyyiuLPmpP\nd7/ad/e3v/0ty3EuOfroo4s+8Xc+1iPX5qNYf1M7Hy72i/VBsS6fPaM2juK9TJyfYu1X7XXiHFC7\nNxs/fnyP75tSSjNnzuzxfdQwd6/43cTvrjaO7rrrrizHOa02jt73vvdlOZ7N/b/ea6AZ+P+HAAAA\nFCwGAQAAGmQxCAAA0CCLQQAAgAY1tYFMLEiNh+zGw3BTKotWa4foxo1oli5dmuW4qUJKZSFrLFqu\nHQweP/+mTZuyHA+OTiml888/P8vxANja+9QK+jtRdL3j4liKGzRceOGFRZ9YIP3UU09ledWqVUWf\n9evXd/ws8ZDxN998M8v33Xdf0edDH/pQlg888MCiTW3jHHZebcODWNw+a9asLE+ZMqXoE3/HN2/e\nnOUxY8YUfeI4WrJkSdHm5ZdfzvK9996b5TjnplTf+Ir+Fa9ZKZUbDW3dujXLvTnovXY9ieLmL7Xr\nbnzv0047LcvmlT0j3g9t2LChaBM3LIsb6tXmsDgvxOvR6NGjiz5xE6zYJ6WU/vSnP2V5+vTpWa7d\n68QxXbu3cb/Tt2pjIt6DxGvUmjVrij5PPPFEj69REze0Gjx4cMc+A5G/DAIAADTIYhAAAKBBFoMA\nAAANaqpmMNqyZUuWX3vttaJNrO2rtdm2bVuW43PKtcNQY23FoEGDslw75DJ+3vj8/mOPPVb0uemm\nm7J87rnnZjk+z197794cuBmf+fZMfa43dQexJmf27NlFn3g4eDyE/tlnny36LFu2LMvLly8v2sRx\nHet6Lr744qLPMccck+Xe1AvR9+I4it/d0KFDiz5x7hg+fHiWJ06cWPSJ89j1119ftInzQKxNa7Ue\no9s8+eSTxc/uueeeLMdrUq3eM46JOIfVrh2xTivW+aRUjuk4F7q+9L9aHVesEfzOd75TtPn73/+e\n5Vjbd9xxxxV9Vq9eneW5c+dm+fTTTy/6xDls0aJFHV/3c5/7XJbjgeMpuY7tDp3270gppTvvvDPL\n//znP7Mcv/+Uyu8u7stQu3+P9ce1OsOd2Udjb+MvgwAAAA2yGAQAAGiQxSAAAECDBv6DsD2IZ+7V\nzkmLzyDXnlOOzxgPGzYsy5MnTy76nHDCCVmONRELFy4s+sSz3uJnqdVRxGezYz1R7VnoWOfRm/qM\nWn0BOyb+O9e+mzi2PvzhD2d52rRpRZ/4nHzt3Ms49s8777wsz5gxo+gT64PU8XSHOGfVvpfYJn6X\ntd/nON/MnDmzaBPrPOL4rJ3Tatz0v/h9/uEPfyjaxHqwESNGZDleO1Iq6wp7U1sTzxCMNWYpldfU\neK2m/9XudeIeBL/5zW+KNvE8uDjXxNqvlFI6/vjjsxznljgWUypr32t1sHF8xvrFWk1rb84ZZNfE\n3+/777+/aHPNNddkOe5vEc85Timlz3/+81mOeyg8//zzRZ9Ys7xu3bqizYQJE7I8EMeEvwwCAAA0\nyGIQAACgQRaDAAAADbIYBAAAaFBTG8jEos+4acbVV19d9FmwYEGWa4d6x007Tj311CxPmTKl6BML\n7V9//fUs1wpdH3rooSzHQu3Ro0cXfTodDF47YHVnimMHYkFtN4r/zvH7ixvMpJTStm3bsvzqq68W\nbaZPn57lOXPmZDluMFL7LPS/2r95/Fnc+KE3h/NGtY0VYtF/3HAkpfIA36OOOirLLRzeuzd4+eWX\ni5/FTWbi9x2/25TK77PTWEyp3DBm7dq1RZujjz46y7XNa+hftbnmhRdeyHK8b6n1i9eOE088sejz\nta99LcuHHXZYlmsbWsX7obhZTEopnXnmmVkeO3Zslh0wv2fEeWPu3LlFm/idxw2DJk2aVPTpdI//\n0ksvFX3itS5ucNUKfxkEAABokMUgAABAgywGAQAAGtR0AUd8bnnixIlFm/nz52e5VgPR6ZDS3hzI\nHtvU3ic+ex8/f+1A53hYZqcaD/YunWoIUyrHUu07Hz58eJZrNWN0p/h9xu/urbfe2uHXrM1ZW7Zs\nyfKyZcs6vk6nmmV2jzhGZs+e3bFNvFbUrklRHDe1mrIf/vCHPb5vSildeOGFWa7VLNO/ateA888/\nP8tPPfVU0WbJkiVZPuWUU7J8wQUXFH3GjRuX5TgmarVeK1euzPKsWbM6fl41y90hfr+1muCTTjop\ny5MnT+7xNVIqa6HjGK7dI48cOTLLtdrTFrjjAwAAaJDFIAAAQIMsBgEAABpkMQgAANAg1bTv0JvN\nVHZmA4Ta68Zi/O3bt2e5djD4G2+80fF1o4MOOqg3H5EBojYmYkF0bWOAOLZ2ZtMR9oxOG8jUxDZx\nPooHjqeU0jXXXJPleOhzSuX8OHXq1B4/K3tG3NQjpXKzsXgNuu2224o+Rx55ZJbjBjLXXXdd0SeO\nm9o16owzzsiycdMdDj300CwvWLCgaBM3loqbk8XXSKmcN+LB34sXLy76bNiwIctf/epXizZDhw4t\nfkb3qf1+x81epk2bluXNmzcXfe6+++4sr1+/Psu1+5rDDjssywcccECPn3Wg8pdBAACABlkMAgAA\nNMhiEAAAoEFqBrtEfGZ+27ZtRZv4XHWsBZs3b17Rp3aYJwNX7bDwNWvWZLk2tiZNmpRlhzzvvXpT\nQxjHSawRXLFiRdEn1gfVDhSfMWNGlmOdB92hVkv1/e9/P8uXX355ln/6058Wfe69994sb9y4McvP\nPfdc0SfWJt5yyy1Fm2HDhhU/Y8+Lc0usB0wppdmzZ2c5zhOxPr3m0UcfzfIzzzxTtPnUpz6V5Q98\n4ANFm97UT9Od4liLNYRxrkkppaeffjrLsRa+dl8TD50fPHjwDn3OgcJvCgAAQIMsBgEAABpkMQgA\nANAgNYN7SHweOp5/UjvrJJ7HtGnTpizHZ59r7+O8poEtnleZUnnWzuTJk4s28awdtRYDx86cn1qr\n6znkkEM6vu4ll1yS5f322683H5HdrPb7/cUvfjHLM2fOzPKPfvSjos8jjzyS5VhD9slPfrLo8+1v\nfzvL8SzKlFyn9ha9OR9u333z28za3BLrCseOHZvlk08+uehzxBFH9Pi+DCxxzhoxYkTR5rjjjsvy\nXXfdleXaOeHjx4/Pcq0WPt6PD8T5yR0fAABAgywGAQAAGmQxCAAA0CCLQQAAgAbZQGYPiQWocfOX\nM888s+gTN5lZunRpluOBzymVRbfxNWoFtey9aoXNY8aM6dgmHgQdx8VALJhuWfw+4yYPtQOcFy9e\nnOU4l6RUHupr3Ow94u/8rFmzsnzrrbfuzo/DABHHVW3zorj5y5AhQ7IcDw+vva65pi21DYPOOuus\nLK9evTrLDz/8cNEnjq0tW7YUbeLGWANxg72B938EAABARxaDAAAADbIYBAAAaJCawS4Rn3evHag5\nePDgLJ9wwglZfu2114o+GzduzHLtMHsGjlhrkVJ58PPatWuLNvGZ+Dge33777Y7vrWZjYIt1hTED\ndFK7TsTrS6zJGog1Wuya2jiK9z8XXHBBllesWFH0WbJkSZZrh85HA/F+yG8YAABAgywGAQAAGmQx\nCAAA0KB9evPs6x7QlR9qd6p9L2+++WaWa+d8dXqdeA7YHj5nsL8fqm5+HKVUjoHenNm0F+rPsWQc\ntcM4oi8YR/QF42gXxHudWh1f7X4oijWre1s9YOrFOPKXQQAAgAZZDAIAADTIYhAAAKBBFoMAAAAN\nsoEMe5INZOgrCu3pC8YRfcE4oi8YR/QFG8gAAABQshgEAABokMUgAABAg7q1ZhAAAIB+5C+DAAAA\nDbIYBAAAaJDFIAAAQIMsBgEAABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADbIYBAAA\naJDFIAAAQIMsBgEAABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADbIYBAAAaJDFIAAA\nQIMsBgEAABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADbIYBAAAaJDFIAAAQIMsBgEA\nABpkMQgAANAgi0EAAIAGWQwCAAA0yGIQAACgQRaDAAAADfo/VfTrYgTpfBUAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAACsCAYAAADBhXmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAGfpJREFUeJzt3XmsXkX9P/BBoS20BQoFWkprCyKbLAKtLRTqWmQJRsAY\njRUVEkNCSDBCYkwIESkxEMFotSSuEKGJpiwREkxQwIVSxELZSqVggZa20IUubC3y/e8X5jPjvQ/9\n3X1er//eT2ee5+k9c885k3s+M7u8++67CQAAgLZ8oL+/AAAAAH3PZBAAAKBBJoMAAAANMhkEAABo\nkMkgAABAg0wGAQAAGmQyCAAA0CCTQQAAgAaZDAIAADTIZBAAAKBBJoMAAAANMhkEAABokMkgAABA\ng0wGAQAAGmQyCAAA0CCTQQAAgAbt2t9f4H94t7+/AH1il15+f+OoHb05loyjdhhH9ATjiJ5gHNET\nuh1H/jIIAADQIJNBAACABpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDII\nAADQIJNBAACABpkMAgAANGjX/v4CAAw+7777bvHaLrvs0mWb+O8AA1kn5zkY7PxlEAAAoEEmgwAA\nAA0yGQQAAGiQmkGAIaxW8/LOO+9keevWrVletmxZ0eeWW27J8nPPPVe0WbVqVZZHjx6d5UsuuaTo\nc/bZZ2d52LBhRRv6X20cbdmyJcsLFy7M8pNPPln0mThxYpbPOuusbtvsumt+q6Jmi95SG+fvt43x\nyWDjL4MAAAANMhkEAABokMkgAABAg0wGAQAAGmQBmT7w3//+t9vXPvCBfF5eK0BWlDy0xCL0HTt2\nZDku8pFSOQZ22223Lv+9k89NqRx/DF7x+L711ltFmyVLlmT5tttuy/Ljjz9e9Nm4cWOWN2/eXLRZ\nuXJllt9+++0sjxgxoujzuc99LssWkBkY4jh67bXXijbXXHNNlu+8884sb9++veizxx57ZPnee+8t\n2syePTvLc+bMyXJcmCgl10e6F8d07VoYr8O1+7cPfvCDXX5O7d/j+8Zrd0rG8EDQyQJC8d6sdq8W\nj+9Av8ca2N8OAACAXmEyCAAA0CCTQQAAgAapGewB8RnjN998M8srVqwo+jzwwANZjpv3nnbaaUWf\ngw8+OMux9qL2THJ8Bt0z6X0jjonaM+Vr1qzJ8qOPPprluBF4SimNGjUqy8cff3yWazVZ8bNr9TbD\nhw/PsnEyeMXalEWLFhVt5s+fn+VNmzZl+bLLLiv6nHDCCVmu1fbFTcavvvrqLNfGp7E2MMVx9Jvf\n/KZoc88992R57733zvIpp5xS9InXrYceeqho88tf/jLL69aty/Lll1/e7fsaV4NHd7V8tTqueHxr\nbWK9X6yfjuMqpZQWL16c5UMOOaRoM2HChCzHa+q2bduKPnvttVeWW66N7uT4xtd2Zj2ETvrEMfH0\n008XbRYuXJjleN6L92UppfSDH/wgy9OnTy/adFd72pf8ZRAAAKBBJoMAAAANMhkEAABokMkgAABA\ngywg0wPixspz587N8q9+9auiT20D3/f6+c9/Xrz22c9+Nsuf//znszxp0qSiz/jx47Mci5hrBaxx\nIRqF+O9fLGSuFZT//e9/z/Ly5cuzXFvoZfLkyVl+5ZVXslzbCHzZsmVZPvPMM4s2cZwweMSxFsfE\nvHnzij5xAasrrrgiyyeddFLRp5NFho488sgsH3fccVmunfdqmy/T/+KiZvF8lVJ57OLCRIceemjR\nJy7qERdjSCmlK6+8Mss333xzlj/xiU8UfWqv0f/iAmZxgb2UUnr11VezvHbt2izXNn4fN25clmvn\no/i+f/3rX7NcW7wojulp06YVbb7yla902ScupPS/vl+r4jUrLmCWUkpPPPFEluOCO/F6VHvf2Kc2\n9uLn/OhHPyra/Oc//8nyG2+8keXawmhx4cipU6cWbQbSvba/DAIAADTIZBAAAKBBJoMAAAANUjP4\nPsX6wJRSuuqqq7J8/fXXZ7n2nHJ8Nnj33XfPcu0Z6ltvvTXLv//977Nce245bsT7xS9+McsXXnhh\n0WfKlClZtpn9+xd/HrVxE59DP+CAA7Jc2+z2qKOOynIcN/fff3/R59prr83yY489VrT5yU9+UrzG\n4BDrJO64444sxzqZlMoa0XPOOSfLtXqMTn7HY21PrMkZO3Zs0ae24TB9Lx6HeOxiDUxK5fXksMMO\ny3JtHMUxEs9pKaW0zz77ZDnWftXq8D/+8Y9nOV776Bvx+K5atSrLd999d9Fn113zW9H9998/y7X1\nEGKf2n1WHNNLly7N8vPPP1/0mTNnTpbPOOOMok0cW7FmsJN7ppbFOtIXXnihaLNo0aIu+9TWRzji\niCOyPHLkyCwvXry46LNgwYIsv/zyy0WbOKbjsayd5+Jm9rXr3EAaE/4yCAAA0CCTQQAAgAaZDAIA\nADRIzWA3tm/fnuXvfOc7RZsbb7yxyz7x2faUUtp3332zvN9++2V53bp1RZ+4R1d8Rj7uC5VS+Uzy\nTTfdlOXa/nIXXHBBlmNdWkr1Z+K7+tzW1epXjj/++CzHZ8w//OEPF31iLU18Dj2+R0plbeJf/vKX\nLr8rg8uOHTuyfOedd2Z5w4YNRZ+4Z2ncf3Rnf39jbUXcx3TPPfcs+qgZHBjisXvwwQezXKv3PPro\no7Mcx03t2MbPqV3rovi+tfNpvB7WrluuSz2rdnzjNegf//hHlms1Weedd16WDz744CzH/eJSKo9l\nPA+mlNKoUaOyPH369CzX9tydPXt2lmPdWUrd3//QtXhPHPenTalcQyGuh/DMM88UfeLx3rhxY5ZX\nr15d9In367VzRBzncQ/or371q0WfL33pS1mujeGBxIgGAABokMkgAABAg0wGAQAAGmQyCAAA0CAL\nyLxHLGxPKaXbb789y7/4xS+KNnFD8VhcXCuOPfXUU7Mci2Xvu+++os+jjz6a5ddffz3LtcVDYjHs\nG2+8keVnn3226BP/P7XN7Ola/LnHhTRSSmnMmDFd9on/XnufWNhcW3QmqhXaM3jF3/u4YFDt9zdu\nDt5bCyLERa2MvcHj0EMPzfLKlSuLNhMnTuzyPeJG0Sml9OSTT2Z5/vz5RZvHHnssy3EMx4Vram3o\nfbV7prhgVTwfzZo1q+gTr1txsY3a+Sle+2rX2DVr1mT5oIMO6va7xMWJLDrU8+LPtLa4SrwnPvvs\ns7NcO7fE+9v169dn+cUXXyz6/PnPf85yJ5vDx4UXL7300qJPXGRmoI8jfxkEAABokMkgAABAg0wG\nAQAAGtR0zWB8Nri2iWV8Fjg+k5xS+Sxw3Jz3nHPOKfp8+tOfzvLixYuz/JnPfKboM2HChCzHzcPj\n89Eplf/H+Dx83KSz1iZuEFp734H+PHR/22233YrXYo1DrAGtPUffXW1XrW40fk7cuD4lx3Mw27Rp\nU5bjGKgd71gPVqu3iToZE92Nz/322694rZPPpvfF4ztu3Lgs12rfhw8fnuV4fXz44YeLPgsWLMjy\nI488UrSJYyLWCJ5++ulFn4G+qfNQEK8TtRrgVatWZTmefyZPnlz0ifcYcSzW6rhiveK2bduKNnH8\nTZ06Ncu777570ce1b2CI15J4rGpjIraJxzLeY6WU0ubNm7Ncq4ON55Y5c+ZkOd6b1z57oPOXQQAA\ngAaZDAIAADTIZBAAAKBBTdcMvvnmm1n+9re/XbRZvXp1t+8Tn3efPXt2lr/whS8UfeJzynH/m7jH\nSkrls/exRvDBBx8s+sT/Y9ybZf/99y/61J7FjuLz3GrOularpZoyZUqWX3nllSyvW7eu6HPggQd2\n+Tl/+tOfitfiMY9jIqXOjjkD02uvvZblWPNQq/nda6+9euW7xHG09957Z7l2XnCuGBjisYv1nbVa\nr3hei3vK3XrrrUWfFStWZLl2Poq1iPG7xLr8lMo6Q+Oqf8RjN3PmzCzXaoTj3sbxnFW7PsXr2lNP\nPVW0iTWsH/rQh7LcW/ur0vM6qSONbeIYefzxx4s+sY6wdt6Ia3xcdNFFWa5dYwcbvwkAAAANMhkE\nAABokMkgAABAg0wGAQAAGjT4qx7fh1hwunTp0iz/7W9/67ZPzfjx47N8/vnnZ3nkyJFFn1jYGtss\nX7686LNkyZIsv/TSS1muFWbHTTj33XffLI8ePbroE3XyM1Cs37Xaz2fPPffM8tq1a7N83XXXFX1m\nzZqV5Vj8fOONNxZ9tm/f3uXnplRuHhwL6x3fgaG2Ie7ixYuzHBenqi1GNWLEiCzHBRxq55JOCvjj\nIkixz7Rp04o+FnEYGOLxfOutt7JcO05xPO62225Zrm0OHxdxqC2+sGzZsizH66MxM3DFe4x4rxPP\nPSmV4ygu/BLPTymV55r777+/aHPiiSdmubcWzqLvxXuW2mv/+te/slwbI3HsjRo1qmhzww03ZHko\njiNnVAAAgAaZDAIAADTIZBAAAKBBTdUMxueJb7rppizH59Rras+7X3DBBVmeNGlSluOz7Sml9PTT\nT2f5sccey/LDDz9c9ImbucbNemu1F7FmJ9YCjRs3ruhD34h1L3FD+VpdzO23357lWDe6ZcuWok8c\nA7VNnuOGvfGZ+IkTJxZ9Yn0QvS/W36SU0gMPPJDlbdu2Zbl2nGJdYTz31frE8VirX/znP/+Z5Q0b\nNmR5ypQpRR/1qANDPC888cQTWT755JOLPhMmTMhyHBNxk++UUjr66KO77JNSSs8++2yWYz1jbXwa\nR32vVlscawaj2rGLYyDWCNbuoeI9Um1thuOOOy7Lak0Hr3gOqN3HPPTQQ1m+8MILs9zJPf6MGTOK\n1yZPnpzloXiu8ZsBAADQIJNBAACABpkMAgAANKipmsH169dn+Y477shybd+s+GxwbY+2WBfx6quv\nZvmee+4p+qxcuTLL//73v7P88ssvF33233//LE+dOjXLtX0S43PVsW7ykUceKfrMnDkzy3GvQnpH\n3N/m4osvLtr88Y9/zHIcn3EPr5TKGo3ank133313luO4ufTSS4s+Y8aM6fK70PO2bt1avPbCCy9k\nOe4reNpppxV9jjjiiCx3V49cUzvejz76aJd9OtnXlN5Xu9Y9//zzWY71NXGMpFTWYMVzTa2m7KCD\nDspy7XwUP+vQQw/t9rvQ++K4qdV7xmMex0itbi+eS2KujZHjjz8+y7VzS3wtfv++vGb152cPBXGs\nrV69umhz/fXXZznu3VwT729/9rOfFW1q57Ghxl8GAQAAGmQyCAAA0CCTQQAAgAaZDAIAADRoyC4g\nUyuQX7FiRZbjxsu1YuhY5BsXYEkppd/+9rdZjovMxAUeat8vbhhfW/Thox/9aJbjojO1/3PchHz7\n9u1Z3rRpU9Fn2LBhxWv0vjjWaht0f+Mb38hyLKJ//fXXiz5xkaC4QFBKKR1yyCFZPvDAA7McF7eh\nb8Tf6aeeeqpoExf6+NSnPpXlSy65pOgzduzYLMfzWm0BqzgGaovZ3HXXXcVr3fWxsELfq13Hbr75\n5iw/99xzWT7mmGOKPnGD8biwSyebfMfNw1MqF2z45je/mWVjpPfV7ofi7+9rr71WtNlvv/26fN93\n3nmneC0uWPXSSy9lubaI3YgRI7K8YcOGok1ccG327Nndvm8cszsz1mp9avdnnfRrVfx5xXvXTvp0\n8jOP42jixIkdfLuhx18GAQAAGmQyCAAA0CCTQQAAgAYN2ZrBmvgse6zTq9U3xGeOazUvS5cuzXJ8\nBjluFp9SSqeeemqWDz/88CyffvrpRZ/ohhtuyPLGjRuLNnGz1j322CPL48aNK/rEn0uNZ9t7X22j\n01iP+pGPfCTLJ554YtEnjtlam5NOOinLcczGzaRTMgb6Qqyvefzxx7vtE+tIY31gSuXxjOe5Ts4L\nixYtKtrEWsOeqL+h523btq14LdYMxhqdE044oegT6wg7qRGMdWaxlj+lcpP5MWPGdPu+9Ky4pkJK\nKX3/+9/Pcu18dMABB2Q53nfVNpCP6yoceeSRWT733HOLPvHceMUVVxRt4vc744wzsvy9732v6BPr\n5WvX4TjO4/mz9nvQwsblO6tWnxrPP3Hc1O5JYl17vN7UaghHjx6d5VaPk78MAgAANMhkEAAAoEEm\ngwAAAA1qqmZw5MiRWZ41a1aW77333qJP3P+mtgdffHY57uN20UUXFX2OPfbYLHeyP9P69euz/NBD\nD2W59ix+fBY71oJNnTq16BNrg9T5DBzxWMRamhdffLHoE8dFrE9NqRwXcZwbA/0j1sWsWbOmaBPP\nUePHj89yrQY4nl/iflux7rkm1nWlVO55GOsxauc1Y6vvPf/888Vr8foSzwETJkwo+sQa9Hgsa3vK\nzZ8/P8u1vW5nzJiR5VbrePpSrKeK9xcppbRgwYIsr1u3rmgTj3kcE3FvypTKOva4x+5BBx1U9In1\nybX1HOJ17cwzz8xybUzH+7lOzk+xjXPa+1P7ecVxFPdQrtU9x+Mdr321vQq7q/9shb8MAgAANMhk\nEAAAoEEmgwAAAA0yGQQAAGjQkF1AplaQGotL582bl+WFCxcWfR555JEs1zZjnj59epY/9rGPddun\nu43da0Wssej/ueeey3InxbFxQ+pp06YVfRTrDx7x+O6zzz5Fm1iIXVscJI7HTjaPpvfF38VYRJ9S\nuYDMxo0bs7xjx45u3zeqHf94Tqqdw+LYiguMdLIwDb1v2bJlxWtxnMSFNA477LCiT23j5/fasmVL\n8drvfve7LNcWmfnWt76VZQty9L2jjz66eO2oo47Kcm3RlnheiJvOf/3rXy/6nHXWWVmOC7vE80hK\n5Tls7ty5RZs4PuP9Tm38Gmt9r/Yzj9eKuGBMbRGseE8cx2Jtc/tJkyZl2QIyAAAANMNkEAAAoEEm\ngwAAAA0asjWDNfH58Lg5c21z+FjPUHueOD673skGpPG1+L61+r+bbropy3Gz3tp3i5tJn3/++Vmu\nPYvP4BFru+KYTqncwLfWRo3gwBTPLbV6mxdeeCHLzzzzTJaXL19e9Dn88MOz3F3tV0plvUXtfd9+\n++0sx3rpPffcs+ijRqfvjRo1qngtXj/i2Iu1X7U+cYzcfPPNRZ9Vq1ZluVabVtsMnN4Vfw9rax3c\ndtttWY73ICmV54ADDjggy8OGDSv6dLfxd+3eZvTo0Vk+4YQTijbxs3ZmQ3n6RxwT8VjWagZjXWE8\nh8X74ZRSOu6447KsZhAAAIBmmAwCAAA0yGQQAACgQSaDAAAADWpqAZkoFg/XNmKOr9WKS7tbDKZW\npNxd4f3mzZuLPosXL+6yT+1zpk6dmuUjjzwyyzaYH9ziMT/iiCOKNhs2bMiyjb8Hj3h8axt/X375\n5Vn+6U9/muUf//jHRZ9TTz01y/E8UVtkaNGiRVmOG4OnVG4yHsdjbQEZ+t7JJ59cvDZmzJgsxwUc\nVqxYUfSJ42TlypVZ/vWvf130iRtBX3vttUUbC1r1v9oxiAsPdbIQ0c6I7xEX8kup/H7Dhw8v2lgw\nZuiIi79Mnjy5aFNb0Oi9OhnTtY3pW+CMCwAA0CCTQQAAgAaZDAIAADSo6ZrBndHJM+edtOluU9Ud\nO3Z02yfWXtQ2Bb744ouzPHbs2G6/G4PX1q1bi9diLUWt/oLBoVbjGzftnjFjRpa/+93vFn0WLFiQ\n5ThGarV9cUPfWl3zpEmTsnzllVdmWY3ywBA37E4ppS9/+ctZvuuuu7J82WWXFX1OOeWULC9ZsiTL\nGzduLPpcddVVWZ41a1bXX5ZBpTfq8jo5b6gHHNri/W5cCyGllF555ZUsx/q/2jiKm9m3eo3yl0EA\nAIAGmQwCAAA0yGQQAACgQWoG+0l8vj3ufxKfY06p3CNwzZo1Wf7kJz9Z9In7SbX6PPRQ9fbbb2f5\nvvvuK9q89dZbWd6+fXtvfiX6WNxL64ILLsjymWeeWfS5+uqrs/zSSy9ledWqVUWfeP6p7Xk4Z86c\nLB944IFZVtczMMQxk1JKP/zhD7tsc/fddxd9/vCHP2R5woQJWb7mmmuKPueee26W7SlI1Ml5opNx\n43wzdHSyp3JcVyPe69Tuq2MtfNwrN6WU9tlnny6/y1DgLAwAANAgk0EAAIAGmQwCAAA0yGQQAACg\nQRaQ6SfdLSAzcuTIos/Xvva1LMeNg+NiMSmlNGLEiJ39igwCcXGYHTt2FG3i5qxvvvlm0WYoFkS3\nKp5L4iIuKaU0b968vvo6DBKjRo3K8nXXXZfluXPnFn3iAla77757lmsL1Vgwhverdn2Ki+HFxUP+\nVz8Gp3gsjz322KLNeeedl+Vbbrkly/GeOaWUVq9eneW1a9cWbWK/2kI0g52zMgAAQINMBgEAABpk\nMggAANCgXWrPWQ8AA/JL9aZ4HGq1X+vWrcty3FAzbp6ZUkoTJ07McqwL6ef6jd5+oH/Ij6N33nkn\ny7UNU5cvX57lY445pmgzfPjwLA/CWove/MJDfhzx/xhH9ATjqJftzL2r61pmyI2jzZs3ZznWxo8f\nP77oE9fnmDlzZtFm7NixWd5113y5lUEwrrr9gv4yCAAA0CCTQQAAgAaZDAIAADRIzeAAVTsu8bVO\nnlPurk8/P+usZrCHdfL7PAieb98ZaivoCcYRPcE46mXxWtfJPoOD8NpnHP1/2Jl75iFKzSAAAAAl\nk0EAAIAGmQwCAAA0yGQQAACgQRaQoT9ZQIaeotCenmAc0ROMI3qCcURPsIAMAAAAJZNBAACABpkM\nAgAANGig1gwCAADQi/xlEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACA\nBpkMAgAANMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAA\nNMhkEAAAoEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAANMhkEAAA\noEEmgwAAAA0yGQQAAGiQySAAAECDTAYBAAAaZDIIAADQIJNBAACABpkMAgAANMhkEAAAoEH/BwQZ\nrYP7gh0hAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "n_iterations = 3\n", + "n_digits = 6\n", + "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n", + "\n", + "with tf.Session() as sess:\n", + " saver.restore(sess, \"my_model_variational.ckpt\")\n", + " target_codings = np.roll(codings_rnd, -1, axis=0)\n", + " for iteration in range(n_iterations + 1):\n", + " codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n", + " outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n", + " plt.figure(figsize=(11, 1.5*n_iterations))\n", + " for digit_index in range(n_digits):\n", + " plt.subplot(1, n_digits, digit_index + 1)\n", + " plot_image(outputs_val[digit_index])\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Exercise solutions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Coming soon..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + }, + "nav_menu": { + "height": "381px", + "width": "453px" + }, + "toc": { + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 6, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/index.ipynb b/index.ipynb index a1c72fecc..5525d783d 100644 --- a/index.ipynb +++ b/index.ipynb @@ -37,7 +37,7 @@ "12. [Distributed TensorFlow](12_distributed_tensorflow.ipynb)\n", "13. [Convolutional Neural Networks](13_convolutional_neural_networks.ipynb)\n", "14. [Recurrent Neural Networks](14_recurrent_neural_networks.ipynb)\n", - "15. Autoencoders (coming soon)\n", + "15. [Autoencoders](15_autoencoders.ipynb)\n", "16. Reinforcement Learning (coming soon)\n", "\n", "## Scientific Python tutorials\n",