TensorFlow_basics_01_linear_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.13.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from GeneralTools.misc_fun import FLAGS\n",
    "\n",
    "FLAGS.TENSORFLOW_VERSION\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 9)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Dataset provided by:\n",
    "Mohan S Acharya, Asfia Armaan, Aneeta S Antony. \n",
    "A Comparison of Regression Models for Prediction of Graduate Admissions, \n",
    "IEEE International Conference on Computational Intelligence in Data Science 2019.\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "import os.path\n",
    "\n",
    "# csv file can also be read using pandas\n",
    "data_file = np.genfromtxt(\n",
    "    os.path.join(FLAGS.DEFAULT_IN, 'graduate_admission/Admission_Predict_Ver1.1.csv'), \n",
    "    delimiter=',', skip_header=1)  # the first row is header, which we skip\n",
    "data_file.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[337.   118.     4.     4.5    4.5    9.65   1.  ]\n [324.   107.     4.     4.     4.5    8.87   1.  ]\n [316.   104.     3.     3.     3.5    8.     1.  ]\n [322.   110.     3.     3.5    2.5    8.67   1.  ]]\n[0.92 0.76 0.72 0.8 ]\n(500,)\n(500, 1)\n"
     ]
    }
   ],
   "source": [
    "# The first row is header, the first column is index, the last column is label\n",
    "x_np = data_file[:, 1:8]\n",
    "y_np = data_file[:, 8]\n",
    "print(x_np[0:4])\n",
    "print(y_np[0:4])\n",
    "print(y_np.shape)\n",
    "y_np = np.expand_dims(y_np, axis=1)\n",
    "print(y_np.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The decay step is 6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1: training loss is 14.6889; test loss is 939.3680\n"
     ]
    },
    {
     "ename": "AssertionError",
     "evalue": "Model diverged with loss = nan at step 43",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-df8551492a82>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m    126\u001b[0m         \u001b[1;31m# check if model produces nan outcome\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    127\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misnan\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 128\u001b[1;33m             \u001b[1;34m'Model diverged with loss = {} at step {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglobal_step_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m         \u001b[1;31m# add summary and print loss every query step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAssertionError\u001b[0m: Model diverged with loss = nan at step 43"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# simple linear regression\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "\"\"\"\n",
    "tf.Graph() initializes a new graph.\n",
    "some_graph.as_default() sets the some_graph as the graph to work on.\n",
    "with some_graph defines a scope. All TensorFlow operations should be under the same graph/scope. \n",
    "\"\"\"\n",
    "with tf.Graph().as_default(), tf.device('cpu:0'):\n",
    "    # Workflow step 1. prepare data\n",
    "    # tf.constant is one way to add the data to graph\n",
    "    data = tf.constant(data_file[:, 1:], dtype=tf.float32)\n",
    "    # shuffle the dataset\n",
    "    data_shuffle = tf.random.shuffle(data, name='data_shuffle')\n",
    "    # divide the dataset into training and test datasets\n",
    "    data_tr = data_shuffle[0:400]  # 400-by-8, print(data_tr.get_shape().as_list()) -> [400, 8]\n",
    "    data_te = data_shuffle[400:500]  # 100-by-8\n",
    "    # scale the data\n",
    "    mean_tr, var_tr = tf.nn.moments(data_tr, axes=[0], keep_dims=True)\n",
    "    std_tr = tf.sqrt(var_tr + FLAGS.EPSI)  # print(mean_tr.get_shape().as_list()) -> [1, 8]\n",
    "    data_tr = (data_tr - mean_tr) / std_tr\n",
    "    data_te = (data_te - mean_tr) / std_tr\n",
    "    # separate the features from targets\n",
    "    x_tr = data_tr[:, :7]\n",
    "    t_tr = tf.expand_dims(data_tr[:, 7], axis=1)\n",
    "    x_te = data_te[:, :7]\n",
    "    t_te = tf.expand_dims(data_te[:, 7], axis=1)\n",
    "    \n",
    "    # Workflow step 2. define the model\n",
    "    \"\"\"\n",
    "    There are at least three ways to initialize trainable parameters:\n",
    "    Using low level API:\n",
    "        1. tf.Variable always initializes a new tensor when called;\n",
    "        2. tf.get_variable initializes a new tensor when first called, can recall a previously \n",
    "        initialized tensor when called again (if variable scope is set properly);\n",
    "    Using high level API\n",
    "        3. tf.layer.* implicitly initializes trainable parameters using tf.get_variable.\n",
    "    \n",
    "    when trainable=True, the parameters are added to GraphKeys.TRAINABLE_VARIABLES and can be \n",
    "    returned using tf.trainable_variables() or tf.get_collection().\n",
    "    \"\"\"\n",
    "    with tf.variable_scope('LinearModel0', reuse=tf.AUTO_REUSE):\n",
    "        w0 = tf.get_variable(\n",
    "            'weight', shape=(x_np.shape[1], y_np.shape[1]), dtype=tf.float32, \n",
    "            initializer=tf.random_normal_initializer, trainable=True)\n",
    "        b0 = tf.get_variable(  # optional if we standardize the inputs and outputs in a linear model\n",
    "            'bias', shape=[], dtype=tf.float32, \n",
    "            initializer=tf.random_normal_initializer, trainable=True)\n",
    "        # print(tf.trainable_variables()) -> [<tf.Variable 'weight:0' shape=(7, 1) dtype=float32_ref>]\n",
    "        y_tr = tf.matmul(x_tr, w0) + b0\n",
    "        y_te = tf.matmul(x_te, w0) + b0\n",
    "    with tf.name_scope('LinearModel1'):\n",
    "        pass\n",
    "    \n",
    "    # Workflow step 3. define the loss function\n",
    "    \"\"\"\n",
    "    tf.losses contains several standard loss functions like MSE, MAE, cross-entropy, hinge\n",
    "    User-defined loss can be added.\n",
    "    The loss function constructed will by default be added to tf.GraphKeys.LOSSES\n",
    "    https://www.tensorflow.org/api_docs/python/tf/losses\n",
    "    \"\"\"\n",
    "    loss = tf.losses.mean_squared_error(\n",
    "        t_tr, y_tr, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, scope='train_loss')\n",
    "    loss_te = tf.losses.mean_squared_error(\n",
    "        t_te, y_te, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, scope='test_loss')\n",
    "    \n",
    "    # Workflow step 4. configure the optimizer\n",
    "    # configure learning rate\n",
    "    global_step = tf.get_variable(\n",
    "        name='global_step', shape=[], dtype=tf.int32, \n",
    "        initializer=tf.constant_initializer(0), trainable=False)\n",
    "    # decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)\n",
    "    init_learning_rate = 1.0\n",
    "    decay_steps = np.round(1000 * np.log(0.96) / np.log(0.001 / init_learning_rate)).astype(np.int32)\n",
    "    print('The decay step is {}'.format(decay_steps))\n",
    "    learning_rate = tf.train.exponential_decay(  # adaptive learning rate\n",
    "            0.5,\n",
    "            global_step=global_step,\n",
    "            decay_steps=decay_steps,\n",
    "            decay_rate=0.96,\n",
    "            staircase=False, name='decay_lr')\n",
    "    # configure optimizer\n",
    "    # see other optimizers at: https://www.tensorflow.org/api_docs/python/tf/train\n",
    "    # include: MomentumOptimizer, AdamOptimizer, RMSPropOptimizer\n",
    "    opt = tf.train.GradientDescentOptimizer(\n",
    "        learning_rate=learning_rate, name='GD')\n",
    "    # configure optimization process\n",
    "    # the following can also be done as \n",
    "    # opt_op = opt.minimize(loss, global_step=global_step)\n",
    "    var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=None)\n",
    "    grads_list = opt.compute_gradients(loss, var_list)\n",
    "    opt_op = opt.apply_gradients(grads_list, global_step=global_step)\n",
    "    \n",
    "    # Workflow step 5. configure the summary\n",
    "    # summary op is always pinned to CPU\n",
    "    tf.summary.histogram('weights', w0)\n",
    "    tf.summary.scalar('bias', b0)  # normally bias would be a vector, so histogram too\n",
    "    tf.summary.scalar('loss/train', loss)\n",
    "    tf.summary.scalar('loss/test', loss_te)\n",
    "    for grads in grads_list:\n",
    "        var_grad = grads[0]\n",
    "        var = grads[1]\n",
    "        var_name = var.name.replace(':', '_')\n",
    "        tf.summary.histogram('grad_' + var_name, var_grad)\n",
    "        tf.summary.histogram(var_name, var)\n",
    "    summary_op = tf.summary.merge_all()\n",
    "    summary_folder = os.path.join(FLAGS.DEFAULT_OUT, 'graduate_admission' + '_log')\n",
    "    if not os.path.exists(summary_folder):\n",
    "        os.makedirs(summary_folder)\n",
    "    \n",
    "    # Workflow step 6. call a session\n",
    "    sess = tf.Session(config=tf.ConfigProto(\n",
    "        allow_soft_placement=True,\n",
    "        log_device_placement=False))\n",
    "    # initialize the graph\n",
    "    init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())\n",
    "    sess.run(init_op)\n",
    "    # configure where to write the summary file\n",
    "    summary_writer = tf.summary.FileWriter(summary_folder, sess.graph)\n",
    "    # run the session\n",
    "    start_time = time.time()\n",
    "    max_iter = 1000\n",
    "    for step in range(max_iter):\n",
    "        loss_value, _, global_step_value = sess.run([loss, opt_op, global_step])\n",
    "        # check if model produces nan outcome\n",
    "        assert not np.isnan(loss_value), \\\n",
    "            'Model diverged with loss = {} at step {}'.format(loss_value, global_step_value)\n",
    "        \n",
    "        # add summary and print loss every query step\n",
    "        if global_step_value % 50 == 1 or global_step_value == max_iter:\n",
    "            loss_te_value = sess.run(loss_te)\n",
    "            if summary_op is not None:\n",
    "                summary_str = sess.run(summary_op)\n",
    "                summary_writer.add_summary(summary_str, global_step=global_step_value)\n",
    "            print('Step {}: training loss is {:.4f}; test loss is {:.4f}'.format(\n",
    "                global_step_value, loss_value, loss_te_value))\n",
    "            \n",
    "    duration = time.time() - start_time\n",
    "    print('Training for {} steps took {:.3f} sec.'.format(max_iter, duration))\n",
    "    sess.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAGMCAYAAAAcHTz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XuQZGWZ5/Hf00UDBd3SIO0ArU2zutuoDEND4e7aE+MADu0VWxg1Rmd3vW0TruOKE/bYrbMEwii10+FtYEYlYkJlFMMbljCgTYQt4waLl2oLRBx6HEcuWwg2SHPRAvry7B/nFJ1d5OVknpPnfd+T309ERlaePFX91CGpfPJ9n/d5zd0FAACAwS0KHQAAAEDqSKgAAABKIqECAAAoiYQKAACgJBIqAACAkkioAAAASiKhAgAAKImECgAAoCQSKgAAgJIOCh1AVY4++mhftWpV6DAAAEBDbN++/QF3X17k3MYkVKtWrdL09HToMAAAQEOY2V1Fz2XKDwAAoCQSKgAAgJJIqAAAAEoioQIAACiJhAoAAKAkEioAAICSSKgAAABKIqECAAAoKaqEysyWmNk9Zub57c2hYwIAAOglqoRK0l9JenboIAAAAPoRTUJlZqdK+jNJ3w8dCwAAQD+iSKjMbJGkT+cP3xEyFgAAgH5FkVBJepekCUmfdPeZ0MEAAAD0I3hCZWYrJF0i6X5Jfxk4HAAAEKsff1n62EnSRcuy+x9/OXRETzkodACSLpO0VNL/cPeHQwcDAAAi9OMvS9f+T2n3XPb44Xuyx5J08uvDxZULOkJlZq+W9FpJN7r750PGAgAAIvbti/cnU/N2z2XHIxAsoTKzwyVdLmm3pHcO+DM2mNm0mU3v3Lmz0vgAAEBEHv5//R2vWcgRqoslrZT0MXf/6SA/wN2vcPcJd59Yvnx5tdEBAIB4HNGhTWWn4zULklCZ2SmS3i3pHmWJFQAAQGdnXSgtHj/w2OLx7HgEQhWlf0LSmKQPSDIzW9LhvEPy5/a5+29riw4AAMRlvvD82xdn03xHPDtLpiIoSJckc/f6/1GzOyUd38e33OXuq7qdMDEx4dPT02XCAgAAeIqZbXf3iSLnBu9DBQAAkLogCZW7r3J363RrOfUt+bFVIeIEAAAoghEqAACAkkioAAAASiKhAgAAKCmGvfyeZkEdFQAAQNQYoQIAACiJhAoAAKAkEioAAICSSKgAAABKIqECAAAoiYQKAACgJBIqAACAkkioAAAASiKhAgAAKImECgAAoCQSKgAAgJJIqAAAAEoioQIAACiJhAoAAKAkEioAAICSSKgAAABKIqECAAAoiYQKAACgJBIqAACAkkioAAAASiKhAgAAKImECgAAoCQSKgAAgJJIqAAAAEoioQIAACiJhAoAAKAkEioAAICSDgr1D5vZhKTXSDpd0vMkLZd0qKQHJE1L+oy7T4WKDwAAoKhgCZWkt0s6v+XxY5L2STpO0jmSzjGzr0n6E3ffHSA+AACAQkJO+d0s6T2STpO01N2Xuvu4pJWStuTnnCdpU6D4AAAACgk2QuXun+tw/B5Jf2Fmx0r6U0lvlnRJjaEBAAD0Jeai9B/m98cFjQIAAKCHmBOqF+f3vwgaBQAAQA8hi9KfxsyWSPp3yorV35AfvjxcRAAAAL0FT6jM7NmS7mnz1OOSPuzuf1dzSAAAAH2JYcpvr6T789uT+bE9ki5Vj9EpM9tgZtNmNr1z587hRgkAANBB8ITK3X/p7se4+zGSxiWtlnSlpA9KusXMXtjle69w9wl3n1i+fHlNEQMAABwoeELVyt33ufu/uPvbJH1UWU+qz5tZVHECAAC0ijlRuSy/P0XSmpCBAAAAdBNzQjXb8vVzg0UBAADQQ8wJ1QktXz8WLAoAAIAegiRUZjZmZtbjtI35/R5l+/4BAABEKdQI1XMkTZvZW/M+VJIkM1tkZqeY2RckvT0/fJm7PxQkSgAAgAJCNvY8VdLfS5KZPa5sWm+ppENazvmspL+oPTIAAIA+hEqo7lW2tcxZkl4k6VhJz1TWHf3nyqb4PuPuNwWKDwAAoLAgCZW7Pynpy/kNAAAgaTGv8gMAAEgCCRUAAEBJJFQAAAAlkVABAACUREIFAABQEgkVAABASSRUAAAAJZFQAQAAlERCBQAAUBIJFQAAQEkkVAAAACWRUAEAAJQUZHNk1GtqZlZbtu7QvbvmdNyycW1ct1rr16wIHRYAAI1BQtVwUzOz2nz1bZrbvVeSNLtrTpuvvk2SSKoAAKgIU34Nt2XrjqeSqXlzu/dqy9YdgSICAKB5SKga7t5dc30dBwAA/SOharjjlo33dRwAAPSPhKrhNq5brfHFYwccG188po3rVgeKCACA5qEoveHmC89Z5QcAwPCQUI2A9WtWkEABADBETPkBAACUREIFAABQEgkVAABASSRUAAAAJZFQAQAAlERCBQAAUBIJFQAAQEnBEiozW2lmF5jZtWZ2t5k9YWaPmtmtZjZpZseGig0AAKAfQRp7mtlzJN0pyVoOPyLpcEkn57cNZnaeu3+n/ggBAACKCzVCNb+53HWSXifpKHc/QtJhkl4h6ReSjpQ0ZWbHhAkRAACgmFAJ1UOS1rj7q9z9q+7+kCS5+5Pu/k1lSdXjkp4h6fxAMQIAABQSZMrP3R+WdGuX5+8ws+9J+kNJp9UVFwDEYmpmlk3NgYTEvDnyg/n9WNezAKBhpmZmtfnq2zS3e68kaXbXnDZffZskkVQBkYqybYKZHSRpbf7wJyFjAYC6bdm646lkat7c7r3asnVHoIgA9BJlQiXpnZKOkbRP0pWdTjKzDWY2bWbTO3furC04ABime3fN9XUcQHjRJVRmdrKkD+cPL3f32zud6+5XuPuEu08sX768ngABYMiOWzbe13EA4UWVUOXNPKeUtU/YLul9YSMCgPptXLda44sPLB8dXzymjetWB4oIQC/RFKWb2VGSbpB0gqSfSXqluz8eNioAqN984Tmr/IB0RJFQmdkRkrZKOknS3ZJe6u73h40KAMJZv2YFCRSQkOBTfmZ2uKTrJU1Iuk9ZMnV32KgAAACKCzpCZWbjkq6V9GJlfade6u4/CxkTwqGRIQAgVcESKjM7WNLVks6QtEvS2d1W9KHZaGQIAEhZkCk/MxuTdJWkl0l6VNLL3f1HIWJBHGhkCABIWagRqrWSzsu/Xixpysw6nXuPu59eS1QIhkaGAICUhUqoWkfGDs1vndA6YQQct2xcs22SJxoZAgBSEGTKz91vdHcreFsVIkbUi0aGAICURdGHCqCR4WhjhSeA1JFQIRo0MmyWokkSKzwBNEHwxp4Ammc+SZrdNSfX/iRpamb2aeeywhNAE5BQAahcP0lSp5Wcs7vmdMKm67R2clvbRAwAYsKUH4CnKVvT1E8bjE4rPCUdMLolMQUIIF6MUAE4QD/TdZ10anfR7ni7FZ4LMQUIIHYkVAAOUEVNUz9tMNavWaFLz/1drVg2ro7tfUWTVwBxY8oPwAGq6FrfbxuM1hWeaye30eQVQHJIqIAKNKmPUlVd6wdtg7Fx3eoD2ihINHkFED+m/ICSqqg5iknorvULpwBXLBvXpef+brIJKoDRwAgVUFK3mqMUk4AYutbT5BVAakiogJKqqDmKzbATmiZNkQKAxJQfUFo/LQLQvClSAJBIqIDSQtccpSa1rWamZma1dnIbXdsBdMWUH1BSDDVHKUlpipSNmwEURUIFVIAi6uKqastQh6YtOAAwPEz5ITlMwaQtpSnSlEbTAITFCBWS0m4K5j1fukUXfOkWrWCqrRZlV+ilNEWa0mgagLBIqJCUdlMwnt83tb4lphYDVdUUpTJFStd2AEUx5Yek9JpqiXm12CBiazGQ2gq9sujaDqAoRqiQlE5TMK2aVN8SW1H0KNYUpTKaBiAsRqiQlHYFzQs1qb4ltgSGJqYA0B4JFaJQdOVe6xSMJNmC55tW3xJbApPSCj0AqBNTfgiu30Ln1imYmAq2hyG2ouiUVugBQJ3M3XuflYCJiQmfnp4OHQYGsHZyW9u6qBXLxnXTpjMDRBSXpieNvYz67w8gHDPb7u4TRc5lhArBxVYnFJtRLopm6xcAqQhWQ2VmS83sHDO7xMy+aWYPmJnntxNDxYX6xVYnhHiMWpuGYWF3AWD4Qo5QnSXp6wH//UZKcXoktjohxIPRy/IY5QPqEXqV368kXS/pg5I2BI4lebE1gSyK5onohNHL8hjlA+oRcoTqWnefmn9gZqvChdIMsTWB7Mco1wkVkeLIYxUYvSyPUT6gHsESKnff2/ss9IM/nM00ylM2tGkojw2egXqwyq9B+MPZTCmPPFaB0ctyGOUD6hG6hgoVoot1MzHyiDKoUQTqwQhVgzA90kyMPKIsRvmA4Us6oTKzDcpXB65cuTJwNHFI6Q/nqBZa96vpUza8DgA0QdIJlbtfIekKKdt6JnA46MMoF1r3q8kjj7wOADRF0gkV0jXqhdb9KjrymNpoD68DAE1BQoUgKLSuXoqjPbwOUFRqHxYweljlhyDogF29i665PbmO2LwOUESqu0BgtJBQIYhRa/Ew7M1pp2ZmtWtud9vn2q0QjMWovQ4wGLbPQQqY8kMQKRda9zv1UMdUXLc3FstjiPHapvw6QH2YGkYKgiZUZnZ0y8MjW75etuC5X7v7vprCQk1SavEwr1dy1C7ZqqPwutsbi0tRF3mn+DpAvejFhhSEHqHa2eH4zQsenyDpzuGGAvTWa+qhXbK18Px5VX667vSGM4x/C6hb03uxoRmooQL60G3qoVOyNWbW9nuq/HTdrhZpWP8WUDe2z0EKgo5QuXv7dxogUt2mHjolW3vdNb54bKifruffWD547e166LcHFqfzSR5NwNQwYscIFdCHbqvSOo0CzX+aHvan6/VrVmjmwrP18Tecwid5AKhZ6BoqICndVqVN3/Vrff57dz/te844cXmtn675JA8A9SOhAvrUKWH5zh3t11h0Og4AaA6m/ICK0CsHAEYXI1RAReiVM1rYWw5AKxIqoCL0yqnWsBKWKn5uihtRAxgupvyAitArpzrD2gy3qp/L3nIAFmKECqgQK+yqMazteqr6udTLAViIESoA0RlWwlLVz+1UF0e9HDC6SKgQvamZWa2d3KYTNl2ntZPbSk/7IH7DSliq+rndGrwCGE0kVIjasGpp+o0htoQuxpiqNKyEpaqfS70cgIXM3UPHUImJiQmfnp4OHQYqtnZyW9tWBCuWjeumTWcO/d+fmpnVxq/eqt179/9/snjMtOWPfy/Ym+fCFWZSlhQ07Q095lV+AEaDmW1394lC55JQIWYnbLpO7V6hJukXk68c+r+/5uIbnrbZsCQdedhizVx4dtvvGfYbdugks+liTrhijg1oon4SKlb5IWqhm2W2S6a6Ha+jPxErzIYn5v5SMccGgBoqRC614t86+hOxwmx4Yu4vFXNsAEioELnQxb/Lxhf3dbyO0aPUksyUxDz6F3NsAJjyQwJCNsu86JwXauNXbtXufS1F6YtMF53zwrbn1zFFOX8tqKWpXugp5m5ijg0ACRXQVb/JS137+dGRfThi3o8x5tgAkFABPfWTvKQ2esSqsQPF/N8v5tgA0DYBiE5dSc6o9LMCgEHRNgFIVJ1L44e1AXEM+klKGaUDUAVW+QERqXJpfK/taZq6aqzddkUXfOkWrbn4hqddgxi2NgLQDCRUQESqSnKKJApN7WfVLimVsmasC68BvZ0AVIWECrVo+ma+VakqySmSKDS1n1W35HPhNWjqKB2A+pFQYeiYVimuqiSnSKLQq2lqqklwr+Sz9Ro0dZQOQP0oSsfQNbn4uWpVLY0v2gSyU0uIlPeNa9evqVXrNaC3E4CqkFBh6GKbVol9VVcVTTvLJgqdkuALvnSLtmzdEd01azUf10XX3K5dcwduYr3wGtDbCUBVgidUZnaMpM2SXiVphaSHJf1A0sfd/dshY0M1YtoyI+WRl36UTRS6JbspXLP5pLRI8kzXeQBVCNrY08xOlrRN0jPzQ49IWqKstsslvd/dJ4v8LBp71q/oSE9MDSTXTm5rm9ytWDaumzadWWssMet0nVrVec1iH1UE0Ez9NPYMVpRuZuOSrlGWTM1IOsndj5B0pKSPSDJJl5rZ2aFiRGf9FJr3Kn6uU2zTjzGampnVb57Y0/O8uq4ZixoApCDklN/5ko6X9JikV7v7rCS5+yOS3mtmz5W0XtKlkm4IFiXa6rfQPJZplZimH2PUbjSxk7quGYsaGKEDUhCybcKb8vur5pOpBbbk96ea2Yk1xYSCUh3pGWbvpVTbDLTq1BTTFjyucyVcqq+1qjBCB6QhSEJlZkslnZY/3NrhtO8pK1CXJIpbIpNq/55hTT825U2vU5LiUrAp2ypfaykmvXRzB9IQasrv+dr/off2die4+z4z2yHpRZJeUFdgKCbl/j3DmH5MYVqqyLRRpynRkEX7RV5rRX63VFd4jvoIHZCKUFN+x7Z8fW+X8+afO7bLOQggpkLzGMT+pld0BC3G7WiKdHQv8rulOtKT6mgwMGpCjVAd3vJ1t3ec3+b3S9o9aWYbJG2QpJUrV1YTGQqLpdA8BrEXuxcdQYu10WW311rR3y32pLeTlEeDgVESKqFaWOM6EHe/QtIVUtaHqoqfCQwi9je9fpKJ1BLlor9b7ElvJ7EmuQAOFCqheqzl63FJj3Y477A25wPRif1NL9Vkooiiv1vsSW83qSW5wCgKlVC11k0dJ6lTEcNx+f0vhxsOUF63N70yfYSq6EGUcjLRS9HfLfakN1W9Xp/00MKoCJVQ3aFsJbZJeqHaJFRmtkjS/F/En9YXGlBe65vIssMW67HH92j3vmxWup/VZVWtTGtyMtHP78ZIT7V6vT5TXVkJDCLYXn5m9gNJp0v6lLu/o83z/1nS/80fnujuXZfisJdfM6X46bZot/EirQjYexAx6/X65PWL1CWxl5+kq/L7N5lZu7YI783vt/dKptBMqTbL7NRtfKEiq8tSXZmG0dDr9cnrF6Mk5F5+n5Z0gbL9/P7RzP6Lu/8076L+vySdm5/3/lABolpFR5vmz2v3yTa2ZpntFH2zKFIQ3qng+ojxxVo7uS2ZkbsURxrRW68FAU1eDAEsFGyEyt3nJL1G0oOSTpV0u5k9LGmXpI3Kaqw2uzsbIzdA0dGm1vM6if3TbZE3i3advtttidKu0ebiRabfPLknmZG7VEca0VuvRrAxNooFhiXklJ/c/VZJJ0n6G0n/JukQZQnWdZL+yN0nA4aHChXtUl1kuiz2T7dtk6Ax07LxxX13+m7XJXzJoQdp994Dax9j7vidaody9Nariz07KmCUhJzykyS5+32S3p3f0FBFayl6jT6l8Om23xV1vTp9L1yZdsKm69r+nFhH7mKoo2HKcXh6rZxkZSVGRfCECqOhaC1Fp/Ok7NNtKm+E/byJ9JtwpFaXEjpelu4DqEPQKT+MjqK1FJ3O+/gbTtFNm85s5Btgv5vfplaXEjpephwB1IGECrUoWksxijUX/SYcqV2j0PHGMOUIoPmCNfasGo09kTJqfIaH5pIABtVPY09qqIAa9EqYQhTu1p3EhUoam7yPIYB4kFABQxZjUXTdMYW8Bk3ex3BYGDEF+seUHzBkMU451R1TjNcA7bXbi3J88VjUdXrAsKSylx8wEmIsiq47phivAdpjVSQwGBIqYMj6bYtQh7pjivEaoD2SX2AwJFTAkIXuw9RO3TGdceJy2YJjoa8B2iP5BQZDQgUMWeg+TKFjmpqZ1de2z6q1WtMknXcaW5LEKMYPAEAKWOUH1CDG/czqiqldTY5L+s4dO4f+b6N/rIoEBkNCBWCoqMlJT4wfAIDYkVABFaBvT2ehN0dGZ7xugepQQwWUNN+3Z3bXnFz7m1ZOzcyGDi0K1OTEidctUC0SKqAk+vZ0F2NRPnjdAlVjyg8dMR1QDDVCvVGTEx9et0C1GKFCW0wHFEffHqSI1y1QLRIqtMV0QHHUCCFFvG6BajHlh7aYDiiOvj1IEa9boFokVGiLpe79GWaNELVsGBZq24DqMOWHtpgOiAO1bACQBkao0BbTAXHoVsvW+t+CUSwACIuECh0xHRBekVq2+VGs+cRrfhRLEv/9AKAmTPkBESuytH1UVmROzcxq7eQ2nbDpOq2d3Ma0J4CokFABEStSyzYKKzKpJQMQOxIqIGJFtm0ZhQaNozIKByBdQWqozOwQSX8o6fSW27H50y9392+FiAuIUa9ato3rVh9QQyU1b0XmKIzCAUhbqKL050siaQIqMAorMlPti8bqS2B0hFzlt0vSdkk/lDQt6asBY8EQ8aYyfE1fkZniKByrL4HREiqh+rGko9zd5w+YWaBQMEy8qaAKKY7CFe0hBqAZgiRU7r4vxL+L+vGmgqqkNgpH3RcwWljlh6HiTQWjahRWXwLYj4QqISk2NuRNBaOK/TCB0ZJ0QmVmG8xs2symd+7cGTqcoUq1sSFvKhhVRXqIAWgOa6kLD8rM5gMZqA/VxMSET09PVxxVPNZObmu7bHzFsnHdtOnMABEVxyo/AECKzGy7u08UObdwUbqZXSjpwgFj+t/u/oEBvxdKuxapVzHxKCdcKf7uKcYMAMPWzyq/RZLGep7V3qDfh1yqjQ17GeW2Cin+7inGDAB1KFxD5e4XubsNeNs0zF9iFDS1FmmU92hL8XdPMWYAqEPITunoQ2tjw9ldcxozO+CNLNXRgZSnMstK8XdPMWYAqEPSq/xGzfo1K54aqdqbLyZIZbVfJ6PcViHF3z3FmJG2FNvFYDSRUCWmaVMuTZ3KLCLF3z3FmGNCctCfVNvFYDQFm/IzsyPVvlj9GWZ2dMvjh919d01hRa9pUy4p7tFWlRR/9xRjjgUF/f1j6yqkJGQN1Yyk49sc/9KCx2dIunHo0SSiiav9UtujrUop/u4hYm5CqwaSg/417QMkmo0pv8Qw5YJR05Rpn6qTg1GYPqRmDykJllC5+6qCLRduDBVjjNjOAqOmKXWDVSYHTUkye+EDJFJC24QE1TXl0oRpFqSvKdM+G9etPqCGSho8ORiV6UNq9pASEiq0RQEtYtGUusEqk4OmJJlFpFhniNFEQoW2RuUTMOJX5chOaFUlB01JMoEmoSgdbY3SJ2DEjbrBp6O2CIgPI1Roi0/AiAnTPgeitgiIDwkV2mrSNAvQRCSZQFxIqNAWn4ABACiOhAod8QkYAIBiKEoHAAAoiYQKAACgJBIqAACAkqih6oHtVwAAQC8kVF2w/QpCI6EHgDQw5ddFU3a5R5rmE/rZXXNy7U/op2ZmQ4cGAFiAhKoLtl9BSCT0AJAOEqouOm2zwvYrqAMJPQCkg4SqCzYgRUgk9ACQDhKqLtjlHiGR0ANAOljl1wPbryAU9lMEgHSQUAERI6EHgDQw5QcAAFASI1TACKJhKABUi4QKGDHsAAAA1WPKDxgxNAwFgOqRUAEjhoahAFA9EipgxNAwFACqFyShMrPlZna+mX3FzH5uZo+b2W/M7J/N7HIze16IuIBRQMNQFDU1M6u1k9t0wqbrtHZyGxtzA12EKkq/d8G//ZikgyWdmN/eZmZvdfcvhggOaDIahqIIFi8A/TF3r/8fNXNJ35X095JucPf7zGxM0n+SdLmkUyTtkXSau/+4yM+cmJjw6enpYYUMACNl7eQ2zbapq1uxbFw3bTozQERA/cxsu7tPFDk3VA3VS9z9Je5+pbvfJ0nuvtfdb5J0tqRfKRvBek+g+ABgpLF4AehPkITK3b/b5bmdkq7PH55WT0QAgFYsXgD6E+sqvwfz+7GuZwEAhoLFC0B/Yu2U/pL8/idBowCAEcXiBaA/0SVUZvYaSfMFYJ8JGQuGi/3kgLitX7OC/yeBgqJKqMxshaQr8ofXuPu3epy/QdIGSVq5cuWQo0OVWJINAGiSwjVUZnahme0Z8PahAj9/iaQpSc+SdJekt/X6Hne/wt0n3H1i+fLlRX8VRID95AAATdLPCNUiDV4k3vX7zOxQSd9QNtW3U9I6d39gwH8LCWBJNgCgSQqPULn7Re5uA942dfq5ZnawpK9KOlPSLklnuzvDFA3HkmwAQJMEbZtgZgdJ+qKkVyrbfuYV7n5LyJhQD5ZkAwCaJFhRupktkvQ5SedKmpN0jrvfHCoe1Isl2QCAJgmSUJmZKVvN90ZJT0o6192/EyIWhMOSbIRE2w4AVQo1QvVRZav49kh6fa/2CABQJdp2AKha7TVUZrZS0gX5Q5f0aTO7r9Ot7vgANB9tOwBULcQIVWsSt1jS7wSIAcAIo20HgKrVPkLl7nf203Kh7vgANB9tOwBULWjbBAAIgbYdAKoW1V5+AFAH2nYAqBoJFYCRRNsOAFViyg8AAKAkEioAAICSSKgAAABKIqECAAAoiYQKAACgJBIqAACAkkioAAAASqIPVY2mZmZpJAgAQAORUNVkamZWm6++7akd7md3zWnz1bdJEkkVAACJY8qvJlu27ngqmZo3t3uvtmzdESgiAABQFRKqmty7a66v4wAAIB0kVDU5btl4X8cBAEA6SKhqsnHdao0vHjvg2PjiMW1ctzpQRAAAoCoUpddkvvCcVX4AADQPCVWN1q9ZQQIFAEADMeUHAABQEgkVAABASSRUAAAAJZFQAQAAlERCBQAAUBIJFQAAQEkkVAAAACWRUAEAAJREY08AI2NqZpbdCgAMRZCEysxWS3qdpBdJWi1puaQlkh6SdIukqyT9g7vvCxEfgOaZmpnV5qtv09zuvZKk2V1z2nz1bZJEUgWgtFBTfq+VdImkV0v6D5IOlfSkpGdJOlvSZyX9k5k9I1B8ABpmy9YdTyVT8+Z279WWrTsCRQSgSUIlVLdL2izpxZKWufth7r5EWUL1Pkl7JP2+pI8Fig9Aw9y7a66v4wDQjyAJlbtf6+6T7n6zuz/ccnynu/+1pMn80BvNbHGIGAE0y3HLxvs6DgD9iHWV3w/z+0MlHRUyEADNsHHdao0vHjvg2PjiMW1ctzpQRACaJNZVfi/O738r6VchAwHQDPOF56zyAzAM0SRUZjYuaaWkN0namB/+W3f3cFEBaJL1a1aQQAEYiuAJlZntkTS24PAeSZ+S9IH6IwIAAOjboe+4AAAJMUlEQVRPDDVU90m6X1LrUptPSvqQu+/u9o1mtsHMps1seufOncOMEQAAoKPCCZWZXWhmewa8fajTz3X3Z7v7MZIOl3S8pI9Ieoek28zsJd1icvcr3H3C3SeWL19e9FcBAACoVD8jVIuUTc0NeuvKM3e7+3sl/bmy1X1fNLPD+4gRAACgdoUTKne/yN1twNumPuO6QtITko6V9LI+vxcAAKBWMdRQPY27PyHpwfzhc0PGAgAA0EuUCZWZLVG2YbIkPRYyFgAAgF6CJFRm1qtdw7slzW8583+GHA4AAEApoUaofmpm7zKz55qZzR80s9Vm9glJl+SHvu7ut4UJEQAAoJhQjT3/vaS/yW9PmNmjytomtO5S+i1J/zVAbAAAAH0JlVCdI+ksSWslHaesXmq3pH+V9ANJX3D36wPFBgAA0BdrylZ5ZrZT0l2h4+jhaEkPhA4iQVy3wXDdBsN1GwzXbTBct8HUdd2Od/dCncMbk1ClwMym3X0idByp4boNhus2GK7bYLhug+G6DSbG6xZl2wQAAICUkFABAACUREJVrytCB5AorttguG6D4boNhus2GK7bYKK7btRQAQAAlMQIFQAAQEkkVAAAACWRUAEAAJREQhWQmS03s/PN7Ctm9nMze9zMfmNm/2xml5vZ80LHGCMzO8TM1pnZX5rZN8zsXjPz/Pay0PGFZGbHmNknWl5P95vZtWZ2VujYYmRmS83sHDO7xMy+aWYPtLyWTgwdX6zMbKWZXZC/tu42syfM7FEzu9XMJs3s2NAxxsjMJvLX2rfM7F/N7OH82s3mf8vWh44xBWa2xMzuafl/9c2hY5IoSg/KzHbrwO1/HpN0cH6TpMclvdXdv1h3bDEzs1MkzXR4+uXu/q0644mFmZ0saZukZ+aHHpG0RNkHJ5f0fnefDBRelPI3sK93ePr57n5HnfGkwMyeo2xXCms5/Iiy/VjH8scPSTrP3b9Tc3hRM7NPSTq/5dBjyt4DDm059jVJf+Luu+uMLSVm9nFJ72459BZ3/2ygcJ7CCFVYB0n6rqT/JulYd18q6TBJvy/pFmX/k12Zv1HiQLskfVvSpKQ/DhxLcGY2LukaZcnUjKST3P0ISUdK+oiyN79LzezscFFG61eSrpf0QUkbAseSgvmk6TpJr5N0VP5aO0zSKyT9QtnrbsrMjgkTYrRulvQeSadJWuruS919XNJKSVvyc86TtClQfNEzs1Ml/Zmk74eOZSFGqAIysz9w9+92eG65pJ9Iepakz7r7W2oNLmJmtkiSe8uL18zmvx7JESozu0DSx5R94j3R3WcXPP91Sesl/cjdTwsQYpTMbMzd97Y8XqUsIZAYoWrLzI6QtMrdb+3w/InKkvpDJV3k7h+sM76Umdk/SPpTSf/m7s8NHU9s8r/935e0RtLpkn6UP8UI1ajrlEzlz+1U9qlZyj7NIOfu+5xPAgu9Kb+/amEylZv/9HsqtUH7tSZTKMbdH+6UTOXP3yHpe/lD/nb154f5/XFBo4jXuyRNSPqku3cq+wiGhCpuD+b3Y13Pwkgzs6Xa/8a1tcNp35P0cP71mUMPCqOOv12DeXF+/4uuZ40gM1sh6RJJ90v6y8DhtEVCFbeX5Pc/CRoFYvd87S8Qvr3dCe6+T9KO/OEL6ggKo8nMDpK0Nn/I364e8hVrJ5vZ30p6Q3748pAxReoySUslvdfdH+51cggH9T4FIZjZa5QNbUrSZ0LGgui1LlG/t8t588+xpB3D9E5Jx0jaJ+nKwLFEycyeLemeNk89LunD7v53NYcUNTN7taTXSrrR3T8fOp5OSKgilA9tzm/8eM0oFlmjL4e3fD3X5bzf5vdLhhgLRli+IvnD+cPL3b3tiCm0V9nUlZStiDxY0h5Jl4rRqQOY2eHKrsluZcl6tJjy65OZXWhmewa8fajAz18iaUrZ6r67JL1t2L9THYZ93Uac9T4FGK68meeUsvYJ2yW9L2xE8XL3X7r7Me5+jKRxSauVjeZ9UNItZvbCoAHG5WJlbSU+5u4/DR1MN4xQ9W+RBi+07Pp9ZnaopG8om+rbKWmduz8w4L8Vm6FdN+ixlq/HJT3a4bzD2pwPlGZmR0m6QdIJkn4m6ZXu/njYqNKQ1zf+i6S3mdkuSX8u6fNmdlr+3MjKmzi/W9n06MWBw+mJEao+uftF7m4D3jo2azOzgyV9VdkKrF2Sznb3HZ3OT82wrhskHVg31W259fxzvxxiLBgxeV+qrZJOknS3pJe6+/3dvwsdXJbfn6Ks19Ko+4SyD9QfkGR5Af9Tt5bzDsmPHdb+x9SDhCoC+aqYL0p6pbLRg1e4+y1ho0JC7lC2tYwktZ0qyBvirc4fRj1sjnTk9S3XKxtVv09ZMnV32KiS1tpDjsae0vH5/ZXKRt4X3uZ9Kn8c9G8bCVVg+Rvd5ySdq6yg+Bx3vzlsVEiJuz8qaTp/+EcdTvuPko7Iv/720INC4+XbHV2rrHfSg8qSqZ+FjSp5J7R8zdR8YkioAjIzU7aa742SnpR0LpuJYkBX5fdvyouDF3pvfr+9SVPJCCMvUbha0hnaX6LAir4uzGws/5vfzcb8fo+yff9Gmruv6lYO0nLqW/Jjq0LFKpFQhfZRZav49kh6Pe0RijOzI83s6Plby1PPaD1uZouDBVmvTytbFbpU0j+a2QukrIu6mf21shFQSXp/oPiiteB1dGTLU8sWvJb4e6ksMVCWwL9M2TTLy939R92/C5KeI2nazN6a96GSlM1SmNkpZvYFSW/PD1/m7g8FiRIDY3PkQMxspbI3QCnrr/Hrbufny2uRM7M7tX9+vZsz3P3G4UYTBzP7PWXTec/MDz2irOfUImU1Vu9398lA4UWrZWPtXk5w9zuHGUsKzOwPJP1T/vBx7d/SqJ173P304UcVvwUbb0vZtXtM2YegQ1qOf1bSf3f3PXXFlqqW/3ej2ByZtgnhtH7aXSzpd0IFgmZw91vN7CRJmyW9StIKZbUtP1DWw4XaKVSh9W/XofmtE1on7Hevsq1lzpL0ImU7FjxT2TX6ubIpvs+4+03BIkQpjFABAACURE0AAABASSRUAAAAJZFQAQAAlERCBQAAUBIJFQAAQEkkVAAAACWRUAEAAJREQgUAAFASCRUAAEBJJFQAAAAl/X9VF4vczBJnyQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 691.2x460.8 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}