From 3c000a7dd741f6b245cd701f3feb861464f063b1 Mon Sep 17 00:00:00 2001
From: Shane Steinert-Threlkeld <ssshanest@gmail.com>
Date: Mon, 22 Jan 2018 11:41:50 +0100
Subject: [PATCH] added prediction / output to notebook

---
 ...NeuralNetworks_TensorFlow_Estimators.ipynb | 1563 ++++++++++++-----
 1 file changed, 1117 insertions(+), 446 deletions(-)

diff --git a/Intro_NeuralNetworks_TensorFlow_Estimators.ipynb b/Intro_NeuralNetworks_TensorFlow_Estimators.ipynb
index ceb4886..002a0bd 100644
--- a/Intro_NeuralNetworks_TensorFlow_Estimators.ipynb
+++ b/Intro_NeuralNetworks_TensorFlow_Estimators.ipynb
@@ -9,7 +9,7 @@
     "### About this notebook:\n",
     "This notebook was written by Shane Steinert-Threlkeld for the Neural Network Methods for Quantifiers coordinated project at the ILLC, Universiteit van Amsterdam in January 2018 (http://shane.st/NNQ).  \n",
     "\n",
-    "It introduces the basics of working with TensorFlow to train neural networks, with an eye to applications to quantifiers.  In particular, the code is a warm-up to understanding this repository: https://github.com/shanest/quantifier-rnn-learning. The main components of that code that this notebook doesn't directly cover are the `lstm_model_fn` and the data generation process, though simpler analogues of both are here.\n",
+    "It introduces the basics of working with TensorFlow to train neural networks, with an eye to applications to quantifiers.  In particular, the code is a warm-up to understanding this repository: https://github.com/shanest/quantifier-rnn-learning. The main components of that code that this notebook doesn't directly cover are the `lstm_model_fn` and the data generation process (`data_gen.py`), though simpler analogues of both are here.\n",
     "\n",
     "There are three sections:\n",
     "\n",
@@ -417,538 +417,538 @@
      "text": [
       "\n",
       "Epoch 0, batch 0, evaluation\n",
-      "{'accuracy': 0.99580377}\n",
+      "{'accuracy': 0.0051880674}\n",
       "\n",
       "Epoch 0, batch 50, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.9473564}\n",
       "\n",
       "Epoch 0, batch 100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99603266}\n",
       "\n",
       "Epoch 0, batch 150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99786371}\n",
       "\n",
       "Epoch 0, batch 200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99824524}\n",
       "\n",
       "Epoch 0, batch 300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 1950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 2950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 3950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 0, batch 4350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 0, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 50, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1800, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1850, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1900, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 1950, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2000, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2050, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2100, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2150, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2200, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2250, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2300, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2350, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2400, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2450, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2500, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2550, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2600, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2650, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2700, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2750, evaluation\n",
-      "{'accuracy': 0.99794}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2800, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
       "\n",
       "Epoch 1, batch 2850, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2900, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 2950, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3000, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99824524}\n",
       "\n",
       "Epoch 1, batch 3050, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 3100, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "{'accuracy': 0.99832153}\n",
+      "\n",
       "Epoch 1, batch 3150, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3200, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3250, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3300, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3350, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3400, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3450, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3500, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99816895}\n",
       "\n",
       "Epoch 1, batch 3550, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99832153}\n",
       "\n",
       "Epoch 1, batch 3600, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99832153}\n",
       "\n",
       "Epoch 1, batch 3650, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99824524}\n",
       "\n",
       "Epoch 1, batch 3700, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99832153}\n",
       "\n",
       "Epoch 1, batch 3750, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 3800, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99847412}\n",
       "\n",
       "Epoch 1, batch 3850, evaluation\n",
-      "{'accuracy': 0.9980163}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 3900, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99847412}\n",
       "\n",
       "Epoch 1, batch 3950, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 4000, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99847412}\n",
       "\n",
       "Epoch 1, batch 4050, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 4100, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 4150, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99847412}\n",
       "\n",
       "Epoch 1, batch 4200, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 4250, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99839783}\n",
       "\n",
       "Epoch 1, batch 4300, evaluation\n",
-      "{'accuracy': 0.99809265}\n",
+      "{'accuracy': 0.99847412}\n",
       "\n",
       "Epoch 1, batch 4350, evaluation\n",
-      "{'accuracy': 0.99809265}\n"
+      "{'accuracy': 0.99847412}\n"
      ]
     }
    ],
@@ -1018,119 +1018,119 @@
      "output_type": "stream",
      "text": [
       "INFO:tensorflow:Using default config.\n",
-      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpytV6Sy\n",
-      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x181daf1f90>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpytV6Sy', '_save_summary_steps': 100}\n",
+      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpC7Ol14\n",
+      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x18131163d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpC7Ol14', '_save_summary_steps': 100}\n",
       "INFO:tensorflow:Create CheckpointSaverHook.\n",
-      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpytV6Sy/model.ckpt.\n",
-      "INFO:tensorflow:loss = 17.2527, step = 1\n",
-      "INFO:tensorflow:global_step/sec: 884.494\n",
-      "INFO:tensorflow:loss = 4.7587, step = 101 (0.115 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1216.25\n",
-      "INFO:tensorflow:loss = 0.769278, step = 201 (0.083 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1244.85\n",
-      "INFO:tensorflow:loss = 0.41293, step = 301 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1258.72\n",
-      "INFO:tensorflow:loss = 0.136705, step = 401 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1256.69\n",
-      "INFO:tensorflow:loss = 0.112782, step = 501 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1215.72\n",
-      "INFO:tensorflow:loss = 0.0807225, step = 601 (0.083 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1231.57\n",
-      "INFO:tensorflow:loss = 0.0384204, step = 701 (0.081 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1247.97\n",
-      "INFO:tensorflow:loss = 0.0692481, step = 801 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1212.96\n",
-      "INFO:tensorflow:loss = 0.115105, step = 901 (0.083 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1175.23\n",
-      "INFO:tensorflow:loss = 0.0309904, step = 1001 (0.085 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1208.08\n",
-      "INFO:tensorflow:loss = 0.0350569, step = 1101 (0.083 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1209.28\n",
-      "INFO:tensorflow:loss = 0.0226983, step = 1201 (0.082 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1173.94\n",
-      "INFO:tensorflow:loss = 0.027183, step = 1301 (0.085 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1186.69\n",
-      "INFO:tensorflow:loss = 0.00683026, step = 1401 (0.084 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1248.59\n",
-      "INFO:tensorflow:loss = 0.0368566, step = 1501 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1250.45\n",
-      "INFO:tensorflow:loss = 0.00637483, step = 1601 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1259.57\n",
-      "INFO:tensorflow:loss = 0.0342509, step = 1701 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1254.58\n",
-      "INFO:tensorflow:loss = 0.0521564, step = 1801 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1226.18\n",
-      "INFO:tensorflow:loss = 0.00563144, step = 1901 (0.082 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1224.16\n",
-      "INFO:tensorflow:loss = 0.0383829, step = 2001 (0.082 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1237.24\n",
-      "INFO:tensorflow:loss = 0.00194878, step = 2101 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1265.98\n",
-      "INFO:tensorflow:loss = 0.00551808, step = 2201 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1268.73\n",
-      "INFO:tensorflow:loss = 0.00700359, step = 2301 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1253.82\n",
-      "INFO:tensorflow:loss = 0.00741444, step = 2401 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1272.98\n",
-      "INFO:tensorflow:loss = 0.00812399, step = 2501 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1244.93\n",
-      "INFO:tensorflow:loss = 0.0192606, step = 2601 (0.081 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1248.94\n",
-      "INFO:tensorflow:loss = 0.00364363, step = 2701 (0.080 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1145\n",
-      "INFO:tensorflow:loss = 0.0195832, step = 2801 (0.087 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1233.9\n",
-      "INFO:tensorflow:loss = 0.0138745, step = 2901 (0.081 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1227.7\n",
-      "INFO:tensorflow:loss = 0.00277025, step = 3001 (0.081 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1202.02\n",
-      "INFO:tensorflow:loss = 0.0078417, step = 3101 (0.083 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1078.82\n",
-      "INFO:tensorflow:loss = 0.00669785, step = 3201 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1216.49\n",
-      "INFO:tensorflow:loss = 0.00290213, step = 3301 (0.082 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1266.51\n",
-      "INFO:tensorflow:loss = 0.0124316, step = 3401 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1293.21\n",
-      "INFO:tensorflow:loss = 0.0294896, step = 3501 (0.077 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1278.43\n",
-      "INFO:tensorflow:loss = 0.00367663, step = 3601 (0.078 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1290.54\n",
-      "INFO:tensorflow:loss = 0.00591448, step = 3701 (0.078 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1263.53\n",
-      "INFO:tensorflow:loss = 0.00585468, step = 3801 (0.079 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1130.45\n",
-      "INFO:tensorflow:loss = 0.0121571, step = 3901 (0.088 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1095.09\n",
-      "INFO:tensorflow:loss = 0.0230224, step = 4001 (0.091 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1286.41\n",
-      "INFO:tensorflow:loss = 0.0113264, step = 4101 (0.078 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1233.15\n",
-      "INFO:tensorflow:loss = 0.00640785, step = 4201 (0.082 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1221.58\n",
-      "INFO:tensorflow:loss = 0.008845, step = 4301 (0.082 sec)\n",
-      "INFO:tensorflow:Saving checkpoints for 4370 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpytV6Sy/model.ckpt.\n",
-      "INFO:tensorflow:Loss for final step: 1.51151e-06.\n",
+      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpC7Ol14/model.ckpt.\n",
+      "INFO:tensorflow:loss = 7.83864, step = 1\n",
+      "INFO:tensorflow:global_step/sec: 713.46\n",
+      "INFO:tensorflow:loss = 2.77174, step = 101 (0.140 sec)\n",
+      "INFO:tensorflow:global_step/sec: 785.337\n",
+      "INFO:tensorflow:loss = 1.20235, step = 201 (0.127 sec)\n",
+      "INFO:tensorflow:global_step/sec: 964.33\n",
+      "INFO:tensorflow:loss = 0.21525, step = 301 (0.104 sec)\n",
+      "INFO:tensorflow:global_step/sec: 951.745\n",
+      "INFO:tensorflow:loss = 0.257051, step = 401 (0.104 sec)\n",
+      "INFO:tensorflow:global_step/sec: 751.033\n",
+      "INFO:tensorflow:loss = 0.18123, step = 501 (0.134 sec)\n",
+      "INFO:tensorflow:global_step/sec: 813.199\n",
+      "INFO:tensorflow:loss = 0.0556332, step = 601 (0.122 sec)\n",
+      "INFO:tensorflow:global_step/sec: 806.457\n",
+      "INFO:tensorflow:loss = 0.0738364, step = 701 (0.125 sec)\n",
+      "INFO:tensorflow:global_step/sec: 352.531\n",
+      "INFO:tensorflow:loss = 0.093898, step = 801 (0.284 sec)\n",
+      "INFO:tensorflow:global_step/sec: 620.11\n",
+      "INFO:tensorflow:loss = 0.0873927, step = 901 (0.161 sec)\n",
+      "INFO:tensorflow:global_step/sec: 658.579\n",
+      "INFO:tensorflow:loss = 0.03727, step = 1001 (0.153 sec)\n",
+      "INFO:tensorflow:global_step/sec: 719.819\n",
+      "INFO:tensorflow:loss = 0.050981, step = 1101 (0.143 sec)\n",
+      "INFO:tensorflow:global_step/sec: 709.984\n",
+      "INFO:tensorflow:loss = 0.0173084, step = 1201 (0.135 sec)\n",
+      "INFO:tensorflow:global_step/sec: 858.31\n",
+      "INFO:tensorflow:loss = 0.0130021, step = 1301 (0.117 sec)\n",
+      "INFO:tensorflow:global_step/sec: 836.946\n",
+      "INFO:tensorflow:loss = 0.0188328, step = 1401 (0.119 sec)\n",
+      "INFO:tensorflow:global_step/sec: 646.221\n",
+      "INFO:tensorflow:loss = 0.0138509, step = 1501 (0.157 sec)\n",
+      "INFO:tensorflow:global_step/sec: 750.603\n",
+      "INFO:tensorflow:loss = 0.00772947, step = 1601 (0.131 sec)\n",
+      "INFO:tensorflow:global_step/sec: 842.042\n",
+      "INFO:tensorflow:loss = 0.00571763, step = 1701 (0.119 sec)\n",
+      "INFO:tensorflow:global_step/sec: 768.651\n",
+      "INFO:tensorflow:loss = 0.00883585, step = 1801 (0.133 sec)\n",
+      "INFO:tensorflow:global_step/sec: 534.622\n",
+      "INFO:tensorflow:loss = 0.00629468, step = 1901 (0.189 sec)\n",
+      "INFO:tensorflow:global_step/sec: 416.212\n",
+      "INFO:tensorflow:loss = 0.00357279, step = 2001 (0.239 sec)\n",
+      "INFO:tensorflow:global_step/sec: 803.303\n",
+      "INFO:tensorflow:loss = 0.0112603, step = 2101 (0.125 sec)\n",
+      "INFO:tensorflow:global_step/sec: 902.397\n",
+      "INFO:tensorflow:loss = 0.0286735, step = 2201 (0.108 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1007.83\n",
+      "INFO:tensorflow:loss = 0.0135154, step = 2301 (0.100 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1056.75\n",
+      "INFO:tensorflow:loss = 0.0426768, step = 2401 (0.094 sec)\n",
+      "INFO:tensorflow:global_step/sec: 691.094\n",
+      "INFO:tensorflow:loss = 0.0169224, step = 2501 (0.146 sec)\n",
+      "INFO:tensorflow:global_step/sec: 847.767\n",
+      "INFO:tensorflow:loss = 0.0102123, step = 2601 (0.117 sec)\n",
+      "INFO:tensorflow:global_step/sec: 402.907\n",
+      "INFO:tensorflow:loss = 0.0128018, step = 2701 (0.248 sec)\n",
+      "INFO:tensorflow:global_step/sec: 600.36\n",
+      "INFO:tensorflow:loss = 0.0142321, step = 2801 (0.168 sec)\n",
+      "INFO:tensorflow:global_step/sec: 469.892\n",
+      "INFO:tensorflow:loss = 0.00369522, step = 2901 (0.216 sec)\n",
+      "INFO:tensorflow:global_step/sec: 869.248\n",
+      "INFO:tensorflow:loss = 0.00704296, step = 3001 (0.111 sec)\n",
+      "INFO:tensorflow:global_step/sec: 908.043\n",
+      "INFO:tensorflow:loss = 0.00423393, step = 3101 (0.110 sec)\n",
+      "INFO:tensorflow:global_step/sec: 734.075\n",
+      "INFO:tensorflow:loss = 0.0125386, step = 3201 (0.137 sec)\n",
+      "INFO:tensorflow:global_step/sec: 830.06\n",
+      "INFO:tensorflow:loss = 0.0157762, step = 3301 (0.120 sec)\n",
+      "INFO:tensorflow:global_step/sec: 757.174\n",
+      "INFO:tensorflow:loss = 0.0223171, step = 3401 (0.133 sec)\n",
+      "INFO:tensorflow:global_step/sec: 683.024\n",
+      "INFO:tensorflow:loss = 0.00733106, step = 3501 (0.146 sec)\n",
+      "INFO:tensorflow:global_step/sec: 827.403\n",
+      "INFO:tensorflow:loss = 0.0088676, step = 3601 (0.121 sec)\n",
+      "INFO:tensorflow:global_step/sec: 786.393\n",
+      "INFO:tensorflow:loss = 0.0182954, step = 3701 (0.128 sec)\n",
+      "INFO:tensorflow:global_step/sec: 725.163\n",
+      "INFO:tensorflow:loss = 0.00439018, step = 3801 (0.137 sec)\n",
+      "INFO:tensorflow:global_step/sec: 908.554\n",
+      "INFO:tensorflow:loss = 0.00565376, step = 3901 (0.114 sec)\n",
+      "INFO:tensorflow:global_step/sec: 814.512\n",
+      "INFO:tensorflow:loss = 0.0366358, step = 4001 (0.118 sec)\n",
+      "INFO:tensorflow:global_step/sec: 909.322\n",
+      "INFO:tensorflow:loss = 0.00599213, step = 4101 (0.110 sec)\n",
+      "INFO:tensorflow:global_step/sec: 837.303\n",
+      "INFO:tensorflow:loss = 0.00331233, step = 4201 (0.119 sec)\n",
+      "INFO:tensorflow:global_step/sec: 697.443\n",
+      "INFO:tensorflow:loss = 0.00354074, step = 4301 (0.144 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 4370 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpC7Ol14/model.ckpt.\n",
+      "INFO:tensorflow:Loss for final step: 0.000285681.\n",
       "WARNING:tensorflow:Casting <dtype: 'float32'> labels to bool.\n",
       "WARNING:tensorflow:Casting <dtype: 'float32'> labels to bool.\n",
-      "INFO:tensorflow:Starting evaluation at 2018-01-19-11:17:09\n",
-      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpytV6Sy/model.ckpt-4370\n",
-      "INFO:tensorflow:Finished evaluation at 2018-01-19-11:17:11\n",
-      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.99794, accuracy_baseline = 0.99794, auc = 0.99981, auc_precision_recall = 1.0, average_loss = 0.00666176, global_step = 4370, label/mean = 0.99794, loss = 87.3157, prediction/mean = 0.998754\n"
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:15\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpC7Ol14/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:17\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, accuracy_baseline = 0.998169, auc = 0.999987, auc_precision_recall = 1.0, average_loss = 0.0053491, global_step = 4370, label/mean = 0.998169, loss = 70.1106, prediction/mean = 0.99876\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'accuracy': 0.99794,\n",
-       " 'accuracy_baseline': 0.99794,\n",
-       " 'auc': 0.99981016,\n",
-       " 'auc_precision_recall': 0.99999958,\n",
-       " 'average_loss': 0.0066617629,\n",
+       "{'accuracy': 0.99816895,\n",
+       " 'accuracy_baseline': 0.99816895,\n",
+       " 'auc': 0.99998724,\n",
+       " 'auc_precision_recall': 1.0,\n",
+       " 'average_loss': 0.0053490987,\n",
        " 'global_step': 4370,\n",
-       " 'label/mean': 0.99794,\n",
-       " 'loss': 87.315727,\n",
-       " 'prediction/mean': 0.99875432}"
+       " 'label/mean': 0.99816895,\n",
+       " 'loss': 70.110634,\n",
+       " 'prediction/mean': 0.9987601}"
       ]
      },
      "execution_count": 9,
@@ -1187,122 +1187,7 @@
    "cell_type": "code",
    "execution_count": 10,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Using default config.\n",
-      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs\n",
-      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x110c48dd0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs', '_save_summary_steps': 100}\n",
-      "INFO:tensorflow:Create CheckpointSaverHook.\n",
-      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs/model.ckpt.\n",
-      "INFO:tensorflow:loss = 0.596629, step = 1\n",
-      "INFO:tensorflow:global_step/sec: 566.967\n",
-      "INFO:tensorflow:loss = 0.0573994, step = 101 (0.181 sec)\n",
-      "INFO:tensorflow:global_step/sec: 760.798\n",
-      "INFO:tensorflow:loss = 0.0411882, step = 201 (0.130 sec)\n",
-      "INFO:tensorflow:global_step/sec: 589.151\n",
-      "INFO:tensorflow:loss = 0.00312614, step = 301 (0.169 sec)\n",
-      "INFO:tensorflow:global_step/sec: 684.26\n",
-      "INFO:tensorflow:loss = 0.00874257, step = 401 (0.149 sec)\n",
-      "INFO:tensorflow:global_step/sec: 894.159\n",
-      "INFO:tensorflow:loss = 0.00712948, step = 501 (0.108 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1025.93\n",
-      "INFO:tensorflow:loss = 0.000943371, step = 601 (0.099 sec)\n",
-      "INFO:tensorflow:global_step/sec: 878.186\n",
-      "INFO:tensorflow:loss = 0.00159582, step = 701 (0.113 sec)\n",
-      "INFO:tensorflow:global_step/sec: 953.353\n",
-      "INFO:tensorflow:loss = 0.000950095, step = 801 (0.104 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1075.2\n",
-      "INFO:tensorflow:loss = 0.00227951, step = 901 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1061.91\n",
-      "INFO:tensorflow:loss = 0.000921319, step = 1001 (0.094 sec)\n",
-      "INFO:tensorflow:global_step/sec: 884.9\n",
-      "INFO:tensorflow:loss = 0.000612237, step = 1101 (0.113 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1005.57\n",
-      "INFO:tensorflow:loss = 0.00123626, step = 1201 (0.099 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1102.06\n",
-      "INFO:tensorflow:loss = 0.00110195, step = 1301 (0.091 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1079.53\n",
-      "INFO:tensorflow:loss = 0.000164814, step = 1401 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1058.67\n",
-      "INFO:tensorflow:loss = 0.000673685, step = 1501 (0.095 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1081.29\n",
-      "INFO:tensorflow:loss = 0.000668124, step = 1601 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 690.002\n",
-      "INFO:tensorflow:loss = 0.000811207, step = 1701 (0.147 sec)\n",
-      "INFO:tensorflow:global_step/sec: 573.175\n",
-      "INFO:tensorflow:loss = 0.0007769, step = 1801 (0.179 sec)\n",
-      "INFO:tensorflow:global_step/sec: 492.145\n",
-      "INFO:tensorflow:loss = 0.00215015, step = 1901 (0.200 sec)\n",
-      "INFO:tensorflow:global_step/sec: 575.96\n",
-      "INFO:tensorflow:loss = 0.00260182, step = 2001 (0.170 sec)\n",
-      "INFO:tensorflow:global_step/sec: 550.97\n",
-      "INFO:tensorflow:loss = 0.000316947, step = 2101 (0.183 sec)\n",
-      "INFO:tensorflow:global_step/sec: 598.624\n",
-      "INFO:tensorflow:loss = 0.000759441, step = 2201 (0.166 sec)\n",
-      "INFO:tensorflow:global_step/sec: 872.029\n",
-      "INFO:tensorflow:loss = 0.00437212, step = 2301 (0.116 sec)\n",
-      "INFO:tensorflow:global_step/sec: 983.177\n",
-      "INFO:tensorflow:loss = 0.000325981, step = 2401 (0.102 sec)\n",
-      "INFO:tensorflow:global_step/sec: 979.202\n",
-      "INFO:tensorflow:loss = 0.000542948, step = 2501 (0.101 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1037.79\n",
-      "INFO:tensorflow:loss = 0.00146643, step = 2601 (0.096 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1078.66\n",
-      "INFO:tensorflow:loss = 0.000219763, step = 2701 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1053\n",
-      "INFO:tensorflow:loss = 0.000202987, step = 2801 (0.095 sec)\n",
-      "INFO:tensorflow:global_step/sec: 963.966\n",
-      "INFO:tensorflow:loss = 0.000171325, step = 2901 (0.104 sec)\n",
-      "INFO:tensorflow:global_step/sec: 716.692\n",
-      "INFO:tensorflow:loss = 0.00612734, step = 3001 (0.143 sec)\n",
-      "INFO:tensorflow:global_step/sec: 533.333\n",
-      "INFO:tensorflow:loss = 0.00103813, step = 3101 (0.184 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1043.57\n",
-      "INFO:tensorflow:loss = 0.163656, step = 3201 (0.096 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1065.02\n",
-      "INFO:tensorflow:loss = 0.000146447, step = 3301 (0.094 sec)\n",
-      "INFO:tensorflow:global_step/sec: 818.431\n",
-      "INFO:tensorflow:loss = 0.000999351, step = 3401 (0.122 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1070.51\n",
-      "INFO:tensorflow:loss = 0.000919003, step = 3501 (0.093 sec)\n",
-      "INFO:tensorflow:global_step/sec: 917.608\n",
-      "INFO:tensorflow:loss = 0.000163469, step = 3601 (0.109 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1073.84\n",
-      "INFO:tensorflow:loss = 0.00029502, step = 3701 (0.094 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1075.6\n",
-      "INFO:tensorflow:loss = 0.00585556, step = 3801 (0.092 sec)\n",
-      "INFO:tensorflow:global_step/sec: 1026.81\n",
-      "INFO:tensorflow:loss = 0.000880797, step = 3901 (0.098 sec)\n",
-      "INFO:tensorflow:global_step/sec: 978.186\n",
-      "INFO:tensorflow:loss = 0.000591939, step = 4001 (0.102 sec)\n",
-      "INFO:tensorflow:global_step/sec: 549.167\n",
-      "INFO:tensorflow:loss = 0.000525635, step = 4101 (0.186 sec)\n",
-      "INFO:tensorflow:global_step/sec: 529.35\n",
-      "INFO:tensorflow:loss = 0.00375145, step = 4201 (0.186 sec)\n",
-      "INFO:tensorflow:global_step/sec: 478.279\n",
-      "INFO:tensorflow:loss = 4.31323e-05, step = 4301 (0.211 sec)\n",
-      "INFO:tensorflow:Saving checkpoints for 4370 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs/model.ckpt.\n",
-      "INFO:tensorflow:Loss for final step: 0.000183804.\n",
-      "INFO:tensorflow:Starting evaluation at 2018-01-19-11:17:22\n",
-      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs/model.ckpt-4370\n",
-      "INFO:tensorflow:Finished evaluation at 2018-01-19-11:17:23\n",
-      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.99794, global_step = 4370, loss = 0.00621985\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'accuracy': 0.99794, 'global_step': 4370, 'loss': 0.0062198518}"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# required arguments; params will contain anything custom you want to pass to the model-building function\n",
     "def ffnn_model_fn(features, labels, mode, params):\n",
@@ -1317,25 +1202,34 @@
     "    # -- logits: [batch_size, num_classes]\n",
     "    # note: default for tf.layers.dense is no activation, i.e. linear\n",
     "    logits = tf.layers.dense(hidden, params['num_classes'])\n",
+    "\n",
+    "    # predictions\n",
+    "    # -- probs: [batch_size, num_classes]\n",
+    "    probs = tf.nn.softmax(logits)\n",
+    "    # predictions to be output; can be customized!\n",
+    "    out_preds = {'probs': probs}\n",
+    "    \n",
+    "    # NOTE: prediction mode needs to be handled first, since TF\n",
+    "    # automatically passes `None` for the `labels` argument in this\n",
+    "    # mode and most loss functions throw an error in that situation\n",
+    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
+    "        return tf.estimator.EstimatorSpec(mode=mode,\n",
+    "                                         predictions=out_preds)\n",
+    "    \n",
+    "    # training\n",
     "    # -- cross_entropy: [batch_size]\n",
-    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\n",
-    "            labels=labels,\n",
-    "            logits=logits)\n",
+    "    cross_entropy = tf.losses.softmax_cross_entropy(\n",
+    "        onehot_labels=labels,\n",
+    "        logits=logits)\n",
     "    # -- loss: []\n",
     "    loss = tf.reduce_mean(cross_entropy)\n",
-    "        \n",
-    "    # training\n",
+    "    \n",
     "    optimizer = tf.train.AdamOptimizer()\n",
     "    # it's important to pass global_step here!\n",
     "    train_op = optimizer.minimize(loss,\n",
     "                                 global_step=tf.train.get_global_step())\n",
     "    \n",
-    "    # predictions\n",
-    "    # -- probs: [batch_size, num_classes]\n",
-    "    probs = tf.nn.softmax(logits)\n",
-    "    \n",
-    "    # predictions to be output; can be customized!\n",
-    "    out_preds = {'probs': probs}\n",
+    "    # evaluation metrics\n",
     "    \n",
     "    # -- predictions: [batch_size]\n",
     "    predictions = tf.argmax(probs, axis=1)\n",
@@ -1351,13 +1245,143 @@
     "    return tf.estimator.EstimatorSpec(mode=mode,\n",
     "                                     loss=loss,\n",
     "                                     train_op=train_op,\n",
-    "                                     predictions=out_preds,\n",
-    "                                     eval_metric_ops=eval_metrics)\n",
-    "\n",
+    "                                     eval_metric_ops=eval_metrics)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Once we have built our `model_fn`, we can train in much the same way as before.  The only real difference is that we pass our new function as the `model_fn` argument to `tf.estimator.Estimator`, instead of initializing one of the predefined estimators."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Using default config.\n",
+      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1\n",
+      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x18130bc690>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1', '_save_summary_steps': 100}\n",
+      "INFO:tensorflow:Create CheckpointSaverHook.\n",
+      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt.\n",
+      "INFO:tensorflow:loss = 0.40373, step = 1\n",
+      "INFO:tensorflow:global_step/sec: 915.072\n",
+      "INFO:tensorflow:loss = 0.0261521, step = 101 (0.111 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1126.2\n",
+      "INFO:tensorflow:loss = 0.00626468, step = 201 (0.089 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1141.61\n",
+      "INFO:tensorflow:loss = 0.00303013, step = 301 (0.088 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1163.8\n",
+      "INFO:tensorflow:loss = 0.00209895, step = 401 (0.086 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1091.23\n",
+      "INFO:tensorflow:loss = 0.0023126, step = 501 (0.092 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1272.9\n",
+      "INFO:tensorflow:loss = 0.00186653, step = 601 (0.078 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1313.97\n",
+      "INFO:tensorflow:loss = 0.000437752, step = 701 (0.076 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1204.31\n",
+      "INFO:tensorflow:loss = 0.0012747, step = 801 (0.083 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1156.87\n",
+      "INFO:tensorflow:loss = 0.000217566, step = 901 (0.086 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1262.09\n",
+      "INFO:tensorflow:loss = 5.47679e-05, step = 1001 (0.079 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1290.67\n",
+      "INFO:tensorflow:loss = 0.00088836, step = 1101 (0.078 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1316.66\n",
+      "INFO:tensorflow:loss = 0.000217853, step = 1201 (0.076 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1255.89\n",
+      "INFO:tensorflow:loss = 0.000710021, step = 1301 (0.080 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1239.65\n",
+      "INFO:tensorflow:loss = 0.000706173, step = 1401 (0.081 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1237.47\n",
+      "INFO:tensorflow:loss = 0.205396, step = 1501 (0.081 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1239.42\n",
+      "INFO:tensorflow:loss = 0.000154034, step = 1601 (0.081 sec)\n",
+      "INFO:tensorflow:global_step/sec: 690.302\n",
+      "INFO:tensorflow:loss = 0.000278102, step = 1701 (0.151 sec)\n",
+      "INFO:tensorflow:global_step/sec: 612.231\n",
+      "INFO:tensorflow:loss = 0.00133857, step = 1801 (0.161 sec)\n",
+      "INFO:tensorflow:global_step/sec: 556.542\n",
+      "INFO:tensorflow:loss = 0.00084015, step = 1901 (0.176 sec)\n",
+      "INFO:tensorflow:global_step/sec: 682.208\n",
+      "INFO:tensorflow:loss = 0.000359679, step = 2001 (0.149 sec)\n",
+      "INFO:tensorflow:global_step/sec: 717.329\n",
+      "INFO:tensorflow:loss = 0.00117734, step = 2101 (0.137 sec)\n",
+      "INFO:tensorflow:global_step/sec: 968.1\n",
+      "INFO:tensorflow:loss = 0.000172806, step = 2201 (0.103 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1285.23\n",
+      "INFO:tensorflow:loss = 0.00153233, step = 2301 (0.078 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1297.81\n",
+      "INFO:tensorflow:loss = 0.00204422, step = 2401 (0.077 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1271.51\n",
+      "INFO:tensorflow:loss = 5.46721e-05, step = 2501 (0.079 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1272.6\n",
+      "INFO:tensorflow:loss = 0.000292676, step = 2601 (0.079 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1164.44\n",
+      "INFO:tensorflow:loss = 0.000216377, step = 2701 (0.086 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1153.32\n",
+      "INFO:tensorflow:loss = 0.00136904, step = 2801 (0.087 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1018.91\n",
+      "INFO:tensorflow:loss = 0.000602928, step = 2901 (0.099 sec)\n",
+      "INFO:tensorflow:global_step/sec: 614.617\n",
+      "INFO:tensorflow:loss = 0.000597476, step = 3001 (0.163 sec)\n",
+      "INFO:tensorflow:global_step/sec: 514.228\n",
+      "INFO:tensorflow:loss = 0.00032866, step = 3101 (0.196 sec)\n",
+      "INFO:tensorflow:global_step/sec: 810.663\n",
+      "INFO:tensorflow:loss = 0.000562001, step = 3201 (0.122 sec)\n",
+      "INFO:tensorflow:global_step/sec: 592.701\n",
+      "INFO:tensorflow:loss = 0.00078518, step = 3301 (0.169 sec)\n",
+      "INFO:tensorflow:global_step/sec: 811.701\n",
+      "INFO:tensorflow:loss = 0.000591503, step = 3401 (0.123 sec)\n",
+      "INFO:tensorflow:global_step/sec: 622.646\n",
+      "INFO:tensorflow:loss = 0.00269765, step = 3501 (0.161 sec)\n",
+      "INFO:tensorflow:global_step/sec: 553.897\n",
+      "INFO:tensorflow:loss = 0.000368476, step = 3601 (0.183 sec)\n",
+      "INFO:tensorflow:global_step/sec: 759.838\n",
+      "INFO:tensorflow:loss = 0.000308464, step = 3701 (0.129 sec)\n",
+      "INFO:tensorflow:global_step/sec: 986.262\n",
+      "INFO:tensorflow:loss = 0.000610318, step = 3801 (0.100 sec)\n",
+      "INFO:tensorflow:global_step/sec: 570.998\n",
+      "INFO:tensorflow:loss = 0.000573463, step = 3901 (0.175 sec)\n",
+      "INFO:tensorflow:global_step/sec: 721.912\n",
+      "INFO:tensorflow:loss = 0.000990761, step = 4001 (0.139 sec)\n",
+      "INFO:tensorflow:global_step/sec: 605.202\n",
+      "INFO:tensorflow:loss = 6.52921e-05, step = 4101 (0.169 sec)\n",
+      "INFO:tensorflow:global_step/sec: 786.133\n",
+      "INFO:tensorflow:loss = 0.000486063, step = 4201 (0.123 sec)\n",
+      "INFO:tensorflow:global_step/sec: 1241.51\n",
+      "INFO:tensorflow:loss = 0.00186887, step = 4301 (0.080 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 4370 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt.\n",
+      "INFO:tensorflow:Loss for final step: 0.00124508.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:28\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:29\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'accuracy': 0.99816895, 'global_step': 4370, 'loss': 0.0045275646}"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
     "tf.reset_default_graph()\n",
     "\n",
     "# hyperparameters\n",
     "hparams = {'hidden_size': 10, 'hidden_activation': tf.nn.relu, 'num_classes': 2}\n",
+    "\n",
+    "# to build custom estimator, use model_fn and params arguments\n",
     "estimator = tf.estimator.Estimator(model_fn=ffnn_model_fn, params=hparams)\n",
     "\n",
     "new_train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
@@ -1378,6 +1402,49 @@
     "estimator.evaluate(input_fn=new_test_input_fn)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Prediction: getting output from an `Estimator`\n",
+    "\n",
+    "The `predict` method allows you to generate predictions from given inputs to the model.  The intended use for this is to use a trained model to generate predictions on new data.  For example, a trained image classifier can be given a new image to label.\n",
+    "\n",
+    "For the purposes of this project, however, we can make use of some flexibility in `predict`: in your `model_fn`, you get to specify exactly what the `predict` method outputs, as a dictionary whose keys are names and values are `Tensor`s (which will be evaluated).\n",
+    "\n",
+    "**EXERCISE**: modify `ffnn_model_fn` so that the hidden layer is also output by `predict`.  For what might this be useful?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "[{'probs': array([  1.16151043e-07,   9.99999881e-01], dtype=float32)}, {'probs': array([  5.35429281e-04,   9.99464571e-01], dtype=float32)}, {'probs': array([  3.53403360e-04,   9.99646544e-01], dtype=float32)}, {'probs': array([  3.46445654e-06,   9.99996543e-01], dtype=float32)}, {'probs': array([  6.28815178e-05,   9.99937057e-01], dtype=float32)}]\n",
+      "(0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1): [  1.16151043e-07   9.99999881e-01]\n",
+      "(0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0): [  5.35429281e-04   9.99464571e-01]\n",
+      "(0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0): [  3.53403360e-04   9.99646544e-01]\n",
+      "(0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1): [  3.46445654e-06   9.99996543e-01]\n",
+      "(0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0): [  6.28815178e-05   9.99937057e-01]\n"
+     ]
+    }
+   ],
+   "source": [
+    "predict_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
+    "    x={\"x\": np.array(testX[:5])},\n",
+    "    shuffle=False)\n",
+    "\n",
+    "predictions = list(estimator.predict(input_fn=predict_input_fn))\n",
+    "print predictions\n",
+    "for idx in range(5):\n",
+    "    print '{}: {}'.format(testX[idx], predictions[idx]['probs'])"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -1390,7 +1457,9 @@
     "\n",
     "With a `SessionRunHook`, you implement behavior that you want to execute before and after every `session.run` call made by `estimator.train()`.  Full documentation here: https://www.tensorflow.org/api_docs/python/tf/train/SessionRunHook.\n",
     "\n",
-    "To implement evaluation during training, we make a new class, extending `SessionRunHook`, which takes a given estimator and input_function, and evaluates the estimator on that input every N steps.  The more complicated Hook in the main repository also implements early stopping and writes the data to a CSV file at the end of training."
+    "To implement evaluation during training, we make a new class, extending `SessionRunHook`, which takes a given estimator and input_function, and evaluates the estimator on that input every N steps.  The more complicated Hook in the main repository also implements early stopping and writes the data to a CSV file at the end of training.\n",
+    "\n",
+    "**EXERCISE**: implement an early stopping condition in `after_run` of the hook below.  Example: when the evaluation loss is below a certain threshold. Hint: `run_context.request_stop()` should be called when your stop condition is met."
    ]
   },
   {
@@ -1402,25 +1471,636 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpPolZkr\n",
-      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': None, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x182193dc50>, '_save_checkpoints_steps': 50, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpPolZkr', '_save_summary_steps': 100}\n",
+      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW\n",
+      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': None, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x10418cad0>, '_save_checkpoints_steps': 50, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW', '_save_summary_steps': 100}\n",
       "INFO:tensorflow:Create CheckpointSaverHook.\n",
-      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpPolZkr/model.ckpt.\n",
-      "INFO:tensorflow:Starting evaluation at 2018-01-19-13:10:03\n",
-      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs/model.ckpt-8740\n",
-      "INFO:tensorflow:Finished evaluation at 2018-01-19-13:10:03\n",
-      "INFO:tensorflow:Saving dict for global step 8740: accuracy = 0.998093, global_step = 8740, loss = 0.00393171\n",
-      "{'loss': 0.0039317091, 'global_step': 8740, 'accuracy': 0.99809265}\n",
-      "INFO:tensorflow:loss = 1.34821, step = 1\n",
-      "INFO:tensorflow:Saving checkpoints for 51 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpPolZkr/model.ckpt.\n",
-      "INFO:tensorflow:Starting evaluation at 2018-01-19-13:10:04\n",
-      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpbiiYqs/model.ckpt-8740\n",
-      "INFO:tensorflow:Finished evaluation at 2018-01-19-13:10:04\n",
-      "INFO:tensorflow:Saving dict for global step 8740: accuracy = 0.998093, global_step = 8740, loss = 0.00393171\n",
-      "{'loss': 0.0039317091, 'global_step': 8740, 'accuracy': 0.99809265}\n",
-      "INFO:tensorflow:Saving checkpoints for 101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpPolZkr/model.ckpt.\n",
-      "INFO:tensorflow:global_step/sec: 35.9235\n",
-      "INFO:tensorflow:Starting evaluation at 2018-01-19-13:10:05\n"
+      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:35\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:36\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.623604, step = 1\n",
+      "INFO:tensorflow:Saving checkpoints for 51 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:38\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:38\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 20.9697\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:39\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:40\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.127373, step = 101 (4.200 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 151 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:42\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:42\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 201 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 21.75\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:44\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:45\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.0452373, step = 201 (4.422 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 251 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:46\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:47\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 301 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 22.3597\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:48\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:49\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.0219958, step = 301 (4.059 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 351 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:50\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:50\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 401 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.3099\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:51\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:52\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.145455, step = 401 (3.096 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 451 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:53\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:53\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 501 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 33.7466\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:54\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:55\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.0072122, step = 501 (2.950 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 551 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:56\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:56\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 601 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 29.3276\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:58\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:37:58\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00469071, step = 601 (3.475 sec)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Saving checkpoints for 651 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:37:59\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:00\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 701 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 28.0371\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:01\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:02\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00151908, step = 701 (3.448 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 751 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:03\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:04\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 801 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 28.8678\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:05\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:05\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000728766, step = 801 (3.578 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 851 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:07\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:07\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 901 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 26.647\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:08\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:09\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00290355, step = 901 (3.936 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 951 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:10\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:11\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1001 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 25.7443\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:13\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:13\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000421366, step = 1001 (4.075 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1051 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:15\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:16\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 22.4276\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:17\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:17\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00179282, step = 1101 (4.089 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1151 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:19\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:19\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1201 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 26.7006\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:20\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:21\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000411152, step = 1201 (4.089 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1251 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:23\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:24\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1301 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 24.8819\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:25\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:25\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00051571, step = 1301 (3.790 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1351 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:27\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:28\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1401 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 24.5547\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:29\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:30\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.0010395, step = 1401 (4.366 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1451 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:31\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:32\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1501 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 24.5165\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:33\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:33\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00350252, step = 1501 (3.708 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1551 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:35\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:35\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1601 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 29.7711\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:36\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:37\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00309886, step = 1601 (3.539 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1651 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:38\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:39\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1701 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 27.4771\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:40\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:40\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000915611, step = 1701 (3.458 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1751 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:41\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:42\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1801 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 33.1174\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:43\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:43\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000335629, step = 1801 (3.043 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1851 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:44\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:45\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 1901 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.5372\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:46\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:46\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000342935, step = 1901 (3.097 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 1951 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:47\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:48\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2001 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.2262\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:49\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:49\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000314185, step = 2001 (3.019 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2051 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:50\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:51\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.003\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:52\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:53\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000935204, step = 2101 (3.209 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2151 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:54\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:54\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2201 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.0718\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:55\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:56\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000288804, step = 2201 (3.010 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2251 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:57\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:57\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2301 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 33.7496\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:38:58\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:38:59\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000773663, step = 2301 (3.058 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2351 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:00\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:00\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2401 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 31.4653\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:01\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:02\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000316038, step = 2401 (3.228 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2451 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:03\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:04\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2501 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 29.6744\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:05\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:06\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00115846, step = 2501 (3.688 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2551 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:07\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:07\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2601 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 23.2515\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:09\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:10\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000238344, step = 2601 (4.303 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2651 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:11\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:12\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2701 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 27.6763\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:13\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:14\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00291038, step = 2701 (3.909 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2751 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:15\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:15\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2801 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 25.8372\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:17\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:18\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00134723, step = 2801 (3.824 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2851 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:19\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:19\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 2901 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 25.3891\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:20\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:21\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000755183, step = 2901 (3.257 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 2951 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:22\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:22\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3001 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 32.8283\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:23\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:24\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.283004, step = 3001 (2.970 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3051 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:25\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:25\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 33.6813\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:26\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:27\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000185685, step = 3101 (2.972 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3151 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:28\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:28\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3201 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 34.4592\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:29\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:30\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000140159, step = 3201 (2.829 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3251 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:31\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:31\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3301 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 36.031\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:32\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:33\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000540794, step = 3301 (2.928 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3351 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:33\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:34\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3401 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 34.1958\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:35\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:36\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000283208, step = 3401 (2.901 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3451 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:37\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:38\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3501 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 27.7768\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:39\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:40\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00125152, step = 3501 (4.178 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3551 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:41\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:42\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3601 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 24.2278\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:43\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:44\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00126311, step = 3601 (4.026 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3651 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:45\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:46\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3701 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 20.8187\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:48\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:49\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000556503, step = 3701 (4.815 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3751 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:50\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:50\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3801 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 28.9301\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:51\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:51\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000581492, step = 3801 (2.887 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3851 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:52\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:53\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 3901 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 35.3321\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:54\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:55\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000448355, step = 3901 (3.076 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 3951 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:56\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:57\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 4001 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 22.665\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:39:58\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:39:59\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.00167535, step = 4001 (4.345 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 4051 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:40:00\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:40:00\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 4101 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 29.1848\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:40:02\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:40:03\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n",
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:loss = 0.000128033, step = 4101 (3.630 sec)\n",
+      "INFO:tensorflow:Saving checkpoints for 4151 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:40:04\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n",
+      "INFO:tensorflow:Finished evaluation at 2018-01-22-10:40:04\n",
+      "INFO:tensorflow:Saving dict for global step 4370: accuracy = 0.998169, global_step = 4370, loss = 0.00452756\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.0045275646, 'global_step': 4370, 'accuracy': 0.99816895}\n",
+      "INFO:tensorflow:Saving checkpoints for 4201 into /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpcxrTsW/model.ckpt.\n",
+      "INFO:tensorflow:global_step/sec: 26.1858\n",
+      "INFO:tensorflow:Starting evaluation at 2018-01-22-10:40:06\n",
+      "INFO:tensorflow:Restoring parameters from /var/folders/l2/l0wvjplx49x12_mv2540hycr0000gn/T/tmpTWGed1/model.ckpt-4370\n"
      ]
     }
    ],
@@ -1468,15 +2148,6 @@
     "new_estimator.train(input_fn=new_train_input_fn,\n",
     "                hooks=[EvalDuringHook(estimator, new_test_input_fn)])"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {