q3-uni.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy.misc\n",
    "import tensorflow as tf\n",
    "import glob\n",
    "import sys\n",
    "import tempfile\n",
    "# you shouldn't need to make any more imports\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Helper functions, DO NOT modify this\n",
    "\n",
    "def get_img_array(path):\n",
    "    \"\"\"\n",
    "    Given path of image, returns it's numpy array\n",
    "    \"\"\"\n",
    "    return scipy.misc.imread(path)\n",
    "\n",
    "def get_files(folder):\n",
    "    \"\"\"\n",
    "    Given path to folder, returns list of files in it\n",
    "    \"\"\"\n",
    "    filenames = [file for file in glob.glob(folder+'*/*')]\n",
    "    filenames.sort()\n",
    "    return filenames\n",
    "\n",
    "def get_label(filepath, label2id):\n",
    "    \"\"\"\n",
    "    Files are assumed to be labeled as: /path/to/file/999_frog.png\n",
    "    Returns label for a filepath\n",
    "    \"\"\"\n",
    "    tokens = filepath.split('/')\n",
    "    label = tokens[-1].split('_')[1][:-4]\n",
    "    if label in label2id:\n",
    "        return label2id[label]\n",
    "    else:\n",
    "        sys.exit(\"Invalid label: \" + label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Functions to load data, DO NOT change these\n",
    "\n",
    "def get_labels(folder, label2id):\n",
    "    \"\"\"\n",
    "    Returns vector of labels extracted from filenames of all files in folder\n",
    "    :param folder: path to data folder\n",
    "    :param label2id: mapping of text labels to numeric ids. (Eg: automobile -> 0)\n",
    "    \"\"\"\n",
    "    files = get_files(folder)\n",
    "    y = []\n",
    "    for f in files:\n",
    "        y.append(get_label(f,label2id))\n",
    "    return np.array(y)\n",
    "\n",
    "def one_hot(y, num_classes=10):\n",
    "    \"\"\"\n",
    "    Converts each label index in y to vector with one_hot encoding\n",
    "    \"\"\"\n",
    "    y_one_hot = np.zeros((y.shape[0], num_classes))\n",
    "    y_one_hot[np.arange(y.shape[0]),y] = 1\n",
    "    return y_one_hot.T\n",
    "\n",
    "def get_label_mapping(label_file):\n",
    "    \"\"\"\n",
    "    Returns mappings of label to index and index to label\n",
    "    The input file has list of labels, each on a separate line.\n",
    "    \"\"\"\n",
    "    with open(label_file, 'r') as f:\n",
    "        id2label = f.readlines()\n",
    "        id2label = [l.strip() for l in id2label]\n",
    "    label2id = {}\n",
    "    count = 0\n",
    "    for label in id2label:\n",
    "        label2id[label] = count\n",
    "        count += 1\n",
    "    return id2label, label2id\n",
    "\n",
    "def get_images(folder):\n",
    "    \"\"\"\n",
    "    returns numpy array of all samples in folder\n",
    "    each column is a sample resized to 30x30 and flattened\n",
    "    \"\"\"\n",
    "    files = get_files(folder)\n",
    "    images = []\n",
    "    count = 0\n",
    "    \n",
    "    for f in files:\n",
    "        count += 1\n",
    "        if count % 10000 == 0:\n",
    "            print(\"Loaded {}/{}\".format(count,len(files)))\n",
    "        img_arr = get_img_array(f)\n",
    "        img_arr = img_arr.flatten() / 255.0\n",
    "        images.append(img_arr)\n",
    "    X = np.column_stack(images)\n",
    "\n",
    "    return X\n",
    "\n",
    "def get_train_data(data_root_path):\n",
    "    \"\"\"\n",
    "    Return X and y\n",
    "    \"\"\"\n",
    "    train_data_path = data_root_path + 'train'\n",
    "    id2label, label2id = get_label_mapping(data_root_path+'labels.txt')\n",
    "    print(label2id)\n",
    "    X = get_images(train_data_path)\n",
    "    y = get_labels(train_data_path, label2id)\n",
    "    return X, y\n",
    "\n",
    "def save_predictions(filename, y):\n",
    "    \"\"\"\n",
    "    Dumps y into .npy file\n",
    "    \"\"\"\n",
    "    np.save(filename, y)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "  \"\"\"weight_variable generates a weight variable of a given shape.\"\"\"\n",
    "  initial = tf.truncated_normal(shape, stddev=0.05)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def bias_variable(shape):\n",
    "  \"\"\"bias_variable generates a bias variable of a given shape.\"\"\"\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 3, 3, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "def norm(x): \n",
    "  return tf.nn.lrn(x, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deepnn(x):\n",
    "  \"\"\"deepnn builds the graph for a deep net for classifying digits.\n",
    "  Args:\n",
    "    x: an input tensor with the dimensions (N_examples, 3072), where 3072 is the\n",
    "    number of pixels in a standard CIFAR image.\n",
    "  Returns:\n",
    "    A tuple (y, keep_prob). y is a tensor of shape (N_examples, 10), with values\n",
    "    equal to the logits of classifying the digit into one of 10 classes (the\n",
    "    digits 0-9). keep_prob is a scalar placeholder for the probability of\n",
    "    dropout.\n",
    "  \"\"\"\n",
    "  # Reshape to use within a convolutional neural net.\n",
    "  # Last dimension is for \"features\" - there is only one here, since images are\n",
    "  # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.\n",
    "  with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 32, 32, 3])\n",
    "    #x_image = tf.image.per_image_standardization(x_image)\n",
    "\n",
    "  # First convolutional layer - maps one grayscale image to 64 feature maps.\n",
    "  with tf.name_scope('conv1'):\n",
    "    W_conv1 = weight_variable([5, 5, 3, 64])\n",
    "    b_conv1 = bias_variable([64])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "\n",
    "  # Pooling layer - downsamples by 2X.\n",
    "  with tf.name_scope('pool1'):\n",
    "    h_pool1 = max_pool_2x2(h_conv1)\n",
    "    \n",
    "  # First Normalization layer.\n",
    "  with tf.name_scope('norm1'):\n",
    "    h_norm1 = norm(h_pool1)\n",
    "\n",
    "  # Second convolutional layer -- maps 64 feature maps to 64.\n",
    "  with tf.name_scope('conv2'):\n",
    "    W_conv2 = weight_variable([5, 5, 64, 64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_norm1, W_conv2) + b_conv2)\n",
    "    \n",
    "  # Second Normalization layer.\n",
    "  with tf.name_scope('norm2'):\n",
    "    h_norm2 = norm(h_conv2)\n",
    "    \n",
    "  # Second pooling layer.\n",
    "  with tf.name_scope('pool2'):\n",
    "    h_pool2 = max_pool_2x2(h_norm2)\n",
    "\n",
    "  # Fully connected layer 1 -- after 2 round of downsampling, our 32x32x3 image\n",
    "  # is down to 8x8x64 feature maps -- maps this to 512 features.\n",
    "  with tf.name_scope('fc1'):\n",
    "    W_fc1 = weight_variable([8*8*64, 512])\n",
    "    b_fc1 = bias_variable([512])\n",
    "\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 8*8*64])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "  # Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "  # features.\n",
    "  with tf.name_scope('dropout'):\n",
    "    keep_prob = tf.placeholder(tf.float32)\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "  # Map the 1024 features to 10 classes, one for each digit\n",
    "  with tf.name_scope('fc2'):\n",
    "    W_fc2 = weight_variable([512, 10])\n",
    "    b_fc2 = bias_variable([10])\n",
    "\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "  #Set up some logging stuff\n",
    "  trainables = tf.trainable_variables()\n",
    "  #Make histograms of WEIGHTS\n",
    "  for i, Variable in enumerate(trainables):\n",
    "    tf.summary.histogram(Variable.name,Variable)  \n",
    "    \n",
    "  return y_conv, keep_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n",
      "Loaded 10000/50000\n",
      "Loaded 20000/50000\n",
      "Loaded 30000/50000\n",
      "Loaded 40000/50000\n",
      "Loaded 50000/50000\n",
      "Loaded 10000/10000\n",
      "Data loading done\n"
     ]
    }
   ],
   "source": [
    "# Load the data, using utility functions from HW1\n",
    "data_root_path = 'cifar10-hw1/'\n",
    "X, Y = get_train_data(data_root_path) # this may take a few minutes\n",
    "X_test = get_images(data_root_path + 'test').astype(np.float32)\n",
    "print('Data loading done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Get_PreprocessInput(X_train):\n",
    "    \"\"\"\n",
    "    Do standard scaling;\n",
    "    --or do whitening or PCA on the data.\n",
    "        \n",
    "    Assumes X is (Nfeatures x Nexamples) numpy array\n",
    "    \"\"\"\n",
    "    mu = X_train.mean(axis=1)\n",
    "    sigma = X_train.std(axis=1)#mu*0.+1.#X_train.std(axis=1)#X_train.var(axis=1)\n",
    "    return mu, sigma\n",
    "\n",
    "    \n",
    "def Do_PreprocessInput(X, mu, sigma):\n",
    "    \"\"\"\n",
    "    Do the preprocessing on the given data, \n",
    "    given some already calculated mu + sigma\n",
    "    which come from the training data.\n",
    "    \"\"\"\n",
    "    X -= mu.reshape(X.shape[0],1).repeat(X.shape[1],axis=1)\n",
    "    epsilon = 10e-9\n",
    "    X /= (sigma.reshape(X.shape[0],1).repeat(X.shape[1],axis=1)+epsilon)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mu, sigma = Get_PreprocessInput(X)\n",
    "X = Do_PreprocessInput(X, mu, sigma)\n",
    "X_test = Do_PreprocessInput(X_test, mu, sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Hold out 10% of the data to use as a validation set:\n",
    "#y_OH = tf.one_hot(Y,10)\n",
    "train_inds = np.random.choice(X.shape[1],int(X.shape[1]*.90),replace=False)\n",
    "validation_inds = np.setdiff1d(np.arange(X.shape[1]),train_inds)\n",
    "X_train = X[:,train_inds].astype(np.float32)\n",
    "y_train = Y[train_inds].astype(np.int32)\n",
    "X_validation = X[:,validation_inds].astype(np.float32)\n",
    "y_validation = Y[validation_inds].astype(np.int32)\n",
    "#X_placeholder = tf.placeholder(X_train.dtype, X_train.shape)\n",
    "#y_placeholder = tf.placeholder(y_train.dtype, y_train.shape)\n",
    "#dataset_train = tf.contrib.data.Dataset.from_tensor_slices((X_placeholder, y_placeholder))\n",
    "#iterator = dataset_train.make_initializable_iterator()\n",
    "#sess.run(iterator.initializer, feed_dict={X_placeholder: X_train,\n",
    "                                          #y_placeholder: tfy_train})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batch(X, y, batch_size):\n",
    "    selector = np.random.choice(y.shape[1], batch_size, replace=False)\n",
    "    return X[:, selector].T, y[:,selector].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 3072])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name conv1/Variable:0 is illegal; using conv1/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name conv1/Variable_1:0 is illegal; using conv1/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable:0 is illegal; using conv2/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable_1:0 is illegal; using conv2/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable:0 is illegal; using fc1/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable_1:0 is illegal; using fc1/Variable_1_0 instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable:0 is illegal; using fc2/Variable_0 instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable_1:0 is illegal; using fc2/Variable_1_0 instead.\n"
     ]
    }
   ],
   "source": [
    "y_conv, keep_prob = deepnn(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('loss'):\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,\n",
    "                                                            logits=y_conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "OPTIMIZER = 'ADAM'#'SGD'\n",
    "\n",
    "#with tf.name_scope('adam_optimizer'):\n",
    "#    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "with tf.name_scope('learning_rate'):\n",
    "    lr = tf.train.exponential_decay(0.1,\n",
    "                                  global_step,\n",
    "                                  450*350,\n",
    "                                  0.1,\n",
    "                                  staircase=True)\n",
    "\n",
    "if OPTIMIZER=='SGD':\n",
    "    with tf.name_scope('gradientDescent_optimizer'):\n",
    "        optimizer = tf.train.GradientDescentOptimizer(lr)\n",
    "        train_step = optimizer.minimize(cross_entropy)\n",
    "if OPTIMIZER=='ADAM':\n",
    "    with tf.name_scope('ADAM_optimizer'):\n",
    "        #Don't need to specify lr, ADAM determines ~optimal lr\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        train_step = optimizer.minimize(cross_entropy)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('accuracy'):\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))\n",
    "    correct_prediction = tf.cast(correct_prediction, tf.float32)\n",
    "accuracy = tf.reduce_mean(correct_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predicted = tf.argmax(y_conv, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving graph to: /var/folders/0h/2s3xs7_d3mq83xk38vh_gbnh0000gn/T/tmpxvbawz9w\n",
      "INFO:tensorflow:Summary name conv1/Variable:0__grad is illegal; using conv1/Variable_0__grad instead.\n",
      "INFO:tensorflow:Summary name conv1/Variable_1:0__grad is illegal; using conv1/Variable_1_0__grad instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable:0__grad is illegal; using conv2/Variable_0__grad instead.\n",
      "INFO:tensorflow:Summary name conv2/Variable_1:0__grad is illegal; using conv2/Variable_1_0__grad instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable:0__grad is illegal; using fc1/Variable_0__grad instead.\n",
      "INFO:tensorflow:Summary name fc1/Variable_1:0__grad is illegal; using fc1/Variable_1_0__grad instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable:0__grad is illegal; using fc2/Variable_0__grad instead.\n",
      "INFO:tensorflow:Summary name fc2/Variable_1:0__grad is illegal; using fc2/Variable_1_0__grad instead.\n",
      "step 0, training accuracy 0.07\n",
      "step 100, training accuracy 0.38\n",
      "step 200, training accuracy 0.45\n",
      "step 300, training accuracy 0.42\n",
      "step 400, training accuracy 0.52\n",
      "step 500, training accuracy 0.58\n",
      "step 600, training accuracy 0.59\n",
      "step 700, training accuracy 0.59\n",
      "step 800, training accuracy 0.5\n",
      "step 900, training accuracy 0.57\n",
      "step 1000, training accuracy 0.64\n",
      "step 1100, training accuracy 0.61\n",
      "step 1200, training accuracy 0.51\n",
      "step 1300, training accuracy 0.65\n",
      "step 1400, training accuracy 0.59\n",
      "step 1500, training accuracy 0.61\n",
      "step 1600, training accuracy 0.66\n",
      "step 1700, training accuracy 0.67\n",
      "step 1800, training accuracy 0.72\n",
      "step 1900, training accuracy 0.55\n",
      "step 2000, training accuracy 0.76\n",
      "step 2100, training accuracy 0.72\n",
      "step 2200, training accuracy 0.66\n",
      "step 2300, training accuracy 0.69\n",
      "step 2400, training accuracy 0.72\n",
      "step 2500, training accuracy 0.75\n",
      "step 2600, training accuracy 0.76\n",
      "step 2700, training accuracy 0.72\n",
      "step 2800, training accuracy 0.81\n",
      "step 2900, training accuracy 0.71\n",
      "step 3000, training accuracy 0.69\n",
      "step 3100, training accuracy 0.74\n",
      "step 3200, training accuracy 0.73\n",
      "step 3300, training accuracy 0.77\n",
      "step 3400, training accuracy 0.71\n",
      "step 3500, training accuracy 0.83\n",
      "step 3600, training accuracy 0.75\n",
      "step 3700, training accuracy 0.73\n",
      "step 3800, training accuracy 0.76\n",
      "step 3900, training accuracy 0.76\n",
      "step 4000, training accuracy 0.81\n",
      "step 4100, training accuracy 0.74\n",
      "step 4200, training accuracy 0.79\n",
      "step 4300, training accuracy 0.87\n",
      "step 4400, training accuracy 0.77\n",
      "step 4500, training accuracy 0.77\n",
      "step 4600, training accuracy 0.83\n",
      "step 4700, training accuracy 0.79\n",
      "step 4800, training accuracy 0.8\n",
      "step 4900, training accuracy 0.84\n",
      "validation accuracy 0.7166\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    \n",
    "    graph_location = tempfile.mkdtemp()\n",
    "    #train_writer = tf.summary.FileWriter(graph_location)\n",
    "    #train_writer.add_graph(tf.get_default_graph())\n",
    "    print('Saving graph to: %s' % graph_location)\n",
    "    train_writer = tf.summary.FileWriter(graph_location,sess.graph)\n",
    "    \n",
    "    \n",
    "    if OPTIMIZER=='ADAM':\n",
    "      tf.summary.scalar('train__learning_rate', optimizer._lr)\n",
    "        \n",
    "    if OPTIMIZER=='SGD':\n",
    "      tf.summary.scalar('train__learning_rate', optimizer._learning_rate)\n",
    "        \n",
    "    #Make histograms of last2 dense layers weights GRADIENTS:\n",
    "    trainables = tf.trainable_variables()\n",
    "\n",
    "    #Make hists of gradients\n",
    "    for i, Variable in enumerate(trainables):\n",
    "      tf.summary.histogram(Variable.name+'__grad',optimizer.compute_gradients(cross_entropy, Variable))\n",
    "\n",
    "    #Add loss, accuracy to tensorboard:\n",
    "    tf.summary.scalar('train__loss', cross_entropy)\n",
    "\n",
    "    tf.train.SummarySaverHook(save_steps=50, output_dir=graph_location,\n",
    "                          summary_op=tf.summary.merge_all, summary_writer=train_writer)\n",
    "    \n",
    "    \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i in range(5000):\n",
    "      batch_xs, batch_ys = get_batch(X_train, one_hot(y_train), 100)\n",
    "      if i % 100 == 0:   \n",
    "        train_accuracy = accuracy.eval(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 1.0}) \n",
    "        print('step %d, training accuracy %g' % (i, train_accuracy))\n",
    "      train_step.run(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 0.4})\n",
    "\n",
    "    print('validation accuracy %g' % accuracy.eval(feed_dict={\n",
    "        x: X_validation.T, y_: one_hot(y_validation).T, keep_prob: 1.0}))\n",
    "    save_predictions('ans1-uni', y_predicted.eval(feed_dict={x: X_test.T, keep_prob: 1.0})) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([3, 1, 0, 4, 5, 0, 8, 4, 8, 1])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test if your numpy file has been saved correctly\n",
    "loaded_y = np.load('ans1-uni.npy')\n",
    "print(loaded_y.shape)\n",
    "loaded_y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}