Added explanation for 1b and added logging and bash script commands

Author: Vineet John

assignments/a5/a5.tex

@@ -54,11 +54,42 @@ \section{Tensorflow - MNIST} % (fold)
             \label{tab:accuracy_comparisons_i}
         \end{table}
 
-        \textbf{Discussion:}
+        \textbf{Discussion:}\\
         The reason than the CNN performs the best is because it considers patches of the image rather than the intensities of sequential pixels. Since the CNN preserves this 2-D information structure, it performs the best of the lot.
 
     % subsection softmax_vs_cnn_vs_fully_connected (end)
 
+    \subsection{CNN - ReLU vs. Sigmoid Units} % (fold)
+    \label{sub:cnn_relu_vs_sigmoid_units}
+
+        \begin{table}[th]
+            \centering
+            \begin{tabular}{| l | r |}
+            \hline
+            \textbf{Neural Net Unit Type} & \textbf{Accuracy} \\
+            \hline
+                \hline
+                Rectified Linear Units & 0.9664 \\
+                \hline
+                Sigmoid Units & 0.7903 \\
+            \hline
+            \end{tabular}
+            \caption{Accuracy Comparisons - II}
+            \label{tab:accuracy_comparisons_ii}
+        \end{table}
+
+        \textbf{Discussion:}\\
+        ReLUs were found to greatly accelerate the convergence of stochastic gradient descent compared to the sigmoid function. It is argued that this is due to its linear, non-saturating form. (Saturation happens for the sigmoid function at the tail of it's outputs 0 and 1, where the gradient is almost 0.)
+    
+    % subsection cnn_relu_vs_sigmoid_units (end)
+
+    \subsection{Tweaking Dropout Level} % (fold)
+    \label{sub:tweaking_dropout_level}
+
+        
+    
+    % subsection tweaking_dropout_level (end)
+
 % section tensorflow_mnist (end)
 
 

tensorflow-mnist/processors/cnn_processor.py

@@ -56,6 +56,7 @@ def process(self):
         accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
         sess.run(tf.global_variables_initializer())
         for i in range(1001):
+            log.info("Training iteration: " + str(i))
             batch = mnist.train.next_batch(50)
             if i % 100 == 0:
                 train_accuracy = accuracy.eval(feed_dict={

tensorflow-mnist/scripts/run-tensorflow-mnist.sh

@@ -7,3 +7,6 @@ CODEDIR=$(dirname "$0")"/../"
 
 # Run Convolutional Neural Network learning processor
 /usr/bin/python3 "$CODEDIR"/tensorflow_mnist.py --mode cnn
+
+# Run Fully-Connected Feedforward neural net
+/usr/bin/python3 "$CODEDIR"/processors/fully_connected_network_processor.py