minor changes

Author: Tinusf

assignment1/config.txt

@@ -7,12 +7,12 @@ validation=./DATA/validate_small.csv
 # layers is a comma-separated list of integers telling us how many nodes in each
 # hidden layer. Special case: If the value is only one element in the list, and
 # its value is 0, you should generate a net without a hidden layer
-layers = 400, 400, 400, 100
+layers = 24, 24
 
 # activations is a comma-separated list of key-words. It will have as many
 # elements as there are elements in the layers-list. Each keyword is a
 # non-linearity function, and legal values are relu, linear, and tanh.
-activations = relu, relu, relu, relu
+activations = relu, relu
 
 # loss_type chooses between L2 loss (for regression) and
 # cross_entropy (for classification).
@@ -26,6 +26,6 @@ learning_rate=5.e-4
 no_epochs=500
 
 # What L2-regularization to use to avoid overfitting.
-L2_regularization=5.E-5
+L2_regularization=0
 
 

assignment1/network.py

@@ -56,8 +56,8 @@ def get_l2_regularization(self, derivate=False, weights=False):
             return l2_derivate_matrix
         else:
             all_weights_squared = np.sum(np.sum(layer.w ** 2) for layer in self.layers)
-            all_biases_squared = np.sum(np.sum(layer.b ** 2) for layer in self.layers)
-            return self.regularization_factor * (all_weights_squared)
+            # all_biases_squared = np.sum(np.sum(layer.b ** 2) for layer in self.layers)
+            return self.regularization_factor * all_weights_squared
 
     def get_loss(self, layer, target_y, estimate_y, derivate=False):
         if layer.loss == "L2":
@@ -74,9 +74,9 @@ def get_loss(self, layer, target_y, estimate_y, derivate=False):
                 print("Lol")
             if derivate:
                 derivate = estimate_y - target_y
-                derivate[(derivate >= -0.000001) & (derivate <= 0.000001)] = 0
-                derivate[(derivate >= 0.999)] = 1
-                derivate[(derivate <= -0.999)] = -1
+                # derivate[(derivate >= -0.000001) & (derivate <= 0.000001)] = 0
+                # derivate[(derivate >= 0.999)] = 1
+                # derivate[(derivate <= -0.999)] = -1
                 return derivate
             return loss
 
@@ -127,8 +127,7 @@ def back_propagation(self, activations, target_y, zs, learning_rate=0.0001):
                 layer.w = layer.w - (learning_rate * np.array(last_error).dot(np.transpose(
                     activations[layer_i])) + self.regularization_factor * layer.w)
 
-                layer.b = layer.b - (learning_rate * last_error + self.regularization_factor *
-                                     layer.b)
+                layer.b = layer.b - (learning_rate * last_error)
             else:
                 layer.b = layer.b - (learning_rate * last_error)
                 layer.w = layer.w - (learning_rate * np.array(last_error).dot(np.transpose(