fix debug deconv swapaxes

wuziheng · Feb 16, 2018 · 8c7ec7d · 8c7ec7d
1 parent d9e7b55
commit 8c7ec7d
Show file tree

Hide file tree

Showing 21 changed files with 434 additions and 263 deletions.
diff --git a/.idea/CNN-Numpy.iml b/.idea/CNN-Numpy.iml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/other.xml b/.idea/other.xml
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/layers/__init__.pyc b/layers/__init__.pyc
diff --git a/layers/base_conv.py b/layers/base_conv.py
@@ -46,7 +46,6 @@ def forward(self, x):
         for i in range(self.batchsize):
             img_i = x[i][np.newaxis, :]
             self.col_image_i = im2col(img_i, self.ksize, self.stride)
-            #print self.col_image_i.shape
             conv_out[i] = np.reshape(np.dot(self.col_image_i, col_weights) + self.bias, self.eta[0].shape)
             self.col_image.append(self.col_image_i)
         self.col_image = np.array(self.col_image)
@@ -71,9 +70,9 @@ def gradient(self, eta):
                 (0, 0), (self.ksize / 2, self.ksize / 2), (self.ksize / 2, self.ksize / 2), (0, 0)),
                              'constant', constant_values=0)
 
-        col_pad_eta = np.array([im2col(pad_eta[i][np.newaxis, :], self.ksize, self.stride) for i in range(self.batchsize)])
         flip_weights = np.flipud(np.fliplr(self.weights))
         col_flip_weights = flip_weights.reshape([-1, self.input_channels])
+        col_pad_eta = np.array([im2col(pad_eta[i][np.newaxis, :], self.ksize, self.stride) for i in range(self.batchsize)])
         next_eta = np.dot(col_pad_eta, col_flip_weights)
         next_eta = np.reshape(next_eta, self.input_shape)
         return next_eta

diff --git a/layers/base_conv.pyc b/layers/base_conv.pyc
diff --git a/layers/bn.py b/layers/bn.py
@@ -7,7 +7,7 @@ def __init__(self, shape):
         self.batch_size = shape[0]
         self.input_data = np.zeros(shape)
 
-        self.alpha = np.zeros(shape[-1])
+        self.alpha = np.ones(shape[-1])
         self.beta = np.zeros(shape[-1])
         self.a_gradient = np.zeros(shape[-1])
         self.b_gradient = np.zeros(shape[-1])
@@ -57,3 +57,23 @@ def backward(self, alpha=0.0001):
         self.alpha -= alpha * self.a_gradient
         self.beta -= alpha * self.b_gradient
 
+
+if __name__ == "__main__":
+    shape = [12,3]
+    bn = BatchNorm(shape)
+    print 'batch:', bn.batch_size
+
+    a = np.random.random(shape)
+    epsilon = 1e-4
+
+    out1 = bn.forward(a+epsilon)
+    out2 = bn.forward(a-epsilon)
+    # print bn.mean
+    # print bn.var
+    # print bn.moving_mean
+    # print bn.moving_var
+    # print bn.mean.shape
+    print out1
+    # print (out1-out2)/(2*epsilon)
+
+    print bn.gradient(np.ones(shape))
diff --git a/layers/pooling.py b/layers/pooling.py
@@ -12,6 +12,13 @@ def __init__(self, shape, ksize=2, stride=2):
         self.integral = np.zeros(shape)
         self.index = np.zeros(shape)
 
+    def gradient(self, eta):
+        # stride = ksize
+        next_eta = np.repeat(eta, self.stride, axis=1)
+        next_eta = np.repeat(next_eta, self.stride, axis=2)
+        next_eta = next_eta*self.index
+        return next_eta/(self.ksize*self.ksize)
+
     def forward(self, x):
         for b in range(x.shape[0]):
             for c in range(self.output_channels):
@@ -55,12 +62,7 @@ def forward(self, x):
         out /= (self.ksize * self.ksize)
         return out
 
-    def gradient(self, eta):
-        # stride = ksize
-        next_eta = np.repeat(eta, self.stride, axis=1)
-        next_eta = np.repeat(next_eta, self.stride, axis=2)
-        next_eta = next_eta*self.index
-        return next_eta/(self.ksize*self.ksize)
+
 
 
 class MaxPooling(object):

diff --git a/layers/pooling.pyc b/layers/pooling.pyc
diff --git a/lenet_tensor.py b/lenet_tensor.py
@@ -10,7 +10,7 @@
 from glob import glob
 import os
 
-VERSION = 'TENSOR_Adam_Dropout0.7_RELU'
+VERSION = 'TENSOR_Adam_RELU'
 
 
 def load_mnist(path, kind='train'):
@@ -36,13 +36,13 @@ def load_mnist(path, kind='train'):
 def inference(x, output_num):
     conv1_out = op.Conv2D((5, 5, 1, 12), input_variable=x, name='conv1', padding='VALID').output_variables
     relu1_out = activation.Relu(input_variable=conv1_out, name='relu1').output_variables
-    dropout1_out = op.DropOut(input_variable=relu1_out, name='dropout1', phase='train', prob=0.7).output_variables
-    pool1_out = op.MaxPooling(ksize=2, input_variable=dropout1_out, name='pool1').output_variables
+    #dropout1_out = op.DropOut(input_variable=relu1_out, name='dropout1', phase='train', prob=0.7).output_variables
+    pool1_out = op.MaxPooling(ksize=2, input_variable=relu1_out, name='pool1').output_variables
 
     conv2_out = op.Conv2D((3, 3, 12, 24), input_variable=pool1_out, name='conv2').output_variables
     relu2_out = activation.Relu(input_variable=conv2_out, name='relu2').output_variables
-    dropout2_out = op.DropOut(input_variable=relu2_out, name='dropout2', phase='train', prob=0.7).output_variables
-    pool2_out = op.MaxPooling(ksize=2, input_variable=dropout2_out, name='pool2').output_variables
+    #dropout2_out = op.DropOut(input_variable=relu2_out, name='dropout2', phase='train', prob=0.7).output_variables
+    pool2_out = op.MaxPooling(ksize=2, input_variable=relu1_out, name='pool2').output_variables
 
     fc_out = op.FullyConnect(output_num=output_num, input_variable=pool2_out, name='fc').output_variables
     return fc_out
@@ -70,7 +70,7 @@ def inference(x, output_num):
 # save train curve config
 loss_collect = []
 acc_collect = []
-
+print 'new'
 with open('logs/%s_log.txt'%VERSION, 'wb') as logf:
     for epoch in range(20):
         # random shuffle
@@ -154,7 +154,7 @@ def inference(x, output_num):
                     s.wait_bp = False
                 if isinstance(s, op.Operator):
                     s.wait_forward = True
-                if isinstance(s,op.Operator) and s.__getattribute__('phase') == 'train':
+                if isinstance(s,op.Operator) and hasattr(s,'phase'): #== 'train':
                     s.phase = 'test'
 
             _loss = sf.loss.eval()
@@ -171,5 +171,5 @@ def inference(x, output_num):
 
         for k in var.GLOBAL_VARIABLE_SCOPE:
             s = var.GLOBAL_VARIABLE_SCOPE[k]
-            if isinstance(s, op.Operator) and s.__getattribute__('phase') == 'test':
+            if isinstance(s, op.Operator) and hasattr(s,'phase') :#== 'test':
                 s.phase = 'train'
diff --git a/logs/TENSOR_Adam_Dropout0.7_RELU_log.txt b/logs/TENSOR_Adam_Dropout0.7_RELU_log.txt
@@ -0,0 +1,18 @@
+2018-02-16 22:45:03  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:    50 , avg_batch_acc: 0.2188  avg_batch_loss: 3.4016  learning_rate 0.000500
+2018-02-16 22:47:25  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   100 , avg_batch_acc: 0.2500  avg_batch_loss: 2.6438  learning_rate 0.000500
+2018-02-16 22:50:03  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   150 , avg_batch_acc: 0.2344  avg_batch_loss: 1.9562  learning_rate 0.000500
+2018-02-16 22:52:44  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   200 , avg_batch_acc: 0.5625  avg_batch_loss: 1.3707  learning_rate 0.000500
+2018-02-16 22:55:24  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   250 , avg_batch_acc: 0.5312  avg_batch_loss: 1.3155  learning_rate 0.000500
+2018-02-16 22:58:02  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   300 , avg_batch_acc: 0.6250  avg_batch_loss: 1.0030  learning_rate 0.000500
+2018-02-16 23:00:41  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   350 , avg_batch_acc: 0.6875  avg_batch_loss: 1.1672  learning_rate 0.000500
+2018-02-16 23:03:20  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   400 , avg_batch_acc: 0.6719  avg_batch_loss: 0.8919  learning_rate 0.000500
+2018-02-16 23:05:59  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   450 , avg_batch_acc: 0.6875  avg_batch_loss: 0.6986  learning_rate 0.000500
+2018-02-16 23:08:37  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   500 , avg_batch_acc: 0.7969  avg_batch_loss: 0.5889  learning_rate 0.000500
+2018-02-16 23:11:18  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   550 , avg_batch_acc: 0.7812  avg_batch_loss: 0.6626  learning_rate 0.000500
+2018-02-16 23:13:58  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   600 , avg_batch_acc: 0.7500  avg_batch_loss: 0.5585  learning_rate 0.000500
+2018-02-16 23:16:42  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   650 , avg_batch_acc: 0.8125  avg_batch_loss: 0.6408  learning_rate 0.000500
+2018-02-16 23:19:23  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   700 , avg_batch_acc: 0.7969  avg_batch_loss: 0.6765  learning_rate 0.000500
+2018-02-16 23:22:03  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   750 , avg_batch_acc: 0.7969  avg_batch_loss: 0.5936  learning_rate 0.000500
+2018-02-16 23:24:39  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   800 , avg_batch_acc: 0.8750  avg_batch_loss: 0.3763  learning_rate 0.000500
+2018-02-16 23:27:18  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   850 , avg_batch_acc: 0.9062  avg_batch_loss: 0.4242  learning_rate 0.000500
+2018-02-16 23:29:56  TENSOR_Adam_Dropout0.7_RELU epoch: 0 ,  batch:   900 , avg_batch_acc: 0.8125  avg_batch_loss: 0.5433  learning_rate 0.000500
diff --git a/logs/TENSOR_Adam_RELU_log.txt b/logs/TENSOR_Adam_RELU_log.txt
diff --git a/plot.pyc b/plot.pyc
diff --git a/tensor/Activation.pyc b/tensor/Activation.pyc
diff --git a/tensor/Operator.py b/tensor/Operator.py
@@ -128,6 +128,7 @@ def _deconv(self, input=Variable, output=Variable, weights=Variable, bias=Variab
 
         col_pad_eta = np.array([im2col(pad_eta[i][np.newaxis, :], self.ksize, self.stride) for i in range(self.batch_size)])
         flip_weights = np.flipud(np.fliplr(weights.data))
+        flip_weights = flip_weights.swapaxes(2,3)
         col_flip_weights = flip_weights.reshape([-1, weights.shape[2]])
         next_eta = np.dot(col_pad_eta, col_flip_weights)
         next_eta = np.reshape(next_eta, input.shape)

diff --git a/tensor/Operator.pyc b/tensor/Operator.pyc
diff --git a/tensor/Variable.pyc b/tensor/Variable.pyc
diff --git a/tensor/__init__.pyc b/tensor/__init__.pyc
diff --git a/tensor/util.pyc b/tensor/util.pyc
diff --git a/test_conv.py b/test_conv.py
@@ -1,63 +1,96 @@
 import tensor.Variable as var
 import tensor.Operator as op
 
-from layers.base_conv import Conv2D
-from layers.pooling import MaxPooling
-from layers.fc import FullyConnect
-from layers.relu import Relu
-from layers.softmax import Softmax
+# from layers.base_conv import Conv2D
+# from layers.pooling import MaxPooling
+# from layers.fc import FullyConnect
+# from layers.relu import Relu
+# from layers.softmax import Softmax
 
-import cv2
+# import cv2
 import numpy as np
 
-img = cv2.imread('layers/test.jpg')
-img = img[np.newaxis, :]
+# img = cv2.imread('layers/test.jpg')
+# img = img[np.newaxis, :]
 
+e=1e-3
 a = var.Variable((1, 128, 128, 3), 'a')
-label = var.Variable([1, 1], 'label')
-import random
-label.data = np.array([random.randint(1,9)])
-label.data = label.data.astype(int)
+b = var.Variable((1, 128, 128, 3), 'b')
 
+b.data = a.data.copy()
+a.data[0,0,0,1] += e
+b.data[0,0,0,1] -= e
+
+# label = var.Variable([1, 1], 'label')
+# import random
+# label.data = np.array([random.randint(1,9)])
+# label.data = label.data.astype(int)
+
+import numpy as np
 conv1_out = op.Conv2D((3, 3, 3, 3), input_variable=a, name='conv1',padding='VALID').output_variables
-relu1_out = op.Relu(input_variable=conv1_out, name='relu1').output_variables
-pool1_out = op.MaxPooling(ksize=2, input_variable=relu1_out, name='pool1').output_variables
-fc1_out = op.FullyConnect(output_num=10, input_variable=pool1_out, name='fc1').output_variables
-sf_out = op.SoftmaxLoss(predict=fc1_out,label=label,name='sf').loss
+conv2_out = op.Conv2D((3, 3, 3, 3), input_variable=b, name='conv2',padding='VALID').output_variables
 
-new_conv1 = op.GLOBAL_VARIABLE_SCOPE['conv1']
-new_fc1 = op.GLOBAL_VARIABLE_SCOPE['fc1']
+conv1 = var.GLOBAL_VARIABLE_SCOPE['conv1']
+conv2 = var.GLOBAL_VARIABLE_SCOPE['conv2']
+var.GLOBAL_VARIABLE_SCOPE['conv1'].weights.data = var.GLOBAL_VARIABLE_SCOPE['conv2'].weights.data
+var.GLOBAL_VARIABLE_SCOPE['conv1'].bias.data = var.GLOBAL_VARIABLE_SCOPE['conv2'].bias.data
 
+# print conv1.weights.data - conv2.weights.data
+# print conv1_out.eval()-conv2_out.eval()
 
-conv1 = Conv2D([1, 128, 128, 3], 3, 3, 1,method='VALID')
-relu1 = Relu(conv1.output_shape)
-pool1 = MaxPooling(conv1.output_shape)
-fc1 = FullyConnect(pool1.output_shape,10)
-sf = Softmax(fc1.output_shape)
 
-conv1.weights = new_conv1.weights.data
-conv1.bias = new_conv1.bias.data
-fc1.weights = new_fc1.weights.data
-fc1.bias = new_fc1.bias.data
+conv1_out.eval()
+conv1_out.diff.data = (np.ones(conv1_out.diff.shape))
+print a.wait_bp, conv1.wait_forward
 
+conv2_out.eval()
+conv2_out.diff.data = (np.ones(conv1_out.diff.shape))
+print b.wait_bp, conv2.wait_forward
 
-out = sf.cal_loss(fc1.forward(pool1.forward(relu1.forward(conv1.forward(img)))), label.data)
-sf.gradient()
-eta = conv1.gradient(relu1.gradient(pool1.gradient(fc1.gradient(sf.eta))))
+# print a.diff_eval()[0,0,0,1]
+# print
 
 
+
+# relu1_out = op.Relu(input_variable=conv1_out, name='relu1').output_variables
+# pool1_out = op.MaxPooling(ksize=2, input_variable=relu1_out, name='pool1').output_variables
+# fc1_out = op.FullyConnect(output_num=10, input_variable=pool1_out, name='fc1').output_variables
+# sf_out = op.SoftmaxLoss(predict=fc1_out,label=label,name='sf').loss
 #
-# new train op
-# give value and forward
-a.data = img
-new_out = sf_out.eval()
+# new_conv1 = op.GLOBAL_VARIABLE_SCOPE['conv1']
+# new_fc1 = op.GLOBAL_VARIABLE_SCOPE['fc1']
 
-# give diff and backward
-new_eta = a.diff_eval()
-print new_eta-eta
 
 
-for k in var.GLOBAL_VARIABLE_SCOPE:
-    s = var.GLOBAL_VARIABLE_SCOPE[k]
-    if isinstance(s,var.Variable) and s.learnable:
-        print s.name, s.parent, s.child
+# conv1 = Conv2D([1, 128, 128, 3], 3, 3, 1,method='VALID')
+# relu1 = Relu(conv1.output_shape)
+# pool1 = MaxPooling(conv1.output_shape)
+# fc1 = FullyConnect(pool1.output_shape,10)
+# sf = Softmax(fc1.output_shape)
+#
+# conv1.weights = new_conv1.weights.data
+# conv1.bias = new_conv1.bias.data
+# fc1.weights = new_fc1.weights.data
+# fc1.bias = new_fc1.bias.data
+#
+#
+# out = sf.cal_loss(fc1.forward(pool1.forward(relu1.forward(conv1.forward(img)))), label.data)
+# sf.gradient()
+# eta = conv1.gradient(relu1.gradient(pool1.gradient(fc1.gradient(sf.eta))))
+
+
+#
+# new train op
+# give value and forward
+# a.data = img
+# new_out = sf_out.eval()
+#
+# # give diff and backward
+# new_eta = a.diff_eval()
+# print new_eta-eta
+#
+#
+# for k in var.GLOBAL_VARIABLE_SCOPE:
+#     s = var.GLOBAL_VARIABLE_SCOPE[k]
+#     if isinstance(s,var.Variable) and s.learnable:
+#         print s.name, s.parent, s.child