cnn

Author: lazyprogrammer

cnn_class/benchmark.py

@@ -0,0 +1,129 @@
+# Vanilla deep network
+
+# https://udemy.com/deep-learning-convolutional-neural-networks-theano-tensorflow
+
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+from scipy.io import loadmat
+from sklearn.utils import shuffle
+
+
+def y2indicator(y):
+    N = len(y)
+    ind = np.zeros((N, 10))
+    for i in xrange(N):
+        ind[i, y[i]] = 1
+    return ind
+
+
+def error_rate(p, t):
+    return np.mean(p != t)
+
+
+def flatten(X):
+    # input will be (32, 32, 3, N)
+    # output will be (N, 3072)
+    N = X.shape[-1]
+    flat = np.zeros((N, 3072))
+    for i in xrange(N):
+        flat[i] = X[:,:,:,i].reshape(3072)
+    return flat
+
+# In [6]: train['X'].shape
+# Out[6]: (32, 32, 3, 73257)
+
+# In [7]: train['y'].shape
+# Out[7]: (73257, 1)
+
+# In [8]: set(train['y'].flatten().tolist())
+# Out[8]: {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+# We will change these to 0..9 to be 0-indexed
+
+# In [12]: test['X'].shape
+# Out[12]: (32, 32, 3, 26032)
+
+# In [13]: test['y'].shape
+# Out[13]: (26032, 1)
+
+def main():
+    train = loadmat('../large_files/train_32x32.mat')
+    test  = loadmat('../large_files/test_32x32.mat')
+
+    # Need to scale! don't leave as 0..255
+    # Y is a N x 1 matrix with values 1..10 (MATLAB indexes by 1)
+    # So flatten it and make it 0..9
+    # Also need indicator matrix for cost calculation
+    Xtrain = flatten(train['X'].astype(np.float32) / 255)
+    Ytrain = train['y'].flatten() - 1
+    Xtrain, Ytrain = shuffle(Xtrain, Ytrain)
+    Ytrain_ind = y2indicator(Ytrain)
+
+    Xtest  = flatten(test['X'].astype(np.float32) / 255)
+    Ytest  = test['y'].flatten() - 1
+    Ytest_ind  = y2indicator(Ytest)
+
+    # gradient descent params
+    max_iter = 20
+    print_period = 10
+    N, D = Xtrain.shape
+    batch_sz = 500
+    n_batches = N / batch_sz
+
+    # initial weights
+    M1 = 1000 # hidden layer size
+    M2 = 500
+    K = 10
+    W1_init = np.random.randn(D, M1) / np.sqrt(D + M1)
+    b1_init = np.zeros(M1)
+    W2_init = np.random.randn(M1, M2) / np.sqrt(M1 + M2)
+    b2_init = np.zeros(M2)
+    W3_init = np.random.randn(M2, K) / np.sqrt(M2 + K)
+    b3_init = np.zeros(K)
+
+    # define variables and expressions
+    X = tf.placeholder(tf.float32, shape=(None, D), name='X')
+    T = tf.placeholder(tf.float32, shape=(None, K), name='T')
+    W1 = tf.Variable(W1_init.astype(np.float32))
+    b1 = tf.Variable(b1_init.astype(np.float32))
+    W2 = tf.Variable(W2_init.astype(np.float32))
+    b2 = tf.Variable(b2_init.astype(np.float32))
+    W3 = tf.Variable(W3_init.astype(np.float32))
+    b3 = tf.Variable(b3_init.astype(np.float32))
+
+    Z1 = tf.nn.relu( tf.matmul(X, W1) + b1 )
+    Z2 = tf.nn.relu( tf.matmul(Z1, W2) + b2 )
+    Yish = tf.matmul(Z2, W3) + b3
+
+    cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(Yish, T))
+
+    train_op = tf.train.RMSPropOptimizer(0.0001, decay=0.99, momentum=0.9).minimize(cost)
+
+    # we'll use this to calculate the error rate
+    predict_op = tf.argmax(Yish, 1)
+
+    LL = []
+    init = tf.initialize_all_variables()
+    with tf.Session() as session:
+        session.run(init)
+
+        for i in xrange(max_iter):
+            for j in xrange(n_batches):
+                Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
+                Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
+
+                session.run(train_op, feed_dict={X: Xbatch, T: Ybatch})
+                if j % print_period == 0:
+                    test_cost = session.run(cost, feed_dict={X: Xtest, T: Ytest_ind})
+                    prediction = session.run(predict_op, feed_dict={X: Xtest})
+                    err = error_rate(prediction, Ytest)
+                    print "Cost / err at iteration i=%d, j=%d: %.3f / %.3f" % (i, j, test_cost, err)
+                    LL.append(test_cost)
+
+    plt.plot(LL)
+    plt.show()
+
+
+if __name__ == '__main__':
+    main()

cnn_class/blur.py

@@ -0,0 +1,54 @@
+# https://udemy.com/deep-learning-convolutional-neural-networks-theano-tensorflow
+
+import numpy as np
+from scipy.signal import convolve2d
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+
+# load the famous Lena image
+img = mpimg.imread('lena.png')
+
+# what does it look like?
+plt.imshow(img)
+plt.show()
+
+# make it B&W
+bw = img.mean(axis=2)
+plt.imshow(bw, cmap='gray')
+plt.show()
+
+# create a Gaussian filter
+W = np.zeros((20, 20))
+for i in xrange(20):
+    for j in xrange(20):
+        dist = (i - 9.5)**2 + (j - 9.5)**2
+        W[i, j] = np.exp(-dist / 50.)
+
+# let's see what the filter looks like
+plt.imshow(W, cmap='gray')
+plt.show()
+
+# now the convolution
+out = convolve2d(bw, W)
+plt.imshow(out, cmap='gray')
+plt.show()
+
+# what's that weird black stuff on the edges? let's check the size of output
+print out.shape
+# after convolution, the output signal is N1 + N2 - 1
+
+
+# we can also just make the output the same size as the input
+out = convolve2d(bw, W, mode='same')
+plt.imshow(out, cmap='gray')
+plt.show()
+print out.shape
+
+
+# in color
+out3 = np.zeros(img.shape)
+for i in xrange(3):
+    out3[:,:,i] = convolve2d(img[:,:,i], W, mode='same')
+plt.imshow(out3)
+plt.show() # does not look like anything
+

cnn_class/cnn_tf.py

@@ -0,0 +1,172 @@
+# New concepts and differences from Theano:
+# - stride is the interval at which to apply the convolution
+# - unlike previous course, we use constant-size input to the network
+#   since not doing that caused us to start swapping
+# - the output after convpool is a different size (8,8) here, (5,5) in Theano
+
+# https://udemy.com/deep-learning-convolutional-neural-networks-theano-tensorflow
+
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+from datetime import datetime
+from scipy.signal import convolve2d
+from scipy.io import loadmat
+from sklearn.utils import shuffle
+
+
+def y2indicator(y):
+    N = len(y)
+    ind = np.zeros((N, 10))
+    for i in xrange(N):
+        ind[i, y[i]] = 1
+    return ind
+
+
+def error_rate(p, t):
+    return np.mean(p != t)
+
+
+def convpool(X, W, b):
+    # just assume pool size is (2,2) because we need to augment it with 1s
+    conv_out = tf.nn.conv2d(X, W, strides=[1, 1, 1, 1], padding='SAME')
+    conv_out = tf.nn.bias_add(conv_out, b)
+    pool_out = tf.nn.max_pool(conv_out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
+    return pool_out
+
+
+def init_filter(shape, poolsz):
+    w = np.random.randn(*shape) / np.sqrt(np.prod(shape[:-1]) + shape[-1]*np.prod(shape[:-2] / np.prod(poolsz)))
+    return w.astype(np.float32)
+
+
+def rearrange(X):
+    # input is (32, 32, 3, N)
+    # output is (N, 32, 32, 3)
+    N = X.shape[-1]
+    out = np.zeros((N, 32, 32, 3), dtype=np.float32)
+    for i in xrange(N):
+        for j in xrange(3):
+            out[i, :, :, j] = X[:, :, j, i]
+    return out / 255
+
+
+def main():
+    train = loadmat('../large_files/train_32x32.mat') # N = 73257
+    test  = loadmat('../large_files/test_32x32.mat') # N = 26032
+
+    # Need to scale! don't leave as 0..255
+    # Y is a N x 1 matrix with values 1..10 (MATLAB indexes by 1)
+    # So flatten it and make it 0..9
+    # Also need indicator matrix for cost calculation
+    Xtrain = rearrange(train['X'])
+    Ytrain = train['y'].flatten() - 1
+    print len(Ytrain)
+    del train
+    Xtrain, Ytrain = shuffle(Xtrain, Ytrain)
+    Ytrain_ind = y2indicator(Ytrain)
+
+    Xtest  = rearrange(test['X'])
+    Ytest  = test['y'].flatten() - 1
+    del test
+    Ytest_ind  = y2indicator(Ytest)
+
+    # gradient descent params
+    max_iter = 20
+    print_period = 10
+    N = Xtrain.shape[0]
+    batch_sz = 500
+    n_batches = N / batch_sz
+
+    # limit samples since input will always have to be same size
+    # you could also just do N = N / batch_sz * batch_sz
+    Xtrain = Xtrain[:73000,]
+    Ytrain = Ytrain[:73000]
+    Xtest = Xtest[:26000,]
+    Ytest = Ytest[:26000]
+    Ytest_ind = Ytest_ind[:26000,]
+    # print "Xtest.shape:", Xtest.shape
+    # print "Ytest.shape:", Ytest.shape
+
+    # initial weights
+    M = 500
+    K = 10
+    poolsz = (2, 2)
+
+    W1_shape = (5, 5, 3, 20) # (filter_width, filter_height, num_color_channels, num_feature_maps)
+    W1_init = init_filter(W1_shape, poolsz)
+    b1_init = np.zeros(W1_shape[-1], dtype=np.float32) # one bias per output feature map
+
+    W2_shape = (5, 5, 20, 50) # (filter_width, filter_height, old_num_feature_maps, num_feature_maps)
+    W2_init = init_filter(W2_shape, poolsz)
+    b2_init = np.zeros(W2_shape[-1], dtype=np.float32)
+
+    # vanilla ANN weights
+    W3_init = np.random.randn(W2_shape[-1]*8*8, M) / np.sqrt(W2_shape[-1]*8*8 + M)
+    b3_init = np.zeros(M, dtype=np.float32)
+    W4_init = np.random.randn(M, K) / np.sqrt(M + K)
+    b4_init = np.zeros(K, dtype=np.float32)
+
+
+    # define variables and expressions
+    # using None as the first shape element takes up too much RAM unfortunately
+    X = tf.placeholder(tf.float32, shape=(batch_sz, 32, 32, 3), name='X')
+    T = tf.placeholder(tf.float32, shape=(batch_sz, K), name='T')
+    W1 = tf.Variable(W1_init.astype(np.float32))
+    b1 = tf.Variable(b1_init.astype(np.float32))
+    W2 = tf.Variable(W2_init.astype(np.float32))
+    b2 = tf.Variable(b2_init.astype(np.float32))
+    W3 = tf.Variable(W3_init.astype(np.float32))
+    b3 = tf.Variable(b3_init.astype(np.float32))
+    W4 = tf.Variable(W4_init.astype(np.float32))
+    b4 = tf.Variable(b4_init.astype(np.float32))
+
+    Z1 = convpool(X, W1, b1)
+    Z2 = convpool(Z1, W2, b2)
+    Z2_shape = Z2.get_shape().as_list()
+    Z2r = tf.reshape(Z2, [Z2_shape[0], np.prod(Z2_shape[1:])])
+    Z3 = tf.nn.relu( tf.matmul(Z2r, W3) + b3 )
+    Yish = tf.matmul(Z3, W4) + b4
+
+    cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(Yish, T))
+
+    train_op = tf.train.RMSPropOptimizer(0.0001, decay=0.99, momentum=0.9).minimize(cost)
+
+    # we'll use this to calculate the error rate
+    predict_op = tf.argmax(Yish, 1)
+
+    t0 = datetime.now()
+    LL = []
+    init = tf.initialize_all_variables()
+    with tf.Session() as session:
+        session.run(init)
+
+        for i in xrange(max_iter):
+            for j in xrange(n_batches):
+                Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
+                Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
+
+                if len(Xbatch) == batch_sz:
+                    session.run(train_op, feed_dict={X: Xbatch, T: Ybatch})
+                    if j % print_period == 0:
+                        # due to RAM limitations we need to have a fixed size input
+                        # so as a result, we have this ugly total cost and prediction computation
+                        test_cost = 0
+                        prediction = np.zeros(len(Xtest))
+                        for k in xrange(len(Xtest) / batch_sz):
+                            Xtestbatch = Xtest[k*batch_sz:(k*batch_sz + batch_sz),]
+                            Ytestbatch = Ytest_ind[k*batch_sz:(k*batch_sz + batch_sz),]
+                            test_cost += session.run(cost, feed_dict={X: Xtestbatch, T: Ytestbatch})
+                            prediction[k*batch_sz:(k*batch_sz + batch_sz)] = session.run(
+                                predict_op, feed_dict={X: Xtestbatch})
+                        err = error_rate(prediction, Ytest)
+                        print "Cost / err at iteration i=%d, j=%d: %.3f / %.3f" % (i, j, test_cost, err)
+                        LL.append(test_cost)
+    print "Elapsed time:", (datetime.now() - t0)
+    plt.plot(LL)
+    plt.show()
+
+
+if __name__ == '__main__':
+    main()