fix load and save

Author: lazyprogrammer

unsupervised_class2/unsupervised.py

@@ -42,20 +42,24 @@ def forward(self, X):
             current_input = Z
         return current_input
 
-    def fit_to_input(self, k, learning_rate=1.0, epochs=100000):
+    def fit_to_input(self, k, learning_rate=1.0, mu=0.99, epochs=100000):
         # This is not very flexible, as you would ideally
         # like to be able to activate any node in any hidden
         # layer, not just the last layer.
         # Exercise for students: modify this function to be able
         # to activate neurons in the middle layers.
         X0 = init_weights((1, self.D))
         X = theano.shared(X0, 'X_shared')
+        dX = theano.shared(np.zeros(X0.shape), 'dX_shared')
         Y = self.forward(X)
         t = np.zeros(self.hidden_layers[-1].M)
         t[k] = 1
 
         cost = -(t*T.log(Y[0]) + (1 - t)*(T.log(1 - Y[0]))).sum()
-        updates = [(X, X - learning_rate*T.grad(cost, X))]
+        updates = [
+            (X, X + mu*dX - learning_rate*T.grad(cost, X)),
+            (dX, mu*dX - learning_rate*T.grad(cost, X)),
+        ]
         train = theano.function(
             inputs=[],
             outputs=cost,
@@ -74,19 +78,22 @@ def fit_to_input(self, k, learning_rate=1.0, epochs=100000):
         return X.eval()
 
     def save(self, filename):
-        arrays = [p.eval() for p in layer.params for layer in self.hidden_layers]
+        arrays = [p.eval() for layer in self.hidden_layers for p in layer.params]
         np.savez(filename, *arrays)
 
     @staticmethod
     def load(filename, UnsupervisedModel=AutoEncoder):
-        dbn = DBN(0, UnsupervisedModel)
+        dbn = DBN([], UnsupervisedModel)
         npz = np.load(filename)
         dbn.hidden_layers = []
         count = 0
         for i in xrange(0, len(npz.files), 3):
-            W = npz[npz[i]]
-            bh = npz[npz[i+1]]
-            bo = npz[npz[i+2]]
+            W = npz['arr_%s' % i]
+            bh = npz['arr_%s' % (i + 1)]
+            bo = npz['arr_%s' % (i + 2)]
+
+            if i == 0:
+                dbn.D = W.shape[0]
 
             ae = UnsupervisedModel.createFromArrays(W, bh, bo, count)
             dbn.hidden_layers.append(ae)

unsupervised_class2/visualize_features.py

@@ -14,15 +14,16 @@
 def main(loadfile=None, savefile=None):
     Xtrain, Ytrain, Xtest, Ytest = getKaggleMNIST()
     if loadfile:
+        dbn = DBN.load(loadfile)
+    else:
         dbn = DBN([1000, 750, 500, 10]) # AutoEncoder is default
         dbn = DBN([1000, 750, 500, 10], UnsupervisedModel=RBM)
         dbn.fit(Xtrain, pretrain_epochs=15)
-    else:
-        dbn.load(loadfile)
 
     if savefile:
         dbn.save(savefile)
 
+    # first layer features
     # initial weight is D x M
     # W = dbn.hidden_layers[0].W.eval()
     # for i in xrange(dbn.hidden_layers[0].M):
@@ -32,10 +33,7 @@ def main(loadfile=None, savefile=None):
     #     if should_quit == 'n':
     #         break
 
-    # TODO: save the weights so I can initialize from them later
-    #       and just do the last step
-
-    # print features learned in the last layer
+    # features learned in the last layer
     for k in xrange(dbn.hidden_layers[-1].M):
         # activate the kth node
         X = dbn.fit_to_input(k)
@@ -47,4 +45,11 @@ def main(loadfile=None, savefile=None):
 
 
 if __name__ == '__main__':
-    main()
+    # to load a saved file
+    main(loadfile='saved.npz')
+
+    # to neither load nor save
+    # main()
+
+    # to save a trained unsupervised deep network
+    # main(savefile='saved.npz')