change optimizer to Adam

Author: zhedongzheng

tensorflow-models/linear_model/linear_regr.py

@@ -45,11 +45,11 @@ def add_backward_path(self):
         l1_loss = tf.reduce_mean(tf.abs(self.W))
         l2_loss = tf.reduce_mean(tf.square(self.W))
         self.loss = regr_loss + self.l1_ratio * l1_loss + (1-self.l1_ratio) * l2_loss
-        self.train_op = tf.train.GradientDescentOptimizer(0.01).minimize(self.loss)
+        self.train_op = tf.train.AdamOptimizer(0.1).minimize(self.loss)
     # end method add_backward_path
 
 
-    def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
+    def fit(self, X, Y, val_data, n_epoch=70, batch_size=100):
         print("Train %d samples | Test %d samples" % (len(X), len(val_data[0])))
         self.sess.run(tf.global_variables_initializer()) # initialize all variables
         for epoch in range(n_epoch):
@@ -63,8 +63,8 @@ def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
                 v_loss = self.sess.run(self.loss, {self.X:X_test_batch, self.Y:Y_test_batch})
                 val_loss_list.append(v_loss)
             val_loss = self.list_avg(val_loss_list)
-
-            print ("%d / %d: train_loss: %.4f | test_loss: %.4f" % (epoch+1, n_epoch, loss, val_loss))
+            if epoch % 5 == 0:
+                print ("%d / %d: train_loss: %.4f | test_loss: %.4f" % (epoch+1, n_epoch, loss, val_loss))
     # end method fit
 
 

tensorflow-models/linear_model/logistic.py

@@ -49,11 +49,11 @@ def add_backward_path(self):
         l2_loss = tf.reduce_mean(tf.square(self.W))
         self.loss = regr_loss + self.l1_ratio * l1_loss + (1-self.l1_ratio) * l2_loss
         self.acc = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(self.pred,1), tf.argmax(self.Y,1)), tf.float32))
-        self.train_op = tf.train.GradientDescentOptimizer(0.005).minimize(self.loss)
+        self.train_op = tf.train.AdamOptimizer().minimize(self.loss)
     # end method add_backward_path
 
 
-    def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
+    def fit(self, X, Y, val_data, n_epoch=20, batch_size=100):
         print("Train %d samples | Test %d samples" % (len(X), len(val_data[0])))
         self.sess.run(tf.global_variables_initializer()) # initialize all variables
         for epoch in range(n_epoch):
@@ -72,7 +72,7 @@ def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
             val_loss, val_acc = self.list_avg(val_loss_list), self.list_avg(val_acc_list)
 
             # verbose
-            if epoch % 20 == 0:
+            if epoch % 5 == 0:
                 print ("%d / %d: train_loss: %.4f train_acc: %.4f | test_loss: %.4f test_acc: %.4f"
                     % (epoch+1, n_epoch, loss, acc, val_loss, val_acc))
     # end method fit

tensorflow-models/linear_model/logistic_test.py

@@ -13,10 +13,10 @@
     Y_test = tf.contrib.keras.utils.to_categorical(y_test)
 
     clf = Logistic(X.shape[1], 2)
-    clf.fit(X_train, Y_train, n_epoch=100, val_data=(X_test, Y_test))
+    clf.fit(X_train, Y_train, val_data=(X_test, Y_test))
     Y_pred = clf.predict(X_test)
     final_acc = (np.argmax(Y_pred,1) == np.argmax(Y_test,1)).astype(float).mean()
-    print("final testing accuracy: %.4f" % final_acc)
+    print("svm (tensorflow): %.4f" % final_acc)
 
     clf = SVC(kernel='linear')
     y_pred = clf.fit(X_train, y_train).predict(X_test)

tensorflow-models/svm/svm_linear_clf.py

@@ -51,7 +51,7 @@ def add_backward_path(self):
         regu_loss = 0.5 * tf.reduce_sum(tf.square(self.W))
         hinge_loss = tf.reduce_sum(tf.maximum(tf.zeros([self.batch_size,1]), 1-self.Y*self.logits))
         self.loss = regu_loss + self.C * hinge_loss
-        self.train_op = tf.train.GradientDescentOptimizer(1e-3).minimize(self.loss)
+        self.train_op = tf.train.AdamOptimizer().minimize(self.loss)
         self.acc = tf.reduce_mean(tf.cast(tf.equal(self.pred, self.Y), tf.float32))
     # end method add_backward_path
 
@@ -66,7 +66,7 @@ def call_b(self, name, shape):
     # end method _b
 
 
-    def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
+    def fit(self, X, Y, val_data, n_epoch=20, batch_size=100):
         print("Train %d samples | Test %d samples" % (len(X), len(val_data[0])))
         log = {'loss':[], 'acc':[], 'val_loss':[], 'val_acc':[]}
 
@@ -93,7 +93,7 @@ def fit(self, X, Y, val_data, n_epoch=100, batch_size=100):
             log['val_loss'].append(val_loss)
             log['val_acc'].append(val_acc)
             # verbose
-            if epoch % 20 == 0:
+            if epoch % 5 == 0:
                 print ("%d / %d: train_loss: %.4f train_acc: %.4f | test_loss: %.4f test_acc: %.4f"
                     % (epoch+1, n_epoch, loss, acc, val_loss, val_acc))
 

tensorflow-models/svm/svm_linear_clf_test.py

@@ -13,7 +13,7 @@
     Y_test = y_test.reshape(-1, 1)
 
     clf = LinearSVMClassifier(X_train.shape[1])
-    log = clf.fit(X_train, Y_train, n_epoch=100, batch_size=100, val_data=(X_test, Y_test))
+    log = clf.fit(X_train, Y_train, batch_size=100, val_data=(X_test, Y_test))
     Y_pred = clf.predict(X_test)
     print("linear svm (tensorflow):", (Y_pred.ravel() == Y_test.ravel()).astype(float).mean())