update

Author: lazyprogrammer

ann_class2/adam.py

@@ -163,23 +163,23 @@ def main():
             # updates
             gW2 = derivative_w2(Z, Ybatch, pYbatch) + reg*W2
             cache_W2 = decay_rate*cache_W2 + (1 - decay_rate)*gW2*gW2
-            dW2 = mu * dW2 - (1 - mu) * lr0 * gW2 / (np.sqrt(cache_W2) + eps)
-            W2 += dW2
+            dW2 = mu * dW2 + (1 - mu) * lr0 * gW2 / (np.sqrt(cache_W2) + eps)
+            W2 -= dW2
 
             gb2 = derivative_b2(Ybatch, pYbatch) + reg*b2
             cache_b2 = decay_rate*cache_b2 + (1 - decay_rate)*gb2*gb2
-            db2 = mu * db2 - (1 - mu) * lr0 * gb2 / (np.sqrt(cache_b2) + eps)
-            b2 += db2
+            db2 = mu * db2 + (1 - mu) * lr0 * gb2 / (np.sqrt(cache_b2) + eps)
+            b2 -= db2
 
             gW1 = derivative_w1(Xbatch, Z, Ybatch, pYbatch, W2) + reg*W1
             cache_W1 = decay_rate*cache_W1 + (1 - decay_rate)*gW1*gW1
-            dW1 = mu * dW1 - (1 - mu) * lr0 * gW1 / (np.sqrt(cache_W1) + eps)
-            W1 += dW1
+            dW1 = mu * dW1 + (1 - mu) * lr0 * gW1 / (np.sqrt(cache_W1) + eps)
+            W1 -= dW1
 
             gb1 = derivative_b1(Z, Ybatch, pYbatch, W2) + reg*b1
             cache_b1 = decay_rate*cache_b1 + (1 - decay_rate)*gb1*gb1
-            db1 = mu * db1 - (1 - mu) * lr0 * gb1 / (np.sqrt(cache_b1) + eps)
-            b1 += db1
+            db1 = mu * db1 + (1 - mu) * lr0 * gb1 / (np.sqrt(cache_b1) + eps)
+            b1 -= db1
 
             if j % print_period == 0:
                 pY, _ = forward(Xtest, W1, b1, W2, b2)