Skip to content

Commit cc5ff21

Browse files
lazyprogrammerlazyprogrammer
committed
py3
1 parent d3ced3d commit cc5ff21

13 files changed

+145
-92
lines changed

‎ann_class/backprop.py

+9-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
# the notes for this class can be found at:
55
# https://deeplearningcourses.com/c/data-science-deep-learning-in-python
66
# https://www.udemy.com/data-science-deep-learning-in-python
7+
from __future__ import print_function, division
8+
from builtins import range
9+
# Note: you may need to update your version of future
10+
# sudo pip install -U future
711

812
import numpy as np
913
import matplotlib.pyplot as plt
@@ -23,7 +27,7 @@ def forward(X, W1, b1, W2, b2):
2327
def classification_rate(Y, P):
2428
n_correct = 0
2529
n_total = 0
26-
for i in xrange(len(Y)):
30+
for i in range(len(Y)):
2731
n_total += 1
2832
if Y[i] == P[i]:
2933
n_correct += 1
@@ -113,7 +117,7 @@ def main():
113117
N = len(Y)
114118
# turn Y into an indicator matrix for training
115119
T = np.zeros((N, K))
116-
for i in xrange(N):
120+
for i in range(N):
117121
T[i, Y[i]] = 1
118122

119123
# let's see what it looks like
@@ -126,15 +130,15 @@ def main():
126130
W2 = np.random.randn(M, K)
127131
b2 = np.random.randn(K)
128132

129-
learning_rate = 1e-6
133+
learning_rate = 1e-3
130134
costs = []
131-
for epoch in xrange(100000):
135+
for epoch in range(1000):
132136
output, hidden = forward(X, W1, b1, W2, b2)
133137
if epoch % 100 == 0:
134138
c = cost(T, output)
135139
P = np.argmax(output, axis=1)
136140
r = classification_rate(Y, P)
137-
print "cost:", c, "classification_rate:", r
141+
print("cost:", c, "classification_rate:", r)
138142
costs.append(c)
139143

140144
# this is gradient ASCENT, not DESCENT

‎ann_class2/dropout_theano.py

+4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55
# For the class Data Science: Practical Deep Learning Concepts in Theano and TensorFlow
66
# https://deeplearningcourses.com/c/data-science-deep-learning-in-theano-tensorflow
77
# https://www.udemy.com/data-science-deep-learning-in-theano-tensorflow
8+
from __future__ import print_function, division
9+
from builtins import range
10+
# Note: you may need to update your version of future
11+
# sudo pip install -U future
812

913
import numpy as np
1014
import theano

‎ann_class2/mlp.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,24 @@
22
# For the class Data Science: Practical Deep Learning Concepts in Theano and TensorFlow
33
# https://deeplearningcourses.com/c/data-science-deep-learning-in-theano-tensorflow
44
# https://www.udemy.com/data-science-deep-learning-in-theano-tensorflow
5+
from __future__ import print_function, division
6+
from builtins import range
7+
# Note: you may need to update your version of future
8+
# sudo pip install -U future
59

610
import numpy as np
711

812
def forward(X, W1, b1, W2, b2):
13+
# sigmoid
914
# Z = 1 / (1 + np.exp(-( X.dot(W1) + b1 )))
1015

11-
# rectifier
16+
# relu
1217
Z = X.dot(W1) + b1
1318
Z[Z < 0] = 0
14-
# print "Z:", Z
1519

1620
A = Z.dot(W2) + b2
1721
expA = np.exp(A)
1822
Y = expA / expA.sum(axis=1, keepdims=True)
19-
# print "Y:", Y, "are any 0?", np.any(Y == 0), "are any nan?", np.any(np.isnan(Y))
20-
# exit()
2123
return Y, Z
2224

2325
def derivative_w2(Z, T, Y):
@@ -32,4 +34,4 @@ def derivative_w1(X, Z, T, Y, W2):
3234

3335
def derivative_b1(Z, T, Y, W2):
3436
# return (( Y-T ).dot(W2.T) * ( Z*(1 - Z) )).sum(axis=0) # for sigmoid
35-
return (( Y-T ).dot(W2.T) * (Z > 0)).sum(axis=0) # for relu
37+
return (( Y-T ).dot(W2.T) * (Z > 0)).sum(axis=0) # for relu

‎ann_class2/momentum.py

+20-17
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
# NOTE: MUST restrict initial values of W by dividing by #
77
# NOTE: sigmoid vs. rectifier for hiddens
88
# We get 15% error rate with sigmoid, 3% error rate with ReLU
9+
from __future__ import print_function, division
10+
from builtins import range
11+
# Note: you may need to update your version of future
12+
# sudo pip install -U future
913

1014
import numpy as np
1115
from sklearn.utils import shuffle
@@ -37,7 +41,7 @@ def main():
3741

3842
N, D = Xtrain.shape
3943
batch_sz = 500
40-
n_batches = N / batch_sz
44+
n_batches = N // batch_sz
4145

4246
M = 300
4347
K = 10
@@ -50,8 +54,8 @@ def main():
5054
# cost = -16
5155
LL_batch = []
5256
CR_batch = []
53-
for i in xrange(max_iter):
54-
for j in xrange(n_batches):
57+
for i in range(max_iter):
58+
for j in range(n_batches):
5559
Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
5660
Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
5761
pYbatch, Z = forward(Xbatch, W1, b1, W2, b2)
@@ -66,17 +70,16 @@ def main():
6670
if j % print_period == 0:
6771
# calculate just for LL
6872
pY, _ = forward(Xtest, W1, b1, W2, b2)
69-
# print "pY:", pY
7073
ll = cost(pY, Ytest_ind)
7174
LL_batch.append(ll)
72-
print "Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)
75+
print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll))
7376

7477
err = error_rate(pY, Ytest)
7578
CR_batch.append(err)
76-
print "Error rate:", err
79+
print("Error rate:", err)
7780

7881
pY, _ = forward(Xtest, W1, b1, W2, b2)
79-
print "Final error rate:", error_rate(pY, Ytest)
82+
print("Final error rate:", error_rate(pY, Ytest))
8083

8184
# 2. batch with momentum
8285
W1 = np.random.randn(D, M) / np.sqrt(D)
@@ -90,8 +93,8 @@ def main():
9093
db2 = 0
9194
dW1 = 0
9295
db1 = 0
93-
for i in xrange(max_iter):
94-
for j in xrange(n_batches):
96+
for i in range(max_iter):
97+
for j in range(n_batches):
9598
Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
9699
Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
97100
pYbatch, Z = forward(Xbatch, W1, b1, W2, b2)
@@ -112,13 +115,13 @@ def main():
112115
# print "pY:", pY
113116
ll = cost(pY, Ytest_ind)
114117
LL_momentum.append(ll)
115-
print "Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)
118+
print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll))
116119

117120
err = error_rate(pY, Ytest)
118121
CR_momentum.append(err)
119-
print "Error rate:", err
122+
print("Error rate:", err)
120123
pY, _ = forward(Xtest, W1, b1, W2, b2)
121-
print "Final error rate:", error_rate(pY, Ytest)
124+
print("Final error rate:", error_rate(pY, Ytest))
122125

123126

124127
# 3. batch with Nesterov momentum
@@ -138,8 +141,8 @@ def main():
138141
vb2 = 0
139142
vW1 = 0
140143
vb1 = 0
141-
for i in xrange(max_iter):
142-
for j in xrange(n_batches):
144+
for i in range(max_iter):
145+
for j in range(n_batches):
143146
# because we want g(t) = grad(f(W(t-1) - lr*mu*dW(t-1)))
144147
# dW(t) = mu*dW(t-1) + g(t)
145148
# W(t) = W(t-1) - mu*dW(t)
@@ -177,13 +180,13 @@ def main():
177180
# print "pY:", pY
178181
ll = cost(pY, Ytest_ind)
179182
LL_nest.append(ll)
180-
print "Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)
183+
print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll))
181184

182185
err = error_rate(pY, Ytest)
183186
CR_nest.append(err)
184-
print "Error rate:", err
187+
print("Error rate:", err)
185188
pY, _ = forward(Xtest, W1, b1, W2, b2)
186-
print "Final error rate:", error_rate(pY, Ytest)
189+
print("Final error rate:", error_rate(pY, Ytest))
187190

188191

189192

‎ann_class2/rmsprop.py

+16-12
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
# For the class Data Science: Practical Deep Learning Concepts in Theano and TensorFlow
33
# https://deeplearningcourses.com/c/data-science-deep-learning-in-theano-tensorflow
44
# https://www.udemy.com/data-science-deep-learning-in-theano-tensorflow
5+
from __future__ import print_function, division
6+
from builtins import range
7+
# Note: you may need to update your version of future
8+
# sudo pip install -U future
59

610
import numpy as np
711
from sklearn.utils import shuffle
@@ -28,7 +32,7 @@ def main():
2832

2933
N, D = Xtrain.shape
3034
batch_sz = 500
31-
n_batches = N / batch_sz
35+
n_batches = N // batch_sz
3236

3337
M = 300
3438
K = 10
@@ -41,8 +45,8 @@ def main():
4145
# cost = -16
4246
LL_batch = []
4347
CR_batch = []
44-
for i in xrange(max_iter):
45-
for j in xrange(n_batches):
48+
for i in range(max_iter):
49+
for j in range(n_batches):
4650
Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
4751
Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
4852
pYbatch, Z = forward(Xbatch, W1, b1, W2, b2)
@@ -60,14 +64,14 @@ def main():
6064
# print "pY:", pY
6165
ll = cost(pY, Ytest_ind)
6266
LL_batch.append(ll)
63-
print "Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)
67+
print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll))
6468

6569
err = error_rate(pY, Ytest)
6670
CR_batch.append(err)
67-
print "Error rate:", err
71+
print("Error rate:", err)
6872

6973
pY, _ = forward(Xtest, W1, b1, W2, b2)
70-
print "Final error rate:", error_rate(pY, Ytest)
74+
print("Final error rate:", error_rate(pY, Ytest))
7175

7276

7377
# 2. RMSprop
@@ -83,9 +87,9 @@ def main():
8387
cache_W1 = 1
8488
cache_b1 = 1
8589
decay_rate = 0.999
86-
eps = 0.0000000001
87-
for i in xrange(max_iter):
88-
for j in xrange(n_batches):
90+
eps = 1e-10
91+
for i in range(max_iter):
92+
for j in range(n_batches):
8993
Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),]
9094
Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),]
9195
pYbatch, Z = forward(Xbatch, W1, b1, W2, b2)
@@ -114,14 +118,14 @@ def main():
114118
# print "pY:", pY
115119
ll = cost(pY, Ytest_ind)
116120
LL_rms.append(ll)
117-
print "Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)
121+
print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll))
118122

119123
err = error_rate(pY, Ytest)
120124
CR_rms.append(err)
121-
print "Error rate:", err
125+
print("Error rate:", err)
122126

123127
pY, _ = forward(Xtest, W1, b1, W2, b2)
124-
print "Final error rate:", error_rate(pY, Ytest)
128+
print("Final error rate:", error_rate(pY, Ytest))
125129

126130
plt.plot(LL_batch, label='const')
127131
plt.plot(LL_rms, label='rms')

‎ann_class2/sgd.py

+25-21
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
# For the class Data Science: Practical Deep Learning Concepts in Theano and TensorFlow
1414
# https://deeplearningcourses.com/c/data-science-deep-learning-in-theano-tensorflow
1515
# https://www.udemy.com/data-science-deep-learning-in-theano-tensorflow
16+
from __future__ import print_function, division
17+
from builtins import range
18+
# Note: you may need to update your version of future
19+
# sudo pip install -U future
1620

1721
import numpy as np
1822
import pandas as pd
@@ -32,7 +36,7 @@ def main():
3236
std = X.std(axis=0)
3337
X = (X - mu) / std
3438

35-
print "Performing logistic regression..."
39+
print("Performing logistic regression...")
3640
Xtrain = X[:-1000,]
3741
Ytrain = Y[:-1000]
3842
Xtest = X[-1000:,]
@@ -49,7 +53,7 @@ def main():
4953
lr = 0.0001
5054
reg = 0.01
5155
t0 = datetime.now()
52-
for i in xrange(200):
56+
for i in range(200):
5357
p_y = forward(Xtrain, W, b)
5458

5559
W += lr*(gradW(Ytrain_ind, p_y, Xtrain) - reg*W)
@@ -61,11 +65,11 @@ def main():
6165
LL.append(ll)
6266
if i % 10 == 0:
6367
err = error_rate(p_y_test, Ytest)
64-
print "Cost at iteration %d: %.6f" % (i, ll)
65-
print "Error rate:", err
68+
print("Cost at iteration %d: %.6f" % (i, ll))
69+
print("Error rate:", err)
6670
p_y = forward(Xtest, W, b)
67-
print "Final error rate:", error_rate(p_y, Ytest)
68-
print "Elapsted time for full GD:", datetime.now() - t0
71+
print("Final error rate:", error_rate(p_y, Ytest))
72+
print("Elapsted time for full GD:", datetime.now() - t0)
6973

7074

7175
# 2. stochastic
@@ -76,9 +80,9 @@ def main():
7680
reg = 0.01
7781

7882
t0 = datetime.now()
79-
for i in xrange(1): # takes very long since we're computing cost for 41k samples
83+
for i in range(1): # takes very long since we're computing cost for 41k samples
8084
tmpX, tmpY = shuffle(Xtrain, Ytrain_ind)
81-
for n in xrange(min(N, 500)): # shortcut so it won't take so long...
85+
for n in range(min(N, 500)): # shortcut so it won't take so long...
8286
x = tmpX[n,:].reshape(1,D)
8387
y = tmpY[n,:].reshape(1,10)
8488
p_y = forward(x, W, b)
@@ -90,13 +94,13 @@ def main():
9094
ll = cost(p_y_test, Ytest_ind)
9195
LL_stochastic.append(ll)
9296

93-
if n % (N/2) == 0:
97+
if n % (N//2) == 0:
9498
err = error_rate(p_y_test, Ytest)
95-
print "Cost at iteration %d: %.6f" % (i, ll)
96-
print "Error rate:", err
99+
print("Cost at iteration %d: %.6f" % (i, ll))
100+
print("Error rate:", err)
97101
p_y = forward(Xtest, W, b)
98-
print "Final error rate:", error_rate(p_y, Ytest)
99-
print "Elapsted time for SGD:", datetime.now() - t0
102+
print("Final error rate:", error_rate(p_y, Ytest))
103+
print("Elapsted time for SGD:", datetime.now() - t0)
100104

101105

102106
# 3. batch
@@ -106,12 +110,12 @@ def main():
106110
lr = 0.0001
107111
reg = 0.01
108112
batch_sz = 500
109-
n_batches = N / batch_sz
113+
n_batches = N // batch_sz
110114

111115
t0 = datetime.now()
112-
for i in xrange(50):
116+
for i in range(50):
113117
tmpX, tmpY = shuffle(Xtrain, Ytrain_ind)
114-
for j in xrange(n_batches):
118+
for j in range(n_batches):
115119
x = tmpX[j*batch_sz:(j*batch_sz + batch_sz),:]
116120
y = tmpY[j*batch_sz:(j*batch_sz + batch_sz),:]
117121
p_y = forward(x, W, b)
@@ -122,13 +126,13 @@ def main():
122126
p_y_test = forward(Xtest, W, b)
123127
ll = cost(p_y_test, Ytest_ind)
124128
LL_batch.append(ll)
125-
if j % (n_batches/2) == 0:
129+
if j % (n_batches//2) == 0:
126130
err = error_rate(p_y_test, Ytest)
127-
print "Cost at iteration %d: %.6f" % (i, ll)
128-
print "Error rate:", err
131+
print("Cost at iteration %d: %.6f" % (i, ll))
132+
print("Error rate:", err)
129133
p_y = forward(Xtest, W, b)
130-
print "Final error rate:", error_rate(p_y, Ytest)
131-
print "Elapsted time for batch GD:", datetime.now() - t0
134+
print("Final error rate:", error_rate(p_y, Ytest))
135+
print("Elapsted time for batch GD:", datetime.now() - t0)
132136

133137

134138

0 commit comments

Comments
 (0)