add regularization code and overfitting code for linear and logistic

Author: lazyprogrammer

linear_regression_class/gradient_descent.py

@@ -0,0 +1,45 @@
+# notes for this course can be found at:
+# https://www.udemy.com/data-science-linear-regression-in-python
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+N = 10
+D = 3
+X = np.zeros((N, D))
+X[:,0] = 1 # bias term
+X[:5,1] = 1
+X[5:,2] = 1
+Y = np.array([0]*5 + [1]*5)
+
+# print X so you know what it looks like
+print "X:", X
+
+# won't work!
+# w = np.linalg.solve(X.T.dot(X), X.T.dot(Y))
+
+# let's try gradient descent
+costs = [] # keep track of squared error cost
+w = np.random.randn(D) / np.sqrt(D) # randomly initialize w
+learning_rate = 0.001
+for t in xrange(1000):
+  # update w
+  Yhat = X.dot(w)
+  delta = Yhat - Y
+  w = w - learning_rate*X.T.dot(delta)
+
+  # find and store the cost
+  mse = delta.dot(delta) / N
+  costs.append(mse)
+
+# plot the costs
+plt.plot(costs)
+plt.show()
+
+print "final w:", w
+
+# plot prediction vs target
+plt.plot(Yhat, label='prediction')
+plt.plot(Y, label='target')
+plt.legend()
+plt.show()

linear_regression_class/l1_regularization.py

@@ -0,0 +1,44 @@
+# notes for this course can be found at:
+# https://www.udemy.com/data-science-linear-regression-in-python
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+N = 50
+D = 50
+
+# uniformly distributed numbers between -5, +5
+X = (np.random.random((N, D)) - 0.5)*10
+
+# true weights - only the first 3 dimensions of X affect Y
+true_w = np.array([1, 0.5, -0.5] + [0]*(D - 3))
+
+# generate Y - add noise with variance 0.5
+Y = X.dot(true_w) + np.random.randn(N)*0.5
+
+# perform gradient descent to find w
+costs = [] # keep track of squared error cost
+w = np.random.randn(D) / np.sqrt(D) # randomly initialize w
+learning_rate = 0.001
+l1 = 10.0 # Also try 5.0, 2.0, 1.0, 0.1 - what effect does it have on w?
+for t in xrange(500):
+  # update w
+  Yhat = X.dot(w)
+  delta = Yhat - Y
+  w = w - learning_rate*(X.T.dot(delta) + l1*np.sign(w))
+
+  # find and store the cost
+  mse = delta.dot(delta) / N
+  costs.append(mse)
+
+# plot the costs
+plt.plot(costs)
+plt.show()
+
+print "final w:", w
+
+# plot our w vs true w
+plt.plot(true_w, label='true w')
+plt.plot(w, label='w_map')
+plt.legend()
+plt.show()

linear_regression_class/l2_regularization.py

@@ -0,0 +1,43 @@
+# demonstration of L2 regularization
+#
+# notes for this course can be found at:
+# https://www.udemy.com/data-science-linear-regression-in-python
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+N = 50
+
+# generate the data
+X = np.linspace(0,10,N)
+Y = 0.5*X + np.random.randn(N)
+
+# make outliers
+Y[-1] += 30
+Y[-2] += 30
+
+# plot the data
+plt.scatter(X, Y)
+plt.show()
+
+# add bias term
+X = np.vstack([np.ones(N), X]).T
+
+# plot the maximum likelihood solution
+w_ml = np.linalg.solve(X.T.dot(X), X.T.dot(Y))
+Yhat_ml = X.dot(w_ml)
+plt.scatter(X[:,1], Y)
+plt.plot(X[:,1], Yhat_ml)
+plt.show()
+
+# plot the regularized solution
+# probably don't need an L2 regularization this high in many problems
+# everything in this example is exaggerated for visualization purposes
+l2 = 1000.0
+w_map = np.linalg.solve(l2*np.eye(2) + X.T.dot(X), X.T.dot(Y))
+Yhat_map = X.dot(w_map)
+plt.scatter(X[:,1], Y)
+plt.plot(X[:,1], Yhat_ml, label='maximum likelihood')
+plt.plot(X[:,1], Yhat_map, label='map')
+plt.legend()
+plt.show()

linear_regression_class/overfitting.py

@@ -5,14 +5,6 @@
 import numpy as np
 import matplotlib.pyplot as plt
 
-# make up some data and plot it
-N = 100
-X = np.linspace(0, 6*np.pi, N)
-Y = np.sin(X)
-
-plt.plot(X, Y)
-plt.show()
-
 
 def make_poly(X, deg):
     n = len(X)
@@ -48,9 +40,6 @@ def fit_and_display(X, Y, sample, deg):
     plt.title("deg = %d" % deg)
     plt.show()
 
-for deg in (5, 6, 7, 8, 9):
-    fit_and_display(X, Y, 10, deg)
-
 
 def get_mse(Y, Yhat):
     d = Y - Yhat
@@ -92,4 +81,15 @@ def plot_train_vs_test_curves(X, Y, sample=20, max_deg=20):
     plt.legend()
     plt.show()
 
-plot_train_vs_test_curves(X, Y)
+if __name__ == "__main__":
+    # make up some data and plot it
+    N = 100
+    X = np.linspace(0, 6*np.pi, N)
+    Y = np.sin(X)
+
+    plt.plot(X, Y)
+    plt.show()
+
+    for deg in (5, 6, 7, 8, 9):
+        fit_and_display(X, Y, 10, deg)
+    plot_train_vs_test_curves(X, Y)

logistic_regression_class/l1_regularization.py

@@ -0,0 +1,47 @@
+# notes for this course can be found at:
+# https://www.udemy.com/data-science-logistic-regression-in-python
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+def sigmoid(z):
+  return 1/(1 + np.exp(-z))
+
+N = 50
+D = 50
+
+# uniformly distributed numbers between -5, +5
+X = (np.random.random((N, D)) - 0.5)*10
+
+# true weights - only the first 3 dimensions of X affect Y
+true_w = np.array([1, 0.5, -0.5] + [0]*(D - 3))
+
+# generate Y - add noise with variance 0.5
+Y = np.round(sigmoid(X.dot(true_w) + np.random.randn(N)*0.5))
+
+# perform gradient descent to find w
+costs = [] # keep track of squared error cost
+w = np.random.randn(D) / np.sqrt(D) # randomly initialize w
+learning_rate = 0.001
+l1 = 3.0 # try different values - what effect does it have on w?
+for t in xrange(5000):
+  # update w
+  Yhat = sigmoid(X.dot(w))
+  delta = Yhat - Y
+  w = w - learning_rate*(X.T.dot(delta) + l1*np.sign(w))
+
+  # find and store the cost
+  cost = -(Y*np.log(Yhat) + (1-Y)*np.log(1 - Yhat)).mean() + l1*np.abs(w).mean()
+  costs.append(cost)
+
+# plot the costs
+plt.plot(costs)
+plt.show()
+
+print "final w:", w
+
+# plot our w vs true w
+plt.plot(true_w, label='true w')
+plt.plot(w, label='w_map')
+plt.legend()
+plt.show()