update

Author: bob7783

svm_class/crossval.py

@@ -0,0 +1,23 @@
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+from datetime import datetime
+from sklearn.model_selection import cross_val_score
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+
+# load the data
+data = load_breast_cancer()
+
+for C in (0.5, 1.0, 5.0, 10.0):
+  pipeline = Pipeline([('scaler', StandardScaler()), ('svm', SVC(C=C))])
+  scores = cross_val_score(pipeline, data.data, data.target, cv=5)
+  print("C:", C, "mean:", scores.mean(), "std:", scores.std())

svm_class/extra_reading.txt

@@ -44,4 +44,19 @@ A Tutorial on Support Vector Regression
 https://alex.smola.org/papers/2003/SmoSch03b.pdf
 
 LIBSVM -- A Library for Support Vector Machines
-https://www.csie.ntu.edu.tw/~cjlin/libsvm/
+https://www.csie.ntu.edu.tw/~cjlin/libsvm/
+
+Random Features for Large-Scale Kernel Machines
+http://www.robots.ox.ac.uk/~vgg/rg/papers/randomfeatures.pdf
+
+Reflections on Random Kitchen Sinks
+http://www.argmin.net/2017/12/05/kitchen-sinks/
+
+Weighted Sums of Random Kitchen Sinks: Replacing minimization with randomization in learning
+https://papers.nips.cc/paper/3495-weighted-sums-of-random-kitchen-sinks-replacing-minimization-with-randomization-in-learning
+
+Using the Nyström Method to Speed Up Kernel Machines
+https://papers.nips.cc/paper/1866-using-the-nystrom-method-to-speed-up-kernel-machines
+
+Nyström Method vs Random Fourier Features: A Theoretical and Empirical Comparison
+https://papers.nips.cc/paper/4588-nystrom-method-vs-random-fourier-features-a-theoretical-and-empirical-comparison

svm_class/fake_neural_net.py

@@ -13,44 +13,83 @@
 from sklearn.svm import LinearSVC
 from sklearn.preprocessing import StandardScaler
 from sklearn.cluster import KMeans
-
-# get the data: https://www.kaggle.com/c/digit-recognizer
-Xtrain, Ytrain, Xtest, Ytest = getKaggleMNIST()
+from sklearn.mixture import GaussianMixture
+from sklearn.model_selection import cross_val_score
+from sklearn.utils import shuffle
+from scipy import stats
 
 
 class SigmoidFeaturizer:
   def __init__(self, gamma=1.0, n_components=100, method='random'):
     self.M = n_components
     self.gamma = gamma
-    assert(method in ('random', 'kmeans'))
+    assert(method in ('random', 'kmeans', 'gmm'))
     self.method = method
 
+  def _subsample_data(self, X, Y, n=10000):
+    if Y is not None:
+      X, Y = shuffle(X, Y)
+      return X[:n], Y[:n]
+    else:
+      X = shuffle(X)
+      return X[:n]
+
   def fit(self, X, Y=None):
     if self.method == 'random':
       N = len(X)
       idx = np.random.randint(N, size=self.M)
       self.samples = X[idx]
     elif self.method == 'kmeans':
+      X, Y = self._subsample_data(X, Y)
+
       print("Fitting kmeans...")
       t0 = datetime.now()
-      kmeans = KMeans(n_clusters=self.M)
+      kmeans = KMeans(n_clusters=len(set(Y)))
       kmeans.fit(X)
       print("Finished fitting kmeans, duration:", datetime.now() - t0)
-      self.samples = kmeans.cluster_centers_
+
+      # calculate the most ambiguous points
+      # we will do this by finding the distance between each point
+      # and all cluster centers
+      # and return which points have the smallest variance
+      dists = kmeans.transform(X) # returns an N x K matrix
+      variances = dists.var(axis=1)
+      idx = np.argsort(variances) # smallest to largest
+      idx = idx[:self.M]
+      self.samples = X[idx]
+    elif self.method == 'gmm':
+      X, Y = self._subsample_data(X, Y)
+
+      print("Fitting GMM")
+      t0 = datetime.now()
+      gmm = GaussianMixture(n_components=len(set(Y)))
+      gmm.fit(X)
+      print("Finished fitting GMM, duration:", datetime.now() - t0)
+
+      # calculate the most ambiguous points
+      probs = gmm.predict_proba(X)
+      ent = stats.entropy(probs.T) # N-length vector of entropies
+      idx = np.argsort(-ent) # negate since we want biggest first
+      idx = idx[:self.M]
+      self.samples = X[idx]
     return self
 
   def transform(self, X):
-    Z = self.gamma * X.dot(self.samples.T) # (Ntest x D) x (D x Nsamples) -> (Ntest x Nsamples)
-    return np.tanh(Z)
+    Z = X.dot(self.samples.T) # (Ntest x D) x (D x Nsamples) -> (Ntest x Nsamples)
+    return np.tanh(self.gamma * Z)
+    # return self.gamma * Z * (Z > 0)
 
   def fit_transform(self, X, Y=None):
     return self.fit(X, Y).transform(X)
 
 
+# get the data: https://www.kaggle.com/c/digit-recognizer
+Xtrain, Ytrain, Xtest, Ytest = getKaggleMNIST()
+
 # with SGD
 pipeline = Pipeline([
   ('scaler', StandardScaler()),
-  ('sigmoid', SigmoidFeaturizer(gamma=0.05, n_components=2000, method='random')),
+  ('sigmoid', SigmoidFeaturizer(gamma=0.05, n_components=2000, method='gmm')),
   ('linear', SGDClassifier(max_iter=1e6, tol=1e-5))
 ])
 
@@ -63,10 +102,16 @@ def fit_transform(self, X, Y=None):
 # ])
 
 
-t0 = datetime.now()
-pipeline.fit(Xtrain, Ytrain)
-print("train duration:", datetime.now() - t0)
-t0 = datetime.now()
-print("train score:", pipeline.score(Xtrain, Ytrain), "duration:", datetime.now() - t0)
-t0 = datetime.now()
-print("test score:", pipeline.score(Xtest, Ytest), "duration:", datetime.now() - t0)
+X = np.vstack((Xtrain, Xtest))
+Y = np.concatenate((Ytrain, Ytest))
+scores = cross_val_score(pipeline, X, Y, cv=5)
+print(scores)
+print("avg:", np.mean(scores))
+
+# t0 = datetime.now()
+# pipeline.fit(Xtrain, Ytrain)
+# print("train duration:", datetime.now() - t0)
+# t0 = datetime.now()
+# print("train score:", pipeline.score(Xtrain, Ytrain), "duration:", datetime.now() - t0)
+# t0 = datetime.now()
+# print("test score:", pipeline.score(Xtest, Ytest), "duration:", datetime.now() - t0)