Decoupled classifier and FeatureSelection, by moving the classifier into FeatyureSelection instead of constantly being passed around

Author: Ben Evans

Classifier.java

@@ -1,11 +1,10 @@
-import java.io.File;
-import java.io.FileNotFoundException;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.PriorityQueue;
-import java.util.Scanner;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 
 /**
@@ -30,6 +29,21 @@ public Classifier(Set<Instance> instances) {
         }
     }
 
+    /**
+     * Classifies and calculates the percentage
+     * of correct classifications in the testingSet
+     * against the training set.
+     */
+    public double classify(){
+        Instance sampleInstance = training.iterator().next();
+        int totalFeatures = sampleInstance.getNumFeatures();
+
+        // To begin with all features are selected
+        Set<Integer> allIndices =  IntStream.rangeClosed(0, totalFeatures - 1)
+                .boxed().collect(Collectors.toSet());
+
+        return classify(allIndices);
+    }
     /**
      * Classifies and calculates the percentage
      * of correct classifications in the testingSet
@@ -70,6 +84,7 @@ public int compare(Result a, Result b) {
         return correct/(double)testing.size();
     }
 
+
     /**
      * Returns the mode of @param list
      * @param list

Criteria.java

@@ -0,0 +1,5 @@
+import java.util.Set;
+
+public interface Criteria {
+    public boolean evaluate(Set<Integer> features, int size);
+}

FeatureSelection.java

@@ -8,19 +8,26 @@
  */
 public abstract class FeatureSelection {
 
+    private Classifier classifier;
+    protected Set<Instance> instances;
+
+    public FeatureSelection(Set<Instance> instances){
+        this.instances = instances;
+        this.classifier = new Classifier(instances);
+    }
 
     /**
     Returns a subset of only the most important features
     chosen by some measure.
      */
-   public abstract Set<Integer> select(Set<Instance> instances, int numFeaturesToSelect);
+   public abstract Set<Integer> select(int numFeaturesToSelect);
 
    /**
    Returns a subset containing only the numFeatures most important
     features. If numFeatures is >= original.size(), the original
     set is returned.
     */
-   public abstract Set<Integer> select(Set<Instance> instances, double goalAccuracy);
+   public abstract Set<Integer> select(double goalAccuracy);
 
     /**
      * Returns the feature in remaining features
@@ -30,15 +37,15 @@ public abstract class FeatureSelection {
      * @param remainingFeatures
      * @return
      */
-    protected int best(Classifier classifier, Set<Integer> selectedFeatures, Set<Integer> remainingFeatures){
+    protected int best(Set<Integer> selectedFeatures, Set<Integer> remainingFeatures){
         double highest = -Integer.MAX_VALUE;
         int selected = -1;
 
         for(int feature: remainingFeatures){
             Set<Integer> newFeatures = new HashSet<>(selectedFeatures);
             newFeatures.add(feature);
 
-            double result = objectiveFunction(classifier, newFeatures);
+            double result = objectiveFunction(newFeatures);
             if(result > highest){
                 highest = result;
                 selected = feature;
@@ -55,15 +62,15 @@ protected int best(Classifier classifier, Set<Integer> selectedFeatures, Set<Int
      * @param features
      * @return
      */
-    protected int worst(Classifier classifier, Set<Integer> features){
+    protected int worst(Set<Integer> features){
         double lowestAccuracy = Integer.MAX_VALUE;
         int selected = -1;
 
         for(int feature: features){
             Set<Integer> newFeatures = new HashSet<>(features);
             newFeatures.remove(feature);
 
-            double result = objectiveFunction(classifier, newFeatures);
+            double result = objectiveFunction(newFeatures);
             if(result < lowestAccuracy){
                 lowestAccuracy = result;
                 selected = feature;
@@ -73,15 +80,23 @@ protected int worst(Classifier classifier, Set<Integer> features){
         return selected;
     }
 
-    public void compareAccuracy(Classifier classifier, Set<Integer> selectedIndices, int totalFeatures) {
-        Set<Integer> allIndices = IntStream.rangeClosed(0, totalFeatures - 1)
+    protected double objectiveFunction(Set<Integer> selectedFeatures) {
+        return classifier.classify(selectedFeatures);
+    }
+
+    protected Set<Integer> getFeatures(){
+        // Extract an instance to check the amount of features, assumes all instances have same # of features
+        Instance sampleInstance = instances.iterator().next();
+        int totalFeatures = sampleInstance.getNumFeatures();
+
+        // To begin with all features are selected
+        return IntStream.rangeClosed(0, totalFeatures - 1)
                 .boxed().collect(Collectors.toSet());
-        System.out.println("Classification accuracy on testing set using all features: " + classifier.classify(allIndices));
-        System.out.println("Classification accuracy on testing set using features " + selectedIndices + ": " + classifier.classify(selectedIndices));
     }
 
-    protected double objectiveFunction(Classifier classifier, Set<Integer> selectedFeatures) {
-        return classifier.classify(selectedFeatures);
+    public void compareAccuracy(Set<Integer> selectedIndices) {
+        System.out.println("Classification accuracy on testing set using all features: " + classifier.classify());
+        System.out.println("Classification accuracy on testing set using features " + selectedIndices + ": " + classifier.classify(selectedIndices));
     }
 
 

Instance.java

@@ -27,6 +27,17 @@ public Instance(double[] features, String label){
 
     public double getFeature(int index){ return features[index]; }
 
+    public Instance createInstance(Set<Integer> featureIndices){
+        double[] features = new double[featureIndices.size()];
+
+        int i=0;
+        for(int index: featureIndices){
+            features[i++] = features[index];
+        }
+
+        return new Instance(features , label);
+    }
+
     public double distanceTo(Instance other,  Set<Integer> indices){
         if(getNumFeatures() != other.getNumFeatures()) throw new IllegalArgumentException("Number of features do not match");
 

SequentialBackwardsSelection.java

@@ -1,35 +1,34 @@
 import java.util.HashSet;
 import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 /**
  * Created by ben on 18/04/17.
  */
 public class SequentialBackwardsSelection extends FeatureSelection {
 
-    @Override
-    public Set<Integer> select(Set<Instance> instances, int numFeaturesToSelect) {
-        // In this case we have no data to use, so return the empty set
-        if(instances == null || instances.isEmpty()) return new HashSet<Integer>();
+    public SequentialBackwardsSelection(Set<Instance> instances){
+        super(instances);
+    }
 
-        // Extract an instance to check the amount of features, assumes all instances have same # of features
-        Instance sampleInstance = instances.iterator().next();
-        int totalFeatures = sampleInstance.getNumFeatures();
+    @Override
+    public Set<Integer> select(int numFeaturesToSelect) {
+        return select((features, size) -> size > numFeaturesToSelect);
+    }
 
-        // To begin with all features are selected
-        Set<Integer> selectedFeatures = IntStream.rangeClosed(0, totalFeatures - 1)
-                .boxed().collect(Collectors.toSet());
+    @Override
+    public Set<Integer> select(double goalAccuracy) {
+        return select((features, size) -> objectiveFunction(features) < goalAccuracy);
+    }
 
-        // Nothing we can do if the number of features to select is greater than or equal to the total size
-        if (numFeaturesToSelect >= totalFeatures){
-            return selectedFeatures;
-        }
+    public Set<Integer> select(Criteria criteria) {
+        // In this case we have no data to use, so return the empty set
+        if (instances == null || instances.isEmpty()) return new HashSet<Integer>();
 
-        Classifier classifier = new Classifier(instances);
+        // To begin with all features are selected
+        Set<Integer> selectedFeatures = getFeatures();
 
-        while (selectedFeatures.size() >= numFeaturesToSelect){
-            int feature = worst(classifier, selectedFeatures);
+        while (criteria.evaluate(selectedFeatures, selectedFeatures.size())){
+            int feature = worst(selectedFeatures);
 
             // No more valid features
             if (feature == -1) break;
@@ -40,9 +39,4 @@ public Set<Integer> select(Set<Instance> instances, int numFeaturesToSelect) {
 
         return selectedFeatures;
     }
-
-    @Override
-    public Set<Integer> select(Set<Instance> instances, double goalAccuracy) {
-        return null;
-    }
 }

SequentialForwardSelection.java

@@ -1,87 +1,47 @@
 import java.util.HashSet;
 import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 /**
  * Created by ben on 8/04/17.
  */
 public class SequentialForwardSelection extends FeatureSelection {
 
-    public Set<Integer> select(final Set<Instance> instances, int numFeaturesToSelect) {
-        // In this case we have no data to use, so return the empty set
-        if(instances == null || instances.isEmpty()) return new HashSet<Integer>();
-
-        // Extract an instance to check the amount of features, assumes all instances have same # of features
-        Instance sampleInstance = instances.iterator().next();
-        int totalFeatures = sampleInstance.getNumFeatures();
-
-        // To begin with no features are selected, so all the indices from 0..totalFeatures are remaining
-        Set<Integer> remainingFeatures = IntStream.rangeClosed(0, totalFeatures - 1)
-                .boxed().collect(Collectors.toSet());
-
-        // Nothing we can do if the number of features to select is greater than or equal to the total size
-        if (numFeaturesToSelect >= totalFeatures){
-            return remainingFeatures;
-        }
-
-        // Subset of only selected features indices
-        Set<Integer> selectedFeatures = new HashSet<>();
-
-        Classifier classifier = new Classifier(instances);
-
-        while (selectedFeatures.size() < numFeaturesToSelect){
-            int feature = best(classifier, selectedFeatures, remainingFeatures);
-
-            // No more valid features
-            if (feature == -1) break;
+    public SequentialForwardSelection(Set<Instance> instances){
+        super(instances);
+    }
 
-            selectedFeatures.add(feature);
-            // Remove the feature so we do not keep selecting the same one
-            remainingFeatures.remove(feature);
-        }
+    public Set<Integer> select(int numFeaturesToSelect) {
+       return select((features, size) -> size < numFeaturesToSelect);
+    }
 
-        compareAccuracy(classifier, selectedFeatures, totalFeatures);
-        return selectedFeatures;
+    public Set<Integer> select(double goalAccuracy) {
+        return select((features, size) -> objectiveFunction( features) < goalAccuracy);
     }
 
-    public Set<Integer> select(final Set<Instance> instances, double goalAccuracy) {
+    public Set<Integer> select(Criteria criteria) {
         // In this case we have no data to use, so return the empty set
         if (instances == null || instances.isEmpty()) return new HashSet<Integer>();
 
-        // Extract an instance to check the amount of features, assumes all instances have same # of features
-        Instance sampleInstance = instances.iterator().next();
-        int totalFeatures = sampleInstance.getNumFeatures();
-
         // To begin with no features are selected, so all the indices from 0..totalFeatures are remaining
-        Set<Integer> remainingFeatures = IntStream.rangeClosed(0, totalFeatures - 1)
-                .boxed().collect(Collectors.toSet());
+        Set<Integer> remainingFeatures = getFeatures();
 
         // Subset of only selected features indices
         Set<Integer> selectedFeatures = new HashSet<>();
 
-        // Track classifiction accuracy
-        double accuracy = 0;
-
         Classifier classifier = new Classifier(instances);
 
-        while (accuracy < goalAccuracy){
-            int feature = best(classifier, selectedFeatures, remainingFeatures);
+        while (criteria.evaluate(selectedFeatures, selectedFeatures.size())){
+            int feature = best(selectedFeatures, remainingFeatures);
             // No more valid features
             if (feature == -1) break;
 
             selectedFeatures.add(feature);
             // Remove the feature so we do not keep selecting the same one
             remainingFeatures.remove(feature);
-
-            accuracy = objectiveFunction(classifier, selectedFeatures);
         }
 
-        compareAccuracy(classifier, selectedFeatures, totalFeatures);
         return selectedFeatures;
     }
 
 
-
-
 }