delta vis

Author: Daniel Kerrigan

demo_data/normal.ipynb

@@ -10,6 +10,8 @@ Server Performance,https://gist.githubusercontent.com/DanielKerrigan/60327aa04a8
 Bank Marketing,https://gist.githubusercontent.com/DanielKerrigan/13ccc22bb97431ba26ad03c3c74864d1/raw/27894766de9389811c665cb40d66d902f4fd5fe5/bank-marketing.csv
 Australia Rain,https://gist.githubusercontent.com/DanielKerrigan/ddb491dcf5b0abc9f2ad2773cc3839ee/raw/3a059f5b079b3a61ad9e2a9004d0b85c575d7177/weather-aus.csv
 UI Test,/datasets/test/ui-test.csv
+Normal,/datasets/normal.csv
+Synthetic,/datasets/synthetic.csv
 data-10-10000.csv,/datasets/test/data-10-10000.csv
 data-100-10000.csv,/datasets/test/data-100-10000.csv
 data-20-10000.csv,/datasets/test/data-20-10000.csv

demo_data/synthetic.ipynb

@@ -222,12 +222,31 @@ function getMetadata(dataset) {
     return acc;
   }, {});
 
+  const classes = {
+    groundTruth: d3.rollup(dataset, v => v.length / dataset.length, d => d.label)
+  };
+
+  if (hasPredictions) {
+    // classes.predictionCounts = d3.rollup(dataset, v => v.length / dataset.length, d => d.prediction);
+    classes.predictions = d3.rollup(
+      dataset,
+      v =>
+        d3.rollup(
+          v,
+          g => g.length / dataset.length,
+          p => p.prediction === p.label ? "correct" : "incorrect"
+        ),
+      d => d.prediction
+    );
+  }
+
   return {
     features: features,
     featureNames: featureNames,
     labelValues: labelValues,
     hasPredictions: hasPredictions,
     size: dataset.length,
+    classes: classes,
   }
 }
 
@@ -246,6 +265,17 @@ function getData(metadata, selectedFeatures, dataset) {
       size: g.length
     };
 
+    const diff = {
+      groundTruth: new Map(
+        metadata.labelValues.map(label => [
+          label,
+          (groundTruth.get(label) / g.length) - metadata.classes.groundTruth.get(label)
+        ])
+      )
+    };
+
+    node.diff = diff;
+
     if (metadata.hasPredictions) {
       // if the dataset has model predictions,
       // also count the number of each prediction
@@ -268,6 +298,24 @@ function getData(metadata, selectedFeatures, dataset) {
 
       node['predictionCounts'] = predictionCounts;
       node['predictionResults'] = predictionResults;
+
+      node.diff.predictions = d3.cross(metadata.labelValues, ["correct", "incorrect"])
+            .map(([label, correct]) => {
+              const predictedCount = predictionResults.get(label)?.get(correct) ?? 0;
+              const predictedPercent = predictedCount / g.length;
+              const rootPercent = metadata.classes.predictions.get(label)?.get(correct) ?? 0;
+              const diffPercent = Math.max(0, predictedPercent - rootPercent);
+              const diffCount = diffPercent * g.length;
+              const sameCount = predictedCount - diffCount;
+
+              return {
+                label,
+                correct,
+                diffCount,
+                sameCount,
+                count: predictedCount,
+              };
+            })
     }
 
     return node;

public/datasets/datasets-test.csv

@@ -2,7 +2,8 @@ import {
   entropy,
   errorDeviation,
   errorCount,
-  errorPercent
+  errorPercent,
+  antiEntropy,
 } from './RatingMetrics.js';
 
 import * as d3 from "d3";
@@ -26,6 +27,8 @@ function getFeatureRatings({criterion, selected, metadata, dataset}) {
     ratings = errorPercent({selected, metadata, dataset, available});
   } else if (criterion === 'errorDeviation') {
     ratings = errorDeviation({selected, metadata, dataset, available})
+  } else if (criterion === 'antiEntropy') {
+    ratings = antiEntropy({selected, metadata, dataset, available})
   }
 
   return normalize(ratings);

public/datasets/normal.csv

@@ -8,6 +8,7 @@ export {
   errorCount,
   errorPercent,
   getErrorCountForSquare,
+  antiEntropy,
 };
 
 /*
@@ -20,7 +21,29 @@ function entropy({selected, metadata, dataset, available}) {
     const data = getData(metadata, sel, dataset);
 
     // give higher rating to lower entropy, so negate it
-    const value = -d3.sum(data, square => {
+    const value = 1-d3.sum(data, square => {
+      const weight = square.size / metadata.size;
+      return weight * H(square);
+    });
+
+    return {feature, value};
+  });
+
+  function H(square) {
+    return -d3.sum(square.groundTruth.values(), v => {
+      const p = v / square.size;
+      return p * Math.log2(p);
+    });
+  }
+}
+
+function antiEntropy({selected, metadata, dataset, available}) {
+  return available.map(feature => {
+    const sel = [...selected, feature];
+    const data = getData(metadata, sel, dataset);
+
+    // give higher rating to lower entropy, so negate it
+    const value = d3.sum(data, square => {
       const weight = square.size / metadata.size;
       return weight * H(square);
     });

public/datasets/synthetic.csv

@@ -9,13 +9,18 @@ import { getData } from './DataTransformer.js';
 
 import * as d3 from "d3";
 
-export { getRecommendedSubsets };
+export { getRecommendedSubsets, timeSubsets, topSubsets, randomSubsets };
+
+const criteria = {
+  'entropy': entropy,
+  'antiEntropy': antiEntropy,
+}
 
 function entropy({set, metadata, dataset}) {
   const data = getData(metadata, set, dataset);
 
   // give higher rating to lower entropy, so negate it
-  const value = -d3.sum(data, square => {
+  const value = 1 - d3.sum(data, square => {
     const weight = square.size / metadata.size;
     return weight * H(square);
   });
@@ -30,18 +35,64 @@ function entropy({set, metadata, dataset}) {
   }
 }
 
-function getRecommendedSubsets({criterion, selected, metadata, dataset}) {
+function antiEntropy({set, metadata, dataset}) {
+  return 1 - entropy({set, metadata, dataset});
+}
+
+function timeSubsets({criterion, selected, metadata, dataset}) {
+  const data = [];
+
+  d3.range(0.5, 1, 0.01).reverse().forEach(threshold => {
+    const t0 = performance.now();
+    const subsets = getRecommendedSubsets({criterion, selected, metadata, dataset}, threshold);
+    const t1 = performance.now();
+
+    data.push({
+      threshold,
+      percentBetter: 1 - threshold,
+      numSubsets: subsets.length,
+      ms: t1 - t0
+    });
+  });
+
+  console.log(JSON.stringify(data));
+}
+
+function topSubsets({criterion, selected, metadata, dataset}) {
+  const data = {};
+
+  [2, 3, 4].forEach(n => {
+    console.log(n);
+    const sets = getPermutations(metadata.featureNames, n)
+      .map(cand => ({
+        set: cand,
+        score: entropy({set: cand, metadata, dataset}),
+      }))
+      .sort((a, b) => d3.descending(a.score, b.score));
+
+    data[n] = sets;
+  });
+
+  console.log(JSON.stringify(data));
+}
+
+function randomSubsets({criterion, selected, metadata, dataset}) {
+  return d3.shuffle(getPermutations(metadata.featureNames, 2)).slice(0, 10);
+}
+
+function getRecommendedSubsets({criterion, selected, metadata, dataset}, percent=1.0) {
   const L = [getLarge1ItemSets({criterion, metadata, dataset})];
 
   for (let k = 1; k < 4; k++) {
     const min = d3.min(L[k - 1], d => d.score);
-    const threshold = min * .75;
+
+    const threshold = min * percent;
 
     const candidates = getCandidates(L[k - 1]);
 
     const candScores = candidates.map(cand => ({
       set: cand,
-      score: entropy({set: cand, metadata, dataset})
+      score: criteria[criterion]({set: cand, metadata, dataset})
     }));
 
     const valid = candScores.filter(({score}) => {
@@ -51,17 +102,20 @@ function getRecommendedSubsets({criterion, selected, metadata, dataset}) {
     L.push(valid);
   }
 
-  return L.flat().sort((a, b) => d3.descending(a.score, b.score)).map(d => d.set);
+  const sorted = L.flat().sort((a, b) => d3.descending(a.score, b.score));
+
+  return sorted.map(d => d.set);
 }
 
 function getLarge1ItemSets({criterion, metadata, dataset}) {
+  const numStart = Math.min(Math.floor(metadata.featureNames.length / 2), 10);
   const scores = metadata.featureNames.map(feature => {
     return {
-      score: entropy({set: [feature], metadata, dataset}),
+      score: criteria[criterion]({set: [feature], metadata, dataset}),
       set: [feature]
     }
   }).sort((a, b) => d3.descending(a.score, b.score))
-    .slice(0, 10);
+    .slice(0, numStart);
 
   return scores;
 }

src/DataTransformer.js

@@ -1,6 +1,9 @@
-import { getRecommendedSubsets } from './SubsetRecommender';
+import { getRecommendedSubsets, timeSubsets, topSubsets, randomSubsets } from './SubsetRecommender';
 
 self.onmessage = e => {
   const recommendedSubsets = getRecommendedSubsets(e.data);
+  // const recommendedSubsets = randomSubsets(e.data);
+  // const recommendedSubsets = timeSubsets(e.data);
+  // topSubsets(e.data);
   postMessage(recommendedSubsets);
 }