refactor Confusion Matrix API

packages/dispatcher/package.json

@@ -4,7 +4,7 @@
   "version": "1.0.0",
   "author": "Microsoft",
   "bugs": "https://github.com/microsoft/botframework-cli/issues",
-  "main": "lib/src/index.js",
+  "main": "lib/index.js",
   "dependencies": {
     "@microsoft/bf-lu": "1.0.0",
     "@oclif/command": "~1.5.19",

packages/dispatcher/src/data/AppColumnarData.ts

@@ -77,9 +77,9 @@ export function exampleFunctionData(): ColumnarData {
     const args: any = parsedKnownArgs[0];
     const unknownArgs: any = parsedKnownArgs[1];
     Utility.debuggingLog(
-        `args=${Utility.JSONstringify(args)}`);
+        `args=${Utility.jsonStringify(args)}`);
     Utility.debuggingLog(
-        `unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
+        `unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
     const debugFlag: boolean = Utility.toBoolean(args.debug);
     Utility.toPrintDebuggingLogToConsole = debugFlag;
     // ---- NOTE-FOR-DEBUGGING ----  console.dir(args);

packages/dispatcher/src/data/AppEntityAnnotatedCorpusData.ts

@@ -53,9 +53,9 @@ export function exampleFunctionData(): EntityAnnotatedCorpusData {
     const args: any = parsedKnownArgs[0];
     const unknownArgs: any = parsedKnownArgs[1];
     Utility.debuggingLog(
-        `args=${Utility.JSONstringify(args)}`);
+        `args=${Utility.jsonStringify(args)}`);
     Utility.debuggingLog(
-        `unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
+        `unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
     const debugFlag: boolean = Utility.toBoolean(args.debug);
     Utility.toPrintDebuggingLogToConsole = debugFlag;
     // ---- NOTE-FOR-DEBUGGING ----  console.dir(args);

packages/dispatcher/src/data/AppLuData.ts

@@ -43,9 +43,9 @@ export async function exampleFunctionData(): Promise<string[]> {
     const args: any = parsedKnownArgs[0];
     const unknownArgs: any = parsedKnownArgs[1];
     Utility.debuggingLog(
-        `args=${Utility.JSONstringify(args)}`);
+        `args=${Utility.jsonStringify(args)}`);
     Utility.debuggingLog(
-        `unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
+        `unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
     const debugFlag: boolean = Utility.toBoolean(args.debug);
     Utility.toPrintDebuggingLogToConsole = debugFlag;
     // ---- NOTE-FOR-DEBUGGING ----  console.dir(args);

packages/dispatcher/src/data/data.ts

@@ -290,13 +290,13 @@ export abstract class Data {
     public getFeaturizerLabels(): string[] {
         return this.getFeaturizer().getLabels();
     }
-    public getFeaturizerLabelMap(): { [id: string]: number; } {
+    public getFeaturizerLabelMap(): { [id: string]: number } {
         return this.getFeaturizer().getLabelMap();
     }
     public getFeaturizerFeatures(): string[] {
         return this.getFeaturizer().getFeatures();
     }
-    public getFeaturizerFeatureMap(): { [id: string]: number; } {
+    public getFeaturizerFeatureMap(): { [id: string]: number } {
         return this.getFeaturizer().getFeatureMap();
     }
 

packages/dispatcher/src/data_structure/IDictionaryNumberIdGenericSet.ts

@@ -0,0 +1,8 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export interface IDictionaryNumberIdGenericSet<T> {
+    [id: number]: Set<T>;
+}

packages/dispatcher/src/data_structure/IDictionaryStringIdGenericSet.ts

@@ -0,0 +1,8 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export interface IDictionaryStringIdGenericSet<T> {
+    [id: string]: Set<T>;
+}

packages/dispatcher/src/data_structure/TMapGenericKeyGenericSet.ts

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export type TMapGenericKeyGenericSet<I, T> = Map<I, Set<T>>;

packages/dispatcher/src/data_structure/TMapNumberKeyGenericSet.ts

@@ -0,0 +1,7 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { TMapGenericKeyGenericSet } from "./TMapGenericKeyGenericSet";
+export type TMapNumberKeyGenericSet<T> = TMapGenericKeyGenericSet<number, T>;

packages/dispatcher/src/data_structure/TMapStringKeyGenericSet.ts

@@ -0,0 +1,7 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { TMapGenericKeyGenericSet } from "./TMapGenericKeyGenericSet";
+export type TMapStringKeyGenericSet<T> = TMapGenericKeyGenericSet<string, T>;

packages/dispatcher/src/index.ts

@@ -3,32 +3,169 @@
  * Licensed under the MIT License.
  */
 
-import { AppAutoActiveLearner } from "./model/supervised/classifier/auto_active_learning/AppAutoActiveLearner";
+import { ColumnarData } from "./data/ColumnarData";
+import { Data } from "./data/Data";
+import { DataUtility } from "./data/DataUtility";
+import { EntityAnnotatedCorpusData } from "./data/EntityAnnotatedCorpusData";
+import { LuData } from "./data/LuData";
+
+import { DictionaryMapUtility } from "./data_structure/DictionaryMapUtility";
+import { IDictionaryNumberIdGenericArray } from "./data_structure/IDictionaryNumberIdGenericArray";
+import { IDictionaryNumberIdGenericArrays } from "./data_structure/IDictionaryNumberIdGenericArrays";
+import { IDictionaryNumberIdGenericSet } from "./data_structure/IDictionaryNumberIdGenericSet";
+import { IDictionaryNumberIdGenericValue } from "./data_structure/IDictionaryNumberIdGenericValue";
+import { IDictionaryStringIdGenericArray } from "./data_structure/IDictionaryStringIdGenericArray";
+import { IDictionaryStringIdGenericArrays } from "./data_structure/IDictionaryStringIdGenericArrays";
+import { IDictionaryStringIdGenericSet } from "./data_structure/IDictionaryStringIdGenericSet";
+import { IDictionaryStringIdGenericValue } from "./data_structure/IDictionaryStringIdGenericValue";
+import { TMapGenericKeyGenericArray } from "./data_structure/TMapGenericKeyGenericArray";
+import { TMapGenericKeyGenericArrays } from "./data_structure/TMapGenericKeyGenericArrays";
+import { TMapGenericKeyGenericSet } from "./data_structure/TMapGenericKeyGenericSet";
+import { TMapGenericKeyGenericValue } from "./data_structure/TMapGenericKeyGenericValue";
+import { TMapNumberKeyGenericArray } from "./data_structure/TMapNumberKeyGenericArray";
+import { TMapNumberKeyGenericArrays } from "./data_structure/TMapNumberKeyGenericArrays";
+import { TMapNumberKeyGenericSet } from "./data_structure/TMapNumberKeyGenericSet";
+import { TMapNumberKeyGenericValue } from "./data_structure/TMapNumberKeyGenericValue";
+import { TMapStringKeyGenericArray } from "./data_structure/TMapStringKeyGenericArray";
+import { TMapStringKeyGenericArrays } from "./data_structure/TMapStringKeyGenericArrays";
+import { TMapStringKeyGenericSet } from "./data_structure/TMapStringKeyGenericSet";
+import { TMapStringKeyGenericValue } from "./data_structure/TMapStringKeyGenericValue";
 
 import { mainConfusionMatrix } from "./mathematics/confusion_matrix/AppConfusionMatrix";
 import { mainConfusionMatrixFunction } from "./mathematics/confusion_matrix/AppConfusionMatrix";
+import { BinaryConfusionMatrix } from "./mathematics/confusion_matrix/BinaryConfusionMatrix";
 import { ConfusionMatrix } from "./mathematics/confusion_matrix/ConfusionMatrix";
+import { ConfusionMatrixBase } from "./mathematics/confusion_matrix/ConfusionMatrixBase";
+import { IConfusionMatrix } from "./mathematics/confusion_matrix/IConfusionMatrix";
+import { IMultiLabelConfusionMatrix } from "./mathematics/confusion_matrix/IMultiLabelConfusionMatrix";
+import { MultiLabelConfusionMatrix } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrix";
+import { MultiLabelConfusionMatrixBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixBase";
+import { MultiLabelConfusionMatrixSubset } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixSubset";
+import { MultiLabelConfusionMatrixWithBinaryArrayBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixWithBinaryArrayBase";
+import { MultiLabelConfusionMatrixWithBinaryBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixWithBinaryBase";
+
+import { IMathematicsHelper } from "./mathematics/mathematics_helper/IMathematicsHelper";
+import { MathematicsHelper } from "./mathematics/mathematics_helper/MathematicsHelper";
+
+import { AbstractBaseBootstrapSampler } from "./mathematics/sampler/AbstractBaseBootstrapSampler";
+import { AbstractBaseReservoirSampler } from "./mathematics/sampler/AbstractBaseReservoirSampler";
+import { AbstractBaseSampler } from "./mathematics/sampler/AbstractBaseSampler";
+import { BootstrapSampler } from "./mathematics/sampler/BootstrapSampler";
+import { BootstrapSamplerDistribution } from "./mathematics/sampler/BootstrapSamplerDistribution";
+import { BootstrapSamplerKeyMap } from "./mathematics/sampler/BootstrapSamplerKeyMap";
+import { BootstrapSamplerKeyMapDistribution } from "./mathematics/sampler/BootstrapSamplerKeyMapDistribution";
+import { ReservoirArraySampler } from "./mathematics/sampler/ReservoirArraySampler";
+import { ReservoirSampler } from "./mathematics/sampler/ReservoirSampler";
+import { ReservoirSamplerKeyMap } from "./mathematics/sampler/ReservoirSamplerKeyMap";
 
 import { mainCrossValidatorWithColumnarContent } from "./model/evaluation/cross_validation/AppCrossValidator";
 import { mainCrossValidatorWithLuContent } from "./model/evaluation/cross_validation/AppCrossValidator";
 import { mainCrossValidator } from "./model/evaluation/cross_validation/AppCrossValidator";
+import { CrossValidator } from "./model/evaluation/cross_validation/CrossValidator";
 
 import { mainPredictor } from "./model/evaluation/predict/AppPredictor";
+import { Predictor } from "./model/evaluation/predict/Predictor";
 
 import { mainDataProfileReporter } from "./model/evaluation/report/AppDataProfileReporter";
+import { DataProfileReporter } from "./model/evaluation/report/DataProfileReporter";
 import { mainModelMetaDataProfileReporter } from "./model/evaluation/report/AppModelMetaDataProfileReporter";
+import { ModelMetaDataProfileReporter } from "./model/evaluation/report/ModelMetaDataProfileReporter";
 import { mainThresholdReporter } from "./model/evaluation/report/AppThresholdReporter";
+import { ThresholdReporter } from "./model/evaluation/report/ThresholdReporter";
+
+import { mainTester } from "./model/evaluation/test/AppTester";
+import { Tester } from "./model/evaluation/test/Tester";
+
+import { AppAutoActiveLearner } from "./model/supervised/classifier/auto_active_learning/AppAutoActiveLearner";
+import { AutoActiveLearner } from "./model/supervised/classifier/auto_active_learning/AutoActiveLearner";
+
+import { AppSoftmaxRegressionSparse } from "./model/supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
+import { SoftmaxRegressionSparse } from "./model/supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
+
+import { Utility } from "./utility/Utility";
+import { ListArrayUtility } from "./utility/ListArrayUtility";
 
 export default {
-    AppAutoActiveLearner,
+    ColumnarData,
+    Data,
+    DataUtility,
+    EntityAnnotatedCorpusData,
+    LuData
+    ,
+    DictionaryMapUtility
+    // IDictionaryNumberIdGenericArray,
+    // IDictionaryNumberIdGenericArrays,
+    // IDictionaryNumberIdGenericSet,
+    // IDictionaryNumberIdGenericValue,
+    // IDictionaryStringIdGenericArray,
+    // IDictionaryStringIdGenericArrays,
+    // IDictionaryStringIdGenericSet,
+    // IDictionaryStringIdGenericValue,
+    // TMapGenericKeyGenericArray,
+    // TMapGenericKeyGenericArrays,
+    // TMapGenericKeyGenericSet,
+    // TMapGenericKeyGenericValue,
+    // TMapNumberKeyGenericArray,
+    // TMapNumberKeyGenericArrays,
+    // TMapNumberKeyGenericSet,
+    // TMapNumberKeyGenericValue,
+    // TMapStringKeyGenericArray,
+    // TMapStringKeyGenericArrays,
+    // TMapStringKeyGenericSet,
+    // TMapStringKeyGenericValue
+    ,
     mainConfusionMatrix,
     mainConfusionMatrixFunction,
+    BinaryConfusionMatrix,
     ConfusionMatrix,
-    mainCrossValidator,
-    mainCrossValidatorWithLuContent,
+    ConfusionMatrixBase,
+    // IConfusionMatrix,
+    // IMultiLabelConfusionMatrix,
+    MultiLabelConfusionMatrix,
+    MultiLabelConfusionMatrixBase,
+    MultiLabelConfusionMatrixSubset,
+    MultiLabelConfusionMatrixWithBinaryArrayBase,
+    MultiLabelConfusionMatrixWithBinaryBase
+    ,
+    // IMathematicsHelper,
+    MathematicsHelper
+    ,
+    AbstractBaseBootstrapSampler,
+    AbstractBaseReservoirSampler,
+    AbstractBaseSampler,
+    BootstrapSampler,
+    BootstrapSamplerDistribution,
+    BootstrapSamplerKeyMap,
+    BootstrapSamplerKeyMapDistribution,
+    ReservoirArraySampler,
+    ReservoirSampler,
+    ReservoirSamplerKeyMap
+    ,
     mainCrossValidatorWithColumnarContent,
+    mainCrossValidatorWithLuContent,
+    mainCrossValidator,
+    CrossValidator
+    ,
     mainPredictor,
+    Predictor
+    ,
     mainDataProfileReporter,
+    DataProfileReporter,
     mainModelMetaDataProfileReporter,
+    ModelMetaDataProfileReporter,
     mainThresholdReporter,
+    ThresholdReporter
+    ,
+    mainTester,
+    Tester
+    ,
+    AppAutoActiveLearner,
+    AutoActiveLearner
+    ,
+    AppSoftmaxRegressionSparse,
+    SoftmaxRegressionSparse
+    ,
+    Utility,
+    ListArrayUtility
+    ,
 };

packages/dispatcher/src/mathematics/confusion_matrix/AppConfusionMatrix.ts

@@ -27,7 +27,7 @@ export function mainConfusionMatrixFunction(
     revisedTextColumnIndex: number,
     lineIndexToStart: number): {
         "labels": string[],
-        "labelMap": { [id: string]: number; },
+        "labelMap": { [id: string]: number },
         "binaryConfusionMatrices": BinaryConfusionMatrix[],
         "confusionMatrix": ConfusionMatrix } {
     // -----------------------------------------------------------------------
@@ -39,9 +39,9 @@ export function mainConfusionMatrixFunction(
     }
     // -----------------------------------------------------------------------
     let labels: string[] = [];
-    let labelMap: { [id: string]: number; } = {};
+    let labelMap: { [id: string]: number } = {};
     if (!Utility.isEmptyString(labelFilename)) {
-        const labelsAndLabelMap: { "stringArray": string[], "stringMap": { [id: string]: number; } } =
+        const labelsAndLabelMap: { "stringArray": string[], "stringMap": { [id: string]: number } } =
             DictionaryMapUtility.buildStringIdNumberValueDictionaryFromUniqueStringArrayFile(labelFilename);
         labels = labelsAndLabelMap.stringArray;
         labelMap = labelsAndLabelMap.stringMap;
@@ -90,7 +90,7 @@ export function mainConfusionMatrixFunction(
         if (label === predictedLabel) {
             numberMatches++;
         }
-        confusionMatrix.addInstance(label, predictedLabel);
+        confusionMatrix.addInstanceByLabel(label, predictedLabel);
     }
     // -----------------------------------------------------------------------
     Utility.debuggingLog(
@@ -103,7 +103,7 @@ export function mainConfusionMatrixFunction(
     Utility.debuggingLog(
         "labels=" + confusionMatrix.getLabels());
     Utility.debuggingLog(
-        Utility.JSONstringify(confusionMatrix.getLabelMap()));
+        Utility.jsonStringify(confusionMatrix.getLabelMap()));
     Utility.debuggingLog(
         "rows=" + confusionMatrix.getConfusionMatrixRows());
     Utility.debuggingLog(
@@ -246,9 +246,9 @@ export function mainConfusionMatrix(): void {
     const args: any = parsedKnownArgs[0];
     const unknownArgs: any = parsedKnownArgs[1];
     Utility.debuggingLog(
-        `args=${Utility.JSONstringify(args)}`);
+        `args=${Utility.jsonStringify(args)}`);
     Utility.debuggingLog(
-        `unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
+        `unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
     const debugFlag: boolean = Utility.toBoolean(args.debug);
     Utility.toPrintDebuggingLogToConsole = debugFlag;
     // ---- NOTE-FOR-DEBUGGING ----  console.dir(args);
@@ -302,7 +302,7 @@ export function mainConfusionMatrix(): void {
     // -----------------------------------------------------------------------
     const mainConfusionMatrixFunctionResult: {
         "labels": string[],
-        "labelMap": { [id: string]: number; },
+        "labelMap": { [id: string]: number },
         "confusionMatrix": ConfusionMatrix } = mainConfusionMatrixFunction(
         scoreFilename,
         labelFilename,
@@ -326,24 +326,24 @@ export function mainConfusionMatrix(): void {
 
 export function exampleFunctionConfusionMatrix(): void {
     const labels: string[] = [ "label0", "label1", "label2" ];
-    const labelMap: { [id: string]: number; } = { };
+    const labelMap: { [id: string]: number } = { };
     labelMap.label0 = 0;
     labelMap.label1 = 1;
     labelMap.label2 = 2;
     const confusionMatrix = new ConfusionMatrix(labels, labelMap);
-    confusionMatrix.addInstance("label0", "label0");
-    confusionMatrix.addInstance("label0", "label1");
-    confusionMatrix.addInstance("label0", "label2");
-    confusionMatrix.addInstance("label1", "label0");
-    confusionMatrix.addInstance("label1", "label1");
-    confusionMatrix.addInstance("label1", "label2");
-    confusionMatrix.addInstance("label2", "label0");
-    confusionMatrix.addInstance("label2", "label1");
-    confusionMatrix.addInstance("label2", "label2");
+    confusionMatrix.addInstanceByLabel("label0", "label0");
+    confusionMatrix.addInstanceByLabel("label0", "label1");
+    confusionMatrix.addInstanceByLabel("label0", "label2");
+    confusionMatrix.addInstanceByLabel("label1", "label0");
+    confusionMatrix.addInstanceByLabel("label1", "label1");
+    confusionMatrix.addInstanceByLabel("label1", "label2");
+    confusionMatrix.addInstanceByLabel("label2", "label0");
+    confusionMatrix.addInstanceByLabel("label2", "label1");
+    confusionMatrix.addInstanceByLabel("label2", "label2");
     Utility.debuggingLog(
         "labels=" + confusionMatrix.getLabels());
     Utility.debuggingLog(
-        Utility.JSONstringify(confusionMatrix.getLabelMap()));
+        Utility.jsonStringify(confusionMatrix.getLabelMap()));
     Utility.debuggingLog(
         "rows=" + confusionMatrix.getConfusionMatrixRows());
     Utility.debuggingLog(