Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

refactor evaluation API #688

Merged
hcyang merged 3 commits into from
Mar 26, 2020
Merged

refactor evaluation API #688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d854c6
refactor evaluation API
hcyang Mar 25, 2020
e7b2b25
fixed missing semicolons
hcyang Mar 26, 2020
be8ce6c
Merge branch 'master' into hunyang/dispatcher_refactorEvaluationAPI
hcyang Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/dispatcher/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
"@microsoft/bf-lu": "1.0.0",
"@oclif/command": "~1.5.19",
"@oclif/config": "~1.13.3",
"tslib": "^1.10.0"
"tslib": "^1.10.0",
Copy link
Contributor

@tsuwandy tsuwandy Mar 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@hcyang , shouldn't this be dev dependencies?

"argparse": "~1.0.10"
},
"devDependencies": {
"@oclif/dev-cli": "^1.22.2",
Expand Down
4 changes: 2 additions & 2 deletions packages/dispatcher/src/data/AppColumnarData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ export function exampleFunctionData(): ColumnarData {
const args: any = parsedKnownArgs[0];
const unknownArgs: any = parsedKnownArgs[1];
Utility.debuggingLog(
`args=${JSON.stringify(args)}`);
`args=${Utility.JSONstringify(args)}`);
Utility.debuggingLog(
`unknownArgs=${JSON.stringify(unknownArgs)}`);
`unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
const debugFlag: boolean = Utility.toBoolean(args.debug);
Utility.toPrintDebuggingLogToConsole = debugFlag;
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
Expand Down
4 changes: 2 additions & 2 deletions packages/dispatcher/src/data/AppEntityAnnotatedCorpusData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ export function exampleFunctionData(): EntityAnnotatedCorpusData {
const args: any = parsedKnownArgs[0];
const unknownArgs: any = parsedKnownArgs[1];
Utility.debuggingLog(
`args=${JSON.stringify(args)}`);
`args=${Utility.JSONstringify(args)}`);
Utility.debuggingLog(
`unknownArgs=${JSON.stringify(unknownArgs)}`);
`unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
const debugFlag: boolean = Utility.toBoolean(args.debug);
Utility.toPrintDebuggingLogToConsole = debugFlag;
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
Expand Down
4 changes: 2 additions & 2 deletions packages/dispatcher/src/data/AppLuData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ export async function exampleFunctionData(): Promise<string[]> {
const args: any = parsedKnownArgs[0];
const unknownArgs: any = parsedKnownArgs[1];
Utility.debuggingLog(
`args=${JSON.stringify(args)}`);
`args=${Utility.JSONstringify(args)}`);
Utility.debuggingLog(
`unknownArgs=${JSON.stringify(unknownArgs)}`);
`unknownArgs=${Utility.JSONstringify(unknownArgs)}`);
const debugFlag: boolean = Utility.toBoolean(args.debug);
Utility.toPrintDebuggingLogToConsole = debugFlag;
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
Expand Down
35 changes: 22 additions & 13 deletions packages/dispatcher/src/data/DataUtility.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
* Licensed under the MIT License.
*/

import { isNull } from "util";
import { isUndefined } from "util";

import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";

import { EntityAnnotatedCorpusData } from "./EntityAnnotatedCorpusData";
Expand Down Expand Up @@ -43,6 +46,8 @@ export class DataUtility {
}
public static async LoadData(
filename: string,
featurizerNullable: NgramSubwordFeaturizer|null = null,
toResetFeaturizerLabelFeatureMaps: boolean = true,
filetype: string = "",
labelColumnIndex: number = 0,
textColumnIndex: number = 1,
Expand All @@ -52,40 +57,44 @@ export class DataUtility {
if (Utility.isEmptyString(filetype)) {
filetype = DataUtility.getDataFileTypeFromFilenameExtension(filename);
}
if (isNull(featurizerNullable) || (isUndefined(featurizerNullable))) {
featurizerNullable = new NgramSubwordFeaturizer();
toResetFeaturizerLabelFeatureMaps = true;
}
if (filetype === DataUtility.DATA_FORMAT_TYPE_LU) {
const luData: LuData =
await LuData.createLuData(
content,
new NgramSubwordFeaturizer(),
true);
featurizerNullable,
toResetFeaturizerLabelFeatureMaps);
return luData;
}
if (filetype === DataUtility.DATA_FORMAT_TYPE_QUESTION_AND_ANSWER) {
const luData: LuData =
await LuData.createLuData(
content,
new NgramSubwordFeaturizer(),
true);
featurizerNullable,
toResetFeaturizerLabelFeatureMaps);
return luData;
}
if (filetype === DataUtility.DATA_FORMAT_TYPE_ENTITY_ANNOTATED_CORPUS) {
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
EntityAnnotatedCorpusData.createEntityAnnotatedCorpusData(
content,
new NgramSubwordFeaturizer(),
featurizerNullable,
linesToSkip,
true);
toResetFeaturizerLabelFeatureMaps);
return entityAnnotatedCorpusData;
}
{
const columnarData: ColumnarData =
ColumnarData.createColumnarData(
content,
new NgramSubwordFeaturizer(),
labelColumnIndex,
textColumnIndex,
linesToSkip,
true);
ColumnarData.createColumnarData(
content,
featurizerNullable,
labelColumnIndex,
textColumnIndex,
linesToSkip,
toResetFeaturizerLabelFeatureMaps);
return columnarData;
}
}
Expand Down
62 changes: 38 additions & 24 deletions packages/dispatcher/src/mathematics/confusion_matrix/ConfusionMatrix.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ export class ConfusionMatrix {
"macroAverageMetrics": { "averagePrecision": number,
"averageRecall": number,
"averageF1Score": number,
"totalMacroAverage": number },
"support": number },
"microAverageMetrics": { "accuracy": number,
"truePositives": number,
"totalMicroAverage": number },
"support": number },
"weightedMacroAverageMetrics": { "weightedAveragePrecision": number,
"weightedAverageRecall": number,
"weightedAverageF1Score": number,
"weightedTotalMacroAverage": number } } {
"weightedAverageRecall": number,
"weightedAverageF1Score": number,
"support": number } } {
const crossValidationBinaryConfusionMatrix: BinaryConfusionMatrix[] =
confusionMatrix.getBinaryConfusionMatrices();
const labelMap: { [id: string]: number; } =
Expand All @@ -45,14 +45,18 @@ export class ConfusionMatrix {
microAverageMetricArray[0];
const truePositives: number =
microAverageMetricArray[1];
const totalMicroAverage: number =
const supportMicroAverage: number =
microAverageMetricArray[2];
const microAverageMetrics: {
"accuracy": number,
"truePositives": number,
"totalMicroAverage": number } = { accuracy,
truePositives,
totalMicroAverage };
"support": number } = {
// tslint:disable-next-line: object-literal-key-quotes
"accuracy": accuracy,
// tslint:disable-next-line: object-literal-key-quotes
"truePositives": truePositives,
// tslint:disable-next-line: object-literal-key-quotes
"support": supportMicroAverage };
const macroAverageMetricArray: [number, number, number, number] =
confusionMatrix.getMacroAverageMetrics();
const averagePrecision: number =
Expand All @@ -61,16 +65,21 @@ export class ConfusionMatrix {
macroAverageMetricArray[1];
const averageF1Score: number =
macroAverageMetricArray[2];
const totalMacroAverage: number =
const supportMacroAverage: number =
macroAverageMetricArray[3];
const macroAverageMetrics: {
"averagePrecision": number,
"averageRecall": number,
"averageF1Score": number,
"totalMacroAverage": number } = { averagePrecision,
averageRecall,
averageF1Score,
totalMacroAverage };
"support": number } = {
// tslint:disable-next-line: object-literal-key-quotes
"averagePrecision": averagePrecision,
// tslint:disable-next-line: object-literal-key-quotes
"averageRecall": averageRecall,
// tslint:disable-next-line: object-literal-key-quotes
"averageF1Score": averageF1Score,
// tslint:disable-next-line: object-literal-key-quotes
"support": supportMacroAverage };
const weightedMacroAverageMetricArray: [number, number, number, number] =
confusionMatrix.getWeightedMacroAverageMetrics();
const weightedAveragePrecision: number =
Expand All @@ -79,31 +88,36 @@ export class ConfusionMatrix {
weightedMacroAverageMetricArray[1];
const weightedAverageF1Score: number =
weightedMacroAverageMetricArray[2];
const weightedTotalMacroAverage: number =
const supportWeightedMacroAverage: number =
weightedMacroAverageMetricArray[3];
const weightedMacroAverageMetrics: {
"weightedAveragePrecision": number,
"weightedAverageRecall": number,
"weightedAverageF1Score": number,
"weightedTotalMacroAverage": number } = { weightedAveragePrecision,
weightedAverageRecall,
weightedAverageF1Score,
weightedTotalMacroAverage };
"support": number } = {
// tslint:disable-next-line: object-literal-key-quotes
"weightedAveragePrecision": weightedAveragePrecision,
// tslint:disable-next-line: object-literal-key-quotes
"weightedAverageRecall": weightedAverageRecall,
// tslint:disable-next-line: object-literal-key-quotes
"weightedAverageF1Score": weightedAverageF1Score,
// tslint:disable-next-line: object-literal-key-quotes
"support": supportWeightedMacroAverage };
const confusionMatrixMetricStructure: {
"confusionMatrix": ConfusionMatrix,
"labelBinaryConfusionMatrixDerivedMetricMap": { [id: string]: { [id: string]: number }; },
"labelBinaryConfusionMatrixMetricMap": { [id: string]: BinaryConfusionMatrix; },
"macroAverageMetrics": { "averagePrecision": number,
"averageRecall": number,
"averageF1Score": number,
"totalMacroAverage": number },
"support": number },
"microAverageMetrics": { "accuracy": number,
"truePositives": number,
"totalMicroAverage": number },
"support": number },
"weightedMacroAverageMetrics": { "weightedAveragePrecision": number,
"weightedAverageRecall": number,
"weightedAverageF1Score": number,
"weightedTotalMacroAverage": number } } = {
"weightedAverageRecall": number,
"weightedAverageF1Score": number,
"support": number } } = {
confusionMatrix,
labelBinaryConfusionMatrixDerivedMetricMap,
labelBinaryConfusionMatrixMetricMap,
Expand Down
Loading