[Perf Tests] Updates to perf test framework (#12662)

Reference - #12653 This PR mainly focuses on - improving the type inference for the options(uses typescript magic to achieve it) - introduces a new parsedOptions that can be consumed in the tests rather than overriding and consuming the original options bag - required for #12737 which adds tests for storage-blob Any issues with the functionality of the options or otherwise will be caught while we add new tests in upcoming PRs.
Azure · Dec 2, 2020 · 0096515 · 0096515
1 parent aae6833
commit 0096515
Show file tree

Hide file tree

Showing 12 changed files with 106 additions and 96 deletions.
diff --git a/sdk/test-utils/perfstress/package.json b/sdk/test-utils/perfstress/package.json
@@ -2,7 +2,7 @@
   "name": "@azure/test-utils-perfstress",
   "version": "1.0.0",
   "description": "Performance and stress test framework for the Azure SDK for JavaScript and TypeScript",
-  "main": "dist/index.js",
+  "main": "dist-esm/src/index.js",
   "module": "dist-esm/src/index.js",
   "browser": {},
   "types": "./typings/src/index.d.ts",
@@ -24,7 +24,7 @@
     "lint": "eslint package.json src test --ext .ts -f html -o perfstress-lintReport.html || exit 0",
     "pack": "npm pack 2>&1",
     "prebuild": "npm run clean",
-    "perfstress-test:node": "rushx build && node dist-esm/test/index.spec.js",
+    "perfstress-test:node": "npm run build && node dist-esm/test/index.spec.js",
     "unit-test:browser": "echo skipped",
     "unit-test:node": "echo skipped",
     "unit-test": "npm run unit-test:node && npm run unit-test:browser",

diff --git a/sdk/test-utils/perfstress/src/options.ts b/sdk/test-utils/perfstress/src/options.ts
@@ -3,15 +3,10 @@
 
 import { default as minimist, ParsedArgs as MinimistParsedArgs } from "minimist";
 
-/**
- * Possible values for each PerfStress option
- */
-export type PerfStressOptionValue = string | number | boolean | undefined;
-
 /**
  * The structure of a PerfStress option. They represent command line parameters.
  */
-export interface PerfStressOption {
+export interface OptionDetails<TType> {
   /**
    * Whether the option is required or not.
    */
@@ -30,14 +25,15 @@ export interface PerfStressOption {
   longName?: string;
   /**
    * The default value that is going to be assigned to the option.
+   * Expected: string | number | boolean | undefined
    */
-  defaultValue?: PerfStressOptionValue;
+  defaultValue?: TType;
   /**
    * The value specified by the user from the command line after either the shortName or the longName.
    * If no value was specified, the defaultValue will be used.
    * If the shortName or longName was specified, but no value was provided, "true" will be set.
    */
-  value?: PerfStressOptionValue;
+  value?: TType;
   /**
    * The description of each option. Descriptions of the assigned options will be shown at the beginning of the test call.
    * Descriptions of all the available options will be shown if the user sends either --help or -h.
@@ -49,30 +45,33 @@ export interface PerfStressOption {
  * A group of options is called PerfStressOptionDictionary,
  * and is shaped as a plain object to make it easier to access them.
  *
- * TNames defines the names of the options. This is necessary to allow TypeScript to suggest the appropriate names
- * for the options.
+ * `keyof TOptions` provides the names of the options. This is necessary to allow TypeScript to suggest the appropriate names
+ * for the options and parsedOptions.
  */
-export type PerfStressOptionDictionary<TNames extends string> = {
-  [longName in TNames]: PerfStressOption;
+export type PerfStressOptionDictionary<TOptions = {}> = {
+  [longName in keyof TOptions]: OptionDetails<TOptions[longName]>;
 };
 
 /**
- * These are the default options longNames.
+ * These represent the default options the tests can assume.
+ *
+ * @interface DefaultPerfStressOptions
  */
-export type DefaultPerfStressOptionNames =
-  | "help"
-  | "parallel"
-  | "duration"
-  | "warmup"
-  | "iterations"
-  | "no-cleanup"
-  | "milliseconds-to-log"
-  | "sync";
+export interface DefaultPerfStressOptions {
+  help: string;
+  parallel: number;
+  duration: number;
+  warmup: number;
+  iterations: number;
+  "no-cleanup": boolean;
+  "milliseconds-to-log": number;
+  sync: boolean;
+}
 
 /**
  * These are the default options in full.
  */
-export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStressOptionNames> = {
+export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStressOptions> = {
   help: {
     description: "Shows all of the available options",
     shortName: "h"
@@ -114,16 +113,18 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStr
 /**
  * Parses the given options by extracting their values through `minimist`, or setting the default value defined in each option.
  * It also overwrites any present longName with the property name of each option.
+ *
  * @param options A dictionary of options to parse using minimist.
  * @returns A new options dictionary.
  */
-export function parsePerfStressOption(
-  options: PerfStressOptionDictionary<string>
-): PerfStressOptionDictionary<string> {
+export function parsePerfStressOption<TOptions>(
+  options: PerfStressOptionDictionary<TOptions>
+): Required<PerfStressOptionDictionary<TOptions>> {
   const minimistResult: MinimistParsedArgs = minimist(process.argv);
-  const result: PerfStressOptionDictionary<string> = {};
+  const result = {};
 
   for (const longName of Object.keys(options)) {
+    // This cast is needed since we're picking up options from process.argv
     const option = (options as any)[longName];
     const { shortName, defaultValue, required } = option;
     const value =
@@ -139,5 +140,5 @@ export function parsePerfStressOption(
     };
   }
 
-  return result;
+  return result as Required<PerfStressOptionDictionary<TOptions>>;
 }
diff --git a/sdk/test-utils/perfstress/src/program.ts b/sdk/test-utils/perfstress/src/program.ts
@@ -7,7 +7,7 @@ import {
   PerfStressOptionDictionary,
   parsePerfStressOption,
   defaultPerfStressOptions,
-  DefaultPerfStressOptionNames
+  DefaultPerfStressOptions
 } from "./options";
 import { PerfStressParallel } from "./parallel";
 
@@ -35,9 +35,9 @@ export type TestType = "";
  */
 export class PerfStressProgram {
   private testName: string;
-  private options: PerfStressOptionDictionary<DefaultPerfStressOptionNames>;
+  private parsedDefaultOptions: Required<PerfStressOptionDictionary<DefaultPerfStressOptions>>;
   private parallelNumber: number;
-  private tests: PerfStressTest<string>[];
+  private tests: PerfStressTest[];
 
   /**
    * Receives a test class to instantiate and execute.
@@ -47,17 +47,16 @@ export class PerfStressProgram {
    *
    * @param testClass The testClass to be instantiated.
    */
-  constructor(testClass: PerfStressTestConstructor<string>) {
+  constructor(testClass: PerfStressTestConstructor) {
     this.testName = testClass.name;
-    this.options = parsePerfStressOption(defaultPerfStressOptions);
-    this.parallelNumber = Number(this.options.parallel.value);
+    this.parsedDefaultOptions = parsePerfStressOption(defaultPerfStressOptions);
+    this.parallelNumber = Number(this.parsedDefaultOptions.parallel.value);
 
     console.log(`=== Creating ${this.parallelNumber} instance(s) of ${this.testName} ===`);
-    this.tests = new Array<PerfStressTest<string>>(this.parallelNumber);
+    this.tests = new Array<PerfStressTest<DefaultPerfStressOptions>>(this.parallelNumber);
 
     for (let i = 0; i < this.parallelNumber; i++) {
       const test = new testClass();
-      test.parseOptions();
       this.tests[i] = test;
     }
   }
@@ -108,7 +107,7 @@ export class PerfStressProgram {
    * @param abortController Allows us to send through a signal determining when to abort any execution.
    */
   private runLoopSync(
-    test: PerfStressTest<string>,
+    test: PerfStressTest,
     parallel: PerfStressParallel,
     durationMilliseconds: number,
     abortController: AbortController
@@ -149,7 +148,7 @@ export class PerfStressProgram {
    * @param abortController Allows us to send through a signal determining when to abort any execution.
    */
   private async runLoopAsync(
-    test: PerfStressTest<string>,
+    test: PerfStressTest,
     parallel: PerfStressParallel,
     durationMilliseconds: number,
     abortController: AbortController
@@ -195,12 +194,12 @@ export class PerfStressProgram {
     // For this reason, we also check if the time has passed inside of runLoop.
     setTimeout(() => abortController.abort(), durationMilliseconds);
 
-    const parallel = Number(this.options.parallel.value);
+    const parallel = Number(this.parsedDefaultOptions.parallel.value);
 
     // This is how we customize how frequently we log how many completed operations have been executed.
     // We don't enforce this inside of runLoop, so it might never be executed, depending on the number
     // of operations running.
-    const millisecondsToLog = Number(this.options["milliseconds-to-log"].value);
+    const millisecondsToLog = Number(this.parsedDefaultOptions["milliseconds-to-log"].value);
     console.log(
       `\n=== ${title} mode, iteration ${iterationIndex}. Logs every ${millisecondsToLog /
         1000}s ===`
@@ -214,7 +213,7 @@ export class PerfStressProgram {
       lastInIteration = inTotal;
     }, millisecondsToLog);
 
-    const isAsync = !this.options.sync.value;
+    const isAsync = !this.parsedDefaultOptions.sync.value;
     const runLoop = isAsync ? this.runLoopAsync : this.runLoopSync;
 
     // Unhandled exceptions should stop the whole PerfStress process.
@@ -279,18 +278,15 @@ export class PerfStressProgram {
   public async run(): Promise<void> {
     // There should be no test execution if the help option is passed.
     // --help, or -h
-    if (this.options.help.value) {
+    if (this.parsedDefaultOptions.help.value) {
       console.log(`=== Help: Options that can be sent to ${this.testName} ===`);
-      console.table(this.tests[0].options);
+      console.table(this.tests[0].parsedOptions);
       return;
     }
 
-    const options = this.options;
+    const options = this.tests[0].parsedOptions;
     console.log("=== Parsed options ===");
-    console.table({
-      ...options,
-      ...this.tests[0].options
-    });
+    console.table(options);
 
     if (this.tests[0].globalSetup) {
       console.log(
@@ -311,7 +307,7 @@ export class PerfStressProgram {
       await this.runTest(0, Number(options.warmup.value), "warmup");
     }
 
-    const iterations = Number(this.options.iterations.value);
+    const iterations = Number(options.iterations.value);
     for (let i = 0; i < iterations; i++) {
       await this.runTest(i, Number(options.duration.value), "test");
     }

diff --git a/sdk/test-utils/perfstress/src/tests.ts b/sdk/test-utils/perfstress/src/tests.ts
@@ -3,13 +3,18 @@
 
 import { AbortSignalLike } from "@azure/abort-controller";
 import { default as minimist, ParsedArgs as MinimistParsedArgs } from "minimist";
-import { PerfStressOptionDictionary, parsePerfStressOption } from "./options";
+import {
+  PerfStressOptionDictionary,
+  parsePerfStressOption,
+  DefaultPerfStressOptions,
+  defaultPerfStressOptions
+} from "./options";
 
 /**
  * Defines the behavior of the PerfStressTest constructor, to use the class as a value.
  */
-export interface PerfStressTestConstructor<TOptionsNames extends string> {
-  new (): PerfStressTest<TOptionsNames>;
+export interface PerfStressTestConstructor<TOptions extends {} = {}> {
+  new (): PerfStressTest<TOptions>;
 }
 
 /**
@@ -21,11 +26,18 @@ export interface PerfStressTestConstructor<TOptionsNames extends string> {
  * and at a local level, which happens once for each initialization of the test class
  * (initializations are as many as the "parallel" command line parameter specifies).
  */
-export abstract class PerfStressTest<TOptionsNames extends string> {
-  public abstract options: PerfStressOptionDictionary<TOptionsNames>;
+export abstract class PerfStressTest<TOptions = {}> {
+  public abstract options: PerfStressOptionDictionary<TOptions>;
 
-  public parseOptions() {
-    this.options = parsePerfStressOption(this.options) as PerfStressOptionDictionary<TOptionsNames>;
+  public get parsedOptions(): PerfStressOptionDictionary<TOptions & DefaultPerfStressOptions> {
+    // This cast is needed because TS thinks
+    //   PerfStressOptionDictionary<TOptions & DefaultPerfStressOptions>
+    //   is different from
+    //   PerfStressOptionDictionary<TOptions> & PerfStressOptionDictionary<DefaultPerfStressOptions>
+    return parsePerfStressOption({
+      ...this.options,
+      ...defaultPerfStressOptions
+    }) as PerfStressOptionDictionary<TOptions & DefaultPerfStressOptions>;
   }
 
   // Before and after running a bunch of the same test.
@@ -45,8 +57,8 @@ export abstract class PerfStressTest<TOptionsNames extends string> {
  * @param tests An array of classes that extend PerfStressTest
  */
 export function selectPerfStressTest(
-  tests: PerfStressTestConstructor<string>[]
-): PerfStressTestConstructor<string> {
+  tests: PerfStressTestConstructor[]
+): PerfStressTestConstructor {
   const testsNames: string[] = tests.map((test) => test.name);
   const minimistResult: MinimistParsedArgs = minimist(process.argv);
   const testName = minimistResult._[minimistResult._.length - 1];

diff --git a/sdk/test-utils/perfstress/test/delay.spec.ts b/sdk/test-utils/perfstress/test/delay.spec.ts
@@ -32,7 +32,7 @@ import { delay } from "@azure/core-http";
  * Completed 8 operations in a weighted-average of 4.00s (2.00 ops/s 0.501 s/op)
  * ```
  */
-export class Delay500ms extends PerfStressTest<string> {
+export class Delay500ms extends PerfStressTest {
   /**
    * This test doesn't receive command line parameters.
    */

diff --git a/sdk/test-utils/perfstress/test/exception.spec.ts b/sdk/test-utils/perfstress/test/exception.spec.ts
@@ -8,7 +8,7 @@ import { PerfStressTest } from "../src";
  * If the option "sync" is passed, errors will be thrown on every test call, where the test being called is simple function.
  * Otherwise, errors thrown on every test call, where the test being called is a function returning a promise (an asynchronous function).
  */
-export class Exception extends PerfStressTest<string> {
+export class Exception extends PerfStressTest {
   public options = {};
 
   run(): void {

diff --git a/sdk/test-utils/perfstress/test/nodeFetch.spec.ts b/sdk/test-utils/perfstress/test/nodeFetch.spec.ts
@@ -6,20 +6,25 @@ import { PerfStressTest, PerfStressOptionDictionary } from "../src";
 import fetch from "node-fetch";
 import * as http from "http";
 
-type OptionNames = "url";
+interface NodeFetchOptions {
+  url: string;
+}
 
-export class NodeFetchTest extends PerfStressTest<string> {
+export class NodeFetchTest extends PerfStressTest<NodeFetchOptions> {
   private static fetchOptions = {
     agent: new http.Agent({ keepAlive: true })
   };
 
   private url: string = "";
 
-  public options: PerfStressOptionDictionary<OptionNames> = {
+  public options: PerfStressOptionDictionary<NodeFetchOptions> = {
     url: {
       required: true,
       description: "Required option",
-      shortName: "u"
+      shortName: "u",
+      longName: "url",
+      defaultValue: "http://bing.com",
+      value: "http://bing.com"
     }
   };
 

diff --git a/sdk/test-utils/perfstress/test/noop.spec.ts b/sdk/test-utils/perfstress/test/noop.spec.ts
@@ -6,7 +6,7 @@ import { PerfStressTest } from "../src";
 /**
  * Should test the raw performance impact of the PerfStress framework for both synchronous and asynchronous tests.
  */
-export class NoOp extends PerfStressTest<string> {
+export class NoOp extends PerfStressTest {
   public options = {};
 
   run(): void {}