Add "Run Test Multiple Times" (#1009)

Adds two new items to the context menu when you right click a test in
the test explorer:

- Run Multiple Times
- Run Until Failure

Selecting either of these promts with a text input where the user can
input how many times they want to run the test(s).

If the user selected Run Until Failure the tests will be run a maximum
number of times, stopping at the first iteration that produces a
failure.

A current limitation in VS Code is if you have multiple tests selected
in the Test Explorer (by shift or ctrl clicking them) only the first is
passed to the command.

Issue: #832

Author: plemarquand

package.json

@@ -201,6 +201,16 @@
         "command": "swift.clearDiagnosticsCollection",
         "title": "Clear Diagnostics Collection",
         "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsMultipleTimes",
+        "title": "Run Multiple Times...",
+        "category": "Swift"
+      },
+      {
+        "command": "swift.runTestsUntilFailure",
+        "title": "Run Until Failure...",
+        "category": "Swift"
       }
     ],
     "configuration": [
@@ -598,6 +608,18 @@
       }
     ],
     "menus": {
+      "testing/item/context": [
+        {
+          "command": "swift.runTestsMultipleTimes",
+          "group": "testExtras",
+          "when": "testId"
+        },
+        {
+          "command": "swift.runTestsUntilFailure",
+          "group": "testExtras",
+          "when": "testId"
+        }
+      ],
       "commandPalette": [
         {
           "command": "swift.createNewProject",

src/TestExplorer/TestExplorer.ts

@@ -41,7 +41,7 @@ export class TestExplorer {
     private onTestItemsDidChangeEmitter = new vscode.EventEmitter<vscode.TestController>();
     public onTestItemsDidChange: vscode.Event<vscode.TestController>;
 
-    private onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
+    public onDidCreateTestRunEmitter = new vscode.EventEmitter<TestRunProxy>();
     public onCreateTestRun: vscode.Event<TestRunProxy>;
 
     constructor(public folderContext: FolderContext) {

src/TestExplorer/TestRunner.ts

@@ -52,20 +52,28 @@ export class TestRunProxy {
     private runStarted: boolean = false;
     private queuedOutput: string[] = [];
     private _testItems: vscode.TestItem[];
+    private iteration: number | undefined;
     public coverage: TestCoverage;
 
+    public testRunCompleteEmitter = new vscode.EventEmitter<void>();
+    public onTestRunComplete: vscode.Event<void>;
+
     // Allows for introspection on the state of TestItems after a test run.
-    public runState = {
-        failed: [] as {
-            test: vscode.TestItem;
-            message: vscode.TestMessage | readonly vscode.TestMessage[];
-        }[],
-        passed: [] as vscode.TestItem[],
-        skipped: [] as vscode.TestItem[],
-        errored: [] as vscode.TestItem[],
-        unknown: 0,
-        output: [] as string[],
-    };
+    public runState = TestRunProxy.initialTestRunState();
+
+    private static initialTestRunState() {
+        return {
+            failed: [] as {
+                test: vscode.TestItem;
+                message: vscode.TestMessage | readonly vscode.TestMessage[];
+            }[],
+            passed: [] as vscode.TestItem[],
+            skipped: [] as vscode.TestItem[],
+            errored: [] as vscode.TestItem[],
+            unknown: 0,
+            output: [] as string[],
+        };
+    }
 
     public get testItems(): vscode.TestItem[] {
         return this._testItems;
@@ -79,6 +87,7 @@ export class TestRunProxy {
     ) {
         this._testItems = args.testItems;
         this.coverage = new TestCoverage(folderContext);
+        this.onTestRunComplete = this.testRunCompleteEmitter.event;
     }
 
     public testRunStarted = () => {
@@ -194,20 +203,37 @@ export class TestRunProxy {
 
     public async end() {
         this.testRun?.end();
+        this.testRunCompleteEmitter.fire();
+    }
+
+    public setIteration(iteration: number) {
+        this.runState = TestRunProxy.initialTestRunState();
+        this.iteration = iteration;
     }
 
     public appendOutput(output: string) {
+        const tranformedOutput = this.prependIterationToOutput(output);
         if (this.testRun) {
-            this.testRun.appendOutput(output);
-            this.runState.output.push(output);
+            this.testRun.appendOutput(tranformedOutput);
+            this.runState.output.push(tranformedOutput);
         } else {
-            this.queuedOutput.push(output);
+            this.queuedOutput.push(tranformedOutput);
         }
     }
 
     public appendOutputToTest(output: string, test: vscode.TestItem, location?: vscode.Location) {
-        this.testRun?.appendOutput(output, location, test);
-        this.runState.output.push(output);
+        const tranformedOutput = this.prependIterationToOutput(output);
+        this.testRun?.appendOutput(tranformedOutput, location, test);
+        this.runState.output.push(tranformedOutput);
+    }
+
+    private prependIterationToOutput(output: string): string {
+        if (this.iteration === undefined) {
+            return output;
+        }
+        const itr = this.iteration + 1;
+        const lines = output.match(/[^\r\n]*[\r\n]*/g);
+        return lines?.map(line => (line ? `\x1b[34mRun ${itr}\x1b[0m ${line}` : "")).join("") ?? "";
     }
 
     public async computeCoverage() {
@@ -222,7 +248,7 @@ export class TestRunProxy {
 
 /** Class used to run tests */
 export class TestRunner {
-    private testRun: TestRunProxy;
+    public testRun: TestRunProxy;
     private testArgs: TestRunArguments;
     private xcTestOutputParser: IXCTestOutputParser;
     private swiftTestOutputParser: SwiftTestingOutputParser;
@@ -256,6 +282,20 @@ export class TestRunner {
         );
     }
 
+    /**
+     * When performing a "Run test multiple times" run set the iteration
+     * so it can be shown in the logs.
+     * @param iteration The iteration counter
+     */
+    public setIteration(iteration: number) {
+        // The SwiftTestingOutputParser holds state and needs to be reset between iterations.
+        this.swiftTestOutputParser = new SwiftTestingOutputParser(
+            this.testRun.testRunStarted,
+            this.testRun.addParameterizedTestCase
+        );
+        this.testRun.setIteration(iteration);
+    }
+
     /**
      * If the request has no test items to include in the run,
      * default to usig all the items in the `TestController`.
@@ -587,6 +627,7 @@ export class TestRunner {
             task.execution.onDidWrite(str => {
                 const replaced = str
                     .replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build.
+                    .replace(/\[1\/1\] Write swift-version-.*/gm, "")
                     .replace(
                         /LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm,
                         ""
@@ -1023,6 +1064,7 @@ export class TestRunnerTestRunState implements ITestRunState {
             return;
         }
         const testItem = this.testRun.testItems[index];
+        this.issues.delete(index);
         this.testRun.started(testItem);
         this.currentTestItem = testItem;
         this.startTimes.set(index, startTime);

src/commands.ts

@@ -33,6 +33,8 @@ import { SwiftProjectTemplate } from "./toolchain/toolchain";
 import { showToolchainSelectionQuickPick, showToolchainError } from "./ui/ToolchainSelection";
 import { captureDiagnostics } from "./commands/captureDiagnostics";
 import { reindexProjectRequest } from "./sourcekit-lsp/lspExtensions";
+import { TestRunner, TestRunnerTestRunState } from "./TestExplorer/TestRunner";
+import { TestKind } from "./TestExplorer/TestKind";
 
 /**
  * References:
@@ -734,6 +736,54 @@ function openInExternalEditor(packageNode: PackageNode) {
     }
 }
 
+async function runTestMultipleTimes(
+    ctx: WorkspaceContext,
+    test: vscode.TestItem,
+    untilFailure: boolean
+) {
+    const str = await vscode.window.showInputBox({
+        prompt: "Label: ",
+        placeHolder: `${untilFailure ? "Maximum " : ""}# of times to run`,
+        validateInput: value => (/^[1-9]\d*$/.test(value) ? undefined : "Enter an integer value"),
+    });
+
+    if (!str || !ctx.currentFolder?.testExplorer) {
+        return;
+    }
+
+    const numExecutions = parseInt(str);
+    const testExplorer = ctx.currentFolder.testExplorer;
+    const runner = new TestRunner(
+        TestKind.standard,
+        new vscode.TestRunRequest([test]),
+        ctx.currentFolder,
+        testExplorer.controller
+    );
+
+    testExplorer.onDidCreateTestRunEmitter.fire(runner.testRun);
+
+    const testRunState = new TestRunnerTestRunState(runner.testRun);
+    const token = new vscode.CancellationTokenSource();
+
+    vscode.commands.executeCommand("workbench.panel.testResults.view.focus");
+
+    for (let i = 0; i < numExecutions; i++) {
+        runner.setIteration(i);
+        runner.testRun.appendOutput(`\x1b[36mBeginning Test Iteration #${i + 1}\x1b[0m\n`);
+
+        await runner.runSession(token.token, testRunState);
+
+        if (
+            untilFailure &&
+            (runner.testRun.runState.failed.length > 0 ||
+                runner.testRun.runState.errored.length > 0)
+        ) {
+            break;
+        }
+    }
+    runner.testRun.end();
+}
+
 /**
  * Switches the target SDK to the platform selected in a QuickPick UI.
  */
@@ -841,6 +891,12 @@ export function register(ctx: WorkspaceContext): vscode.Disposable[] {
         vscode.commands.registerCommand("swift.run", () => runBuild(ctx)),
         vscode.commands.registerCommand("swift.debug", () => debugBuild(ctx)),
         vscode.commands.registerCommand("swift.cleanBuild", () => cleanBuild(ctx)),
+        vscode.commands.registerCommand("swift.runTestsMultipleTimes", item =>
+            runTestMultipleTimes(ctx, item, false)
+        ),
+        vscode.commands.registerCommand("swift.runTestsUntilFailure", item =>
+            runTestMultipleTimes(ctx, item, true)
+        ),
         // Note: This is only available on macOS (gated in `package.json`) because its the only OS that has the iOS SDK available.
         vscode.commands.registerCommand("swift.switchPlatform", () => switchPlatform()),
         vscode.commands.registerCommand("swift.resetPackage", () => resetPackage(ctx)),

test/suite/testexplorer/TestExplorerIntegration.test.ts

@@ -15,6 +15,7 @@
 import * as vscode from "vscode";
 import * as assert from "assert";
 import { beforeEach } from "mocha";
+import { when, anything } from "ts-mockito";
 import { testAssetUri } from "../../fixtures";
 import { globalWorkspaceContextPromise } from "../extension.test";
 import { TestExplorer } from "../../../src/TestExplorer/TestExplorer";
@@ -29,6 +30,7 @@ import { WorkspaceContext } from "../../../src/WorkspaceContext";
 import { TestRunProxy } from "../../../src/TestExplorer/TestRunner";
 import { Version } from "../../../src/utilities/version";
 import { TestKind } from "../../../src/TestExplorer/TestKind";
+import { mockNamespace } from "../../unit-tests/MockUtils";
 
 suite("Test Explorer Suite", function () {
     const MAX_TEST_RUN_TIME_MINUTES = 5;
@@ -49,18 +51,10 @@ suite("Test Explorer Suite", function () {
         )[0];
     }
 
-    async function runTest(
+    async function gatherTests(
         controller: vscode.TestController,
-        runProfile: TestKind,
         ...tests: string[]
-    ): Promise<TestRunProxy> {
-        const targetProfile = testExplorer.testRunProfiles.find(
-            profile => profile.label === runProfile
-        );
-        if (!targetProfile) {
-            throw new Error(`Unable to find run profile named ${runProfile}`);
-        }
-
+    ): Promise<vscode.TestItem[]> {
         const testItems = tests.map(test => {
             const testItem = getTestItem(controller, test);
             if (!testItem) {
@@ -79,6 +73,21 @@ suite("Test Explorer Suite", function () {
             return testItem;
         });
 
+        return testItems;
+    }
+
+    async function runTest(
+        controller: vscode.TestController,
+        runProfile: TestKind,
+        ...tests: string[]
+    ): Promise<TestRunProxy> {
+        const targetProfile = testExplorer.testRunProfiles.find(
+            profile => profile.label === runProfile
+        );
+        if (!targetProfile) {
+            throw new Error(`Unable to find run profile named ${runProfile}`);
+        }
+        const testItems = await gatherTests(controller, ...tests);
         const request = new vscode.TestRunRequest(testItems);
 
         return (
@@ -233,6 +242,37 @@ suite("Test Explorer Suite", function () {
                 ],
             });
         });
+
+        suite("Runs multiple", function () {
+            const numIterations = 5;
+            const windowMock = mockNamespace(vscode, "window");
+
+            test("@slow runs an swift-testing test multiple times", async function () {
+                const testItems = await gatherTests(
+                    testExplorer.controller,
+                    "PackageTests.MixedXCTestSuite/testPassing"
+                );
+
+                await testExplorer.folderContext.workspaceContext.focusFolder(
+                    testExplorer.folderContext
+                );
+
+                // Stub the showInputBox method to return the input text
+                when(windowMock.showInputBox(anything())).thenReturn(
+                    Promise.resolve(`${numIterations}`)
+                );
+
+                vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);
+
+                const testRun = await eventPromise(testExplorer.onCreateTestRun);
+
+                await eventPromise(testRun.onTestRunComplete);
+
+                assertTestResults(testRun, {
+                    passed: ["PackageTests.MixedXCTestSuite/testPassing"],
+                });
+            });
+        });
     });
 
     suite("XCTest", () => {
@@ -262,6 +302,37 @@ suite("Test Explorer Suite", function () {
                 passed: ["PackageTests.DebugReleaseTestSuite/testRelease"],
             });
         });
+
+        suite("Runs multiple", function () {
+            const numIterations = 5;
+            const windowMock = mockNamespace(vscode, "window");
+
+            test("@slow runs an XCTest multiple times", async function () {
+                const testItems = await gatherTests(
+                    testExplorer.controller,
+                    "PackageTests.topLevelTestPassing()"
+                );
+
+                await testExplorer.folderContext.workspaceContext.focusFolder(
+                    testExplorer.folderContext
+                );
+
+                // Stub the showInputBox method to return the input text
+                when(windowMock.showInputBox(anything())).thenReturn(
+                    Promise.resolve(`${numIterations}`)
+                );
+
+                vscode.commands.executeCommand("swift.runTestsMultipleTimes", testItems[0]);
+
+                const testRun = await eventPromise(testExplorer.onCreateTestRun);
+
+                await eventPromise(testRun.onTestRunComplete);
+
+                assertTestResults(testRun, {
+                    passed: ["PackageTests.topLevelTestPassing()"],
+                });
+            });
+        });
     });
 
     // Do coverage last as it does a full rebuild, causing the stage after it to have to rebuild as well.