Simplistic watch mode for runtests (#51461)

* Simplistic watch mode for runtests

* Use esbuild WatchMode object for testRunner updates

* switch AbortController to CancelToken

Author: rbuckton

Herebyfile.mjs

@@ -6,12 +6,15 @@ import { task } from "hereby";
 import _glob from "glob";
 import util from "util";
 import chalk from "chalk";
-import { exec, readJson, getDiffTool, getDirSize, memoize, needsUpdate } from "./scripts/build/utils.mjs";
-import { runConsoleTests, refBaseline, localBaseline, refRwcBaseline, localRwcBaseline } from "./scripts/build/tests.mjs";
+import { exec, readJson, getDiffTool, getDirSize, memoize, needsUpdate, Debouncer, Deferred } from "./scripts/build/utils.mjs";
+import { runConsoleTests, refBaseline, localBaseline, refRwcBaseline, localRwcBaseline, cleanTestDirs } from "./scripts/build/tests.mjs";
 import { buildProject as realBuildProject, cleanProject, watchProject } from "./scripts/build/projects.mjs";
 import { localizationDirectories } from "./scripts/build/localization.mjs";
 import cmdLineOptions from "./scripts/build/options.mjs";
 import esbuild from "esbuild";
+import chokidar from "chokidar";
+import { EventEmitter } from "events";
+import { CancelToken } from "@esfx/canceltoken";
 
 const glob = util.promisify(_glob);
 
@@ -191,6 +194,7 @@ async function runDtsBundler(entrypoint, output) {
  * @property {string[]} [external]
  * @property {boolean} [exportIsTsObject]
  * @property {boolean} [treeShaking]
+ * @property {esbuild.WatchMode} [watchMode]
  */
 function createBundler(entrypoint, outfile, taskOptions = {}) {
     const getOptions = memoize(async () => {
@@ -269,7 +273,7 @@ function createBundler(entrypoint, outfile, taskOptions = {}) {
 
     return {
         build: async () => esbuild.build(await getOptions()),
-        watch: async () => esbuild.build({ ...await getOptions(), watch: true, logLevel: "info" }),
+        watch: async () => esbuild.build({ ...await getOptions(), watch: taskOptions.watchMode ?? true, logLevel: "info" }),
     };
 }
 
@@ -461,7 +465,7 @@ export const dts = task({
 
 
 const testRunner = "./built/local/run.js";
-
+const watchTestsEmitter = new EventEmitter();
 const { main: tests, watch: watchTests } = entrypointBuildTask({
     name: "tests",
     description: "Builds the test infrastructure",
@@ -482,6 +486,11 @@ const { main: tests, watch: watchTests } = entrypointBuildTask({
             "mocha",
             "ms",
         ],
+        watchMode: {
+            onRebuild() {
+                watchTestsEmitter.emit("rebuild");
+            }
+        }
     },
 });
 export { tests, watchTests };
@@ -625,6 +634,117 @@ export const runTests = task({
 //     "   --shardId": "1-based ID of this shard (default: 1)",
 // };
 
+export const runTestsAndWatch = task({
+    name: "runtests-watch",
+    dependencies: [watchTests],
+    run: async () => {
+        if (!cmdLineOptions.tests && !cmdLineOptions.failed) {
+            console.log(chalk.redBright(`You must specifiy either --tests/-t or --failed to use 'runtests-watch'.`));
+            return;
+        }
+
+        let watching = true;
+        let running = true;
+        let lastTestChangeTimeMs = Date.now();
+        let testsChangedDeferred = /** @type {Deferred<void>} */(new Deferred());
+        let testsChangedCancelSource = CancelToken.source();
+
+        const testsChangedDebouncer = new Debouncer(1_000, endRunTests);
+        const testCaseWatcher = chokidar.watch([
+            "tests/cases/**/*.*",
+            "tests/lib/**/*.*",
+            "tests/projects/**/*.*",
+        ], {
+            ignorePermissionErrors: true,
+            alwaysStat: true
+        });
+
+        process.on("SIGINT", endWatchMode);
+        process.on("SIGKILL", endWatchMode);
+        process.on("beforeExit", endWatchMode);
+        watchTestsEmitter.on("rebuild", onRebuild);
+        testCaseWatcher.on("all", onChange);
+
+        while (watching) {
+            const promise = testsChangedDeferred.promise;
+            const token = testsChangedCancelSource.token;
+            if (!token.signaled) {
+                running = true;
+                try {
+                    await runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, { token, watching: true });
+                }
+                catch {
+                    // ignore
+                }
+                running = false;
+            }
+            if (watching) {
+                console.log(chalk.yellowBright(`[watch] test run complete, waiting for changes...`));
+                await promise;
+            }
+        }
+
+        function onRebuild() {
+            beginRunTests(testRunner);
+        }
+
+        /**
+         * @param {'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'} eventName
+         * @param {string} path
+         * @param {fs.Stats | undefined} stats
+         */
+        function onChange(eventName, path, stats) {
+            switch (eventName) {
+                case "change":
+                case "unlink":
+                case "unlinkDir":
+                    break;
+                case "add":
+                case "addDir":
+                    // skip files that are detected as 'add' but haven't actually changed since the last time tests were
+                    // run.
+                    if (stats && stats.mtimeMs <= lastTestChangeTimeMs) {
+                        return;
+                    }
+                    break;
+            }
+            beginRunTests(path);
+        }
+
+        /**
+         * @param {string} path
+         */
+        function beginRunTests(path) {
+            if (testsChangedDebouncer.empty) {
+                console.log(chalk.yellowBright(`[watch] tests changed due to '${path}', restarting...`));
+                if (running) {
+                    console.log(chalk.yellowBright("[watch] aborting in-progress test run..."));
+                }
+                testsChangedCancelSource.cancel();
+                testsChangedCancelSource = CancelToken.source();
+            }
+
+            testsChangedDebouncer.enqueue();
+        }
+
+        function endRunTests() {
+            lastTestChangeTimeMs = Date.now();
+            testsChangedDeferred.resolve();
+            testsChangedDeferred = /** @type {Deferred<void>} */(new Deferred());
+        }
+
+        function endWatchMode() {
+            if (watching) {
+                watching = false;
+                console.log(chalk.yellowBright("[watch] exiting watch mode..."));
+                testsChangedCancelSource.cancel();
+                testCaseWatcher.close();
+                watchTestsEmitter.off("rebuild", onRebuild);
+            }
+        }
+    },
+});
+
 export const runTestsParallel = task({
     name: "runtests-parallel",
     description: "Runs all the tests in parallel using the built run.js file.",

package-lock.json

@@ -39,6 +39,7 @@
         "!**/.gitattributes"
     ],
     "devDependencies": {
+        "@esfx/canceltoken": "^1.0.0",
         "@octokit/rest": "latest",
         "@types/chai": "latest",
         "@types/fs-extra": "^9.0.13",
@@ -56,6 +57,7 @@
         "azure-devops-node-api": "^11.2.0",
         "chai": "latest",
         "chalk": "^4.1.2",
+        "chokidar": "^3.5.3",
         "del": "^6.1.1",
         "diff": "^5.1.0",
         "esbuild": "^0.15.13",

package.json

@@ -2,9 +2,11 @@ import del from "del";
 import fs from "fs";
 import os from "os";
 import path from "path";
+import chalk from "chalk";
 import cmdLineOptions from "./options.mjs";
 import { exec } from "./utils.mjs";
 import { findUpFile, findUpRoot } from "./findUpDir.mjs";
+import { CancelError } from "@esfx/canceltoken";
 
 const mochaJs = path.resolve(findUpRoot(), "node_modules", "mocha", "bin", "_mocha");
 export const localBaseline = "tests/baselines/local/";
@@ -17,8 +19,11 @@ export const localTest262Baseline = "internal/baselines/test262/local";
  * @param {string} runJs
  * @param {string} defaultReporter
  * @param {boolean} runInParallel
+ * @param {object} options
+ * @param {import("@esfx/canceltoken").CancelToken} [options.token]
+ * @param {boolean} [options.watching]
  */
-export async function runConsoleTests(runJs, defaultReporter, runInParallel) {
+export async function runConsoleTests(runJs, defaultReporter, runInParallel, options = {}) {
     let testTimeout = cmdLineOptions.timeout;
     const tests = cmdLineOptions.tests;
     const inspect = cmdLineOptions.break || cmdLineOptions.inspect;
@@ -31,7 +36,14 @@ export async function runConsoleTests(runJs, defaultReporter, runInParallel) {
     const shards = +cmdLineOptions.shards || undefined;
     const shardId = +cmdLineOptions.shardId || undefined;
     if (!cmdLineOptions.dirty) {
+        if (options.watching) {
+            console.log(chalk.yellowBright(`[watch] cleaning test directories...`));
+        }
         await cleanTestDirs();
+
+        if (options.token?.signaled) {
+            return;
+        }
     }
 
     if (fs.existsSync(testConfigFile)) {
@@ -56,6 +68,10 @@ export async function runConsoleTests(runJs, defaultReporter, runInParallel) {
         testTimeout = 400000;
     }
 
+    if (options.watching) {
+        console.log(chalk.yellowBright(`[watch] running tests...`));
+    }
+
     if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || shards || shardId) {
         writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId);
     }
@@ -114,7 +130,8 @@ export async function runConsoleTests(runJs, defaultReporter, runInParallel) {
 
     try {
         setNodeEnvToDevelopment();
-        const { exitCode } = await exec(process.execPath, args);
+
+        const { exitCode } = await exec(process.execPath, args, { token: options.token });
         if (exitCode !== 0) {
             errorStatus = exitCode;
             error = new Error(`Process exited with status code ${errorStatus}.`);
@@ -132,8 +149,17 @@ export async function runConsoleTests(runJs, defaultReporter, runInParallel) {
     await deleteTemporaryProjectOutput();
 
     if (error !== undefined) {
-        process.exitCode = typeof errorStatus === "number" ? errorStatus : 2;
-        throw error;
+        if (error instanceof CancelError) {
+            throw error;
+        }
+
+        if (options.watching) {
+            console.error(`${chalk.redBright(error.name)}: ${error.message}`);
+        }
+        else {
+            process.exitCode = typeof errorStatus === "number" ? errorStatus : 2;
+            throw error;
+        }
     }
 }