test: run watch cases concurrently (#9541)

test: run test cases parallel
web-infra-dev · Mar 4, 2025 · 672a375 · 672a375 · github-actions · Mar 4, 2025
1 parent 44f96be
commit 672a375
Show file tree

Hide file tree

Showing 11 changed files with 235 additions and 13 deletions.
diff --git a/packages/rspack-test-tools/etc/test-tools.api.md b/packages/rspack-test-tools/etc/test-tools.api.md
@@ -34,17 +34,31 @@ export class BasicCaseCreator<T extends ECompilerType> {
     // (undocumented)
     create(name: string, src: string, dist: string, temp?: string): ITester | undefined;
     // (undocumented)
+    protected createConcurrentEnv(): ITestEnv & IConcurrentTestEnv;
+    // (undocumented)
     protected createEnv(testConfig: TTestConfig<T>): ITestEnv;
     // (undocumented)
     protected createTester(name: string, src: string, dist: string, temp: string | void, testConfig: TTestConfig<T>): ITester;
     // (undocumented)
+    protected currentConcurrent: number;
+    // (undocumented)
     protected describe(name: string, tester: ITester, testConfig: TTestConfig<T>): void;
     // (undocumented)
+    protected describeConcurrent(name: string, tester: ITester, testConfig: TTestConfig<T>): void;
+    // (undocumented)
+    protected getMaxConcurrent(): number;
+    // (undocumented)
     protected _options: IBasicCaseCreatorOptions<T>;
     // (undocumented)
     protected readTestConfig(src: string): TTestConfig<T>;
     // (undocumented)
+    protected registerConcurrentTask(name: string, starter: () => void): () => void;
+    // (undocumented)
     protected skip(name: string, reason: string | boolean): void;
+    // (undocumented)
+    protected tasks: [string, () => void][];
+    // (undocumented)
+    protected tryRunTask(): void;
 }
 
 // @public (undocumented)
@@ -578,6 +592,8 @@ export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
     // (undocumented)
     clean?: boolean;
     // (undocumented)
+    concurrent?: boolean | number;
+    // (undocumented)
     contextValue?: Record<string, unknown>;
     // (undocumented)
     describe?: boolean;
@@ -688,6 +704,14 @@ export interface ICompareOptions {
     snapshot?: string;
 }
 
+// @public (undocumented)
+interface IConcurrentTestEnv {
+    // (undocumented)
+    clear: () => void;
+    // (undocumented)
+    run: () => Promise<void>;
+}
+
 // @public (undocumented)
 export interface IConfigProcessorOptions<T extends ECompilerType> extends IMultiTaskProcessorOptions<T> {
 }
@@ -1573,12 +1597,13 @@ export type TTestConfig<T extends ECompilerType> = {
     beforeExecute?: () => void;
     afterExecute?: () => void;
     moduleScope?: (ms: IBasicModuleScope, stats?: TCompilerStatsCompilation<T>) => IBasicModuleScope;
-    checkStats?: (stepName: string, jsonStats: TCompilerStatsCompilation<T>, stringStats: String) => boolean;
+    checkStats?: (stepName: string, jsonStats: TCompilerStatsCompilation<T> | undefined, stringStats: String) => boolean;
     findBundle?: (index: number, options: TCompilerOptions<T>, stepName?: string) => string | string[];
     bundlePath?: string[];
     nonEsmThis?: (p: string | string[]) => Object;
     modules?: Record<string, Object>;
     timeout?: number;
+    concurrent?: boolean;
 };
 
 // @public (undocumented)

diff --git a/packages/rspack-test-tools/src/case/hot-step.ts b/packages/rspack-test-tools/src/case/hot-step.ts
@@ -26,7 +26,8 @@ function getCreator(target: TTarget) {
 						configFiles: ["rspack.config.js", "webpack.config.js"]
 					})
 				],
-				runner: HotStepRunnerFactory
+				runner: HotStepRunnerFactory,
+				concurrent: true
 			})
 		);
 	}

diff --git a/packages/rspack-test-tools/src/case/hot.ts b/packages/rspack-test-tools/src/case/hot.ts
@@ -26,7 +26,8 @@ function getCreator(target: TTarget) {
 						configFiles: ["rspack.config.js", "webpack.config.js"]
 					})
 				],
-				runner: HotRunnerFactory
+				runner: HotRunnerFactory,
+				concurrent: true
 			})
 		);
 	}

diff --git a/packages/rspack-test-tools/src/case/new-incremental.ts b/packages/rspack-test-tools/src/case/new-incremental.ts
@@ -37,6 +37,8 @@ function getHotCreator(target: TTarget, documentType: EDocumentType) {
 					})
 				],
 				runner: HotRunnerFactory
+				// TODO: enable concurrent then rspack will be hanged
+				// concurrent: true
 			})
 		);
 	}
@@ -101,7 +103,8 @@ const watchCreator = new BasicCaseCreator({
 						watchState
 					)
 		);
-	}
+	},
+	concurrent: true
 });
 
 export function createWatchNewIncrementalCase(

diff --git a/packages/rspack-test-tools/src/case/watch.ts b/packages/rspack-test-tools/src/case/watch.ts
@@ -50,7 +50,8 @@ const creator = new BasicCaseCreator({
 						watchState
 					)
 		);
-	}
+	},
+	concurrent: true
 });
 
 export function createWatchCase(

diff --git a/packages/rspack-test-tools/src/compiler.ts b/packages/rspack-test-tools/src/compiler.ts
@@ -84,6 +84,10 @@ export class TestCompilerManager<T extends ECompilerType>
 			throw new Error("Compiler should be created before watch");
 		this.compilerInstance!.watch(
 			{
+				// IMPORTANT:
+				// This is a workaround for the issue that watchpack cannot detect the file change in time
+				// so we set the poll to 300ms to make it more sensitive to the file change
+				poll: 300,
 				aggregateTimeout: timeout
 			},
 			(error, newStats) => {

diff --git a/packages/rspack-test-tools/src/helper/util/currentWatchStep.js b/packages/rspack-test-tools/src/helper/util/currentWatchStep.js
@@ -1 +1 @@
-exports.step = undefined;
+exports.step = {};
diff --git a/packages/rspack-test-tools/src/processor/watch.ts b/packages/rspack-test-tools/src/processor/watch.ts
@@ -60,7 +60,8 @@ export class WatchProcessor<
 	async build(context: ITestContext) {
 		const compiler = this.getCompiler(context);
 		const currentWatchStepModule = require(currentWatchStepModulePath);
-		currentWatchStepModule.step = this._watchOptions.stepName;
+		currentWatchStepModule.step[this._options.name] =
+			this._watchOptions.stepName;
 		fs.mkdirSync(this._watchOptions.tempDir, { recursive: true });
 		copyDiff(
 			path.join(context.getSource(), this._watchOptions.stepName),
@@ -345,7 +346,8 @@ export class WatchStepProcessor<
 	async build(context: ITestContext) {
 		const compiler = this.getCompiler(context);
 		const currentWatchStepModule = require(currentWatchStepModulePath);
-		currentWatchStepModule.step = this._watchOptions.stepName;
+		currentWatchStepModule.step[this._options.name] =
+			this._watchOptions.stepName;
 		const task = new Promise((resolve, reject) => {
 			compiler.getEmitter().once(ECompilerEvent.Build, (e, stats) => {
 				if (e) return reject(e);

diff --git a/packages/rspack-test-tools/src/test/creator.ts b/packages/rspack-test-tools/src/test/creator.ts
@@ -14,6 +14,11 @@ import type {
 } from "../type";
 import { Tester } from "./tester";
 
+interface IConcurrentTestEnv {
+	clear: () => void;
+	run: () => Promise<void>;
+}
+
 export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
 	clean?: boolean;
 	describe?: boolean;
@@ -34,9 +39,15 @@ export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
 		context: ITestContext
 	) => TRunnerFactory<ECompilerType>;
 	[key: string]: unknown;
+	concurrent?: boolean | number;
 }
 
+const DEFAULT_MAX_CONCURRENT = 5;
+
 export class BasicCaseCreator<T extends ECompilerType> {
+	protected currentConcurrent = 0;
+	protected tasks: [string, () => void][] = [];
+
 	constructor(protected _options: IBasicCaseCreatorOptions<T>) {}
 
 	create(name: string, src: string, dist: string, temp?: string) {
@@ -55,16 +66,97 @@ export class BasicCaseCreator<T extends ECompilerType> {
 		}
 
 		const tester = this.createTester(name, src, dist, temp, testConfig);
+		const concurrent =
+			testConfig.concurrent ?? this._options.concurrent ?? false;
 
 		if (this._options.describe) {
-			describe(name, () => this.describe(name, tester, testConfig));
+			if (concurrent) {
+				describe(name, () => this.describeConcurrent(name, tester, testConfig));
+			} else {
+				describe(name, () => this.describe(name, tester, testConfig));
+			}
 		} else {
-			this.describe(name, tester, testConfig);
+			if (concurrent) {
+				this.describeConcurrent(name, tester, testConfig);
+			} else {
+				this.describe(name, tester, testConfig);
+			}
 		}
 
 		return tester;
 	}
 
+	protected describeConcurrent(
+		name: string,
+		tester: ITester,
+		testConfig: TTestConfig<T>
+	) {
+		beforeAll(async () => {
+			await tester.prepare();
+		});
+
+		let starter = null;
+		let chain = new Promise<void>((resolve, reject) => {
+			starter = resolve;
+		});
+		const ender = this.registerConcurrentTask(name, starter!);
+		const env = this.createConcurrentEnv();
+		let bailout = false;
+		for (let index = 0; index < tester.total; index++) {
+			let stepSignalResolve = null;
+			let stepSignalReject = null;
+			const stepSignal = new Promise((resolve, reject) => {
+				stepSignalResolve = resolve;
+				stepSignalReject = reject;
+			});
+			const description =
+				typeof this._options.description === "function"
+					? this._options.description(name, index)
+					: `step ${index ? `[${index}]` : ""} should pass`;
+			it(
+				description,
+				async () => {
+					await stepSignal;
+				},
+				this._options.timeout || 180000
+			);
+
+			chain = chain.then(async () => {
+				try {
+					if (bailout) {
+						throw `Case "${name}" step ${index + 1} bailout because ${tester.step + 1} failed`;
+					}
+					env.clear();
+					await tester.compile();
+					await tester.check(env);
+					await env.run();
+					const context = tester.getContext();
+					if (!tester.next() && context.hasError()) {
+						bailout = true;
+						const errors = context
+							.getError()
+							.map(i => `${i.stack}`.split("\n").join("\t\n"))
+							.join("\n\n");
+						throw new Error(
+							`Case "${name}" failed at step ${tester.step + 1}:\n${errors}`
+						);
+					}
+					stepSignalResolve!();
+				} catch (e) {
+					stepSignalReject!(e);
+				}
+			});
+		}
+
+		chain.finally(() => {
+			ender();
+		});
+
+		afterAll(async () => {
+			await tester.resume();
+		});
+	}
+
 	protected describe(
 		name: string,
 		tester: ITester,
@@ -74,12 +166,12 @@ export class BasicCaseCreator<T extends ECompilerType> {
 			await tester.prepare();
 		});
 
+		let bailout = false;
 		for (let index = 0; index < tester.total; index++) {
 			const description =
 				typeof this._options.description === "function"
 					? this._options.description(name, index)
 					: `step ${index ? `[${index}]` : ""} should pass`;
-			let bailout = false;
 			it(
 				description,
 				async () => {
@@ -110,6 +202,72 @@ export class BasicCaseCreator<T extends ECompilerType> {
 		});
 	}
 
+	protected createConcurrentEnv(): ITestEnv & IConcurrentTestEnv {
+		const tasks: [string, () => Promise<void> | void][] = [];
+		const beforeTasks: (() => Promise<void> | void)[] = [];
+		const afterTasks: (() => Promise<void> | void)[] = [];
+		return {
+			clear: () => {
+				tasks.length = 0;
+				beforeTasks.length = 0;
+				afterTasks.length = 0;
+			},
+			run: async () => {
+				const runFn = async (
+					fn: (done?: (e?: Error) => void) => Promise<void> | void
+				) => {
+					if (fn.length) {
+						await new Promise<void>((resolve, reject) => {
+							fn(e => {
+								if (e) {
+									reject(e);
+								} else {
+									resolve();
+								}
+							});
+						});
+					} else {
+						const res = fn();
+						if (typeof res?.then === "function") {
+							await res;
+						}
+					}
+				};
+
+				for (const [description, fn] of tasks) {
+					for (const before of beforeTasks) {
+						await runFn(before);
+					}
+					try {
+						await runFn(fn);
+					} catch (e) {
+						throw new Error(
+							`Error: ${description} failed\n${(e as Error).stack}`
+						);
+					}
+					for (const after of afterTasks) {
+						await runFn(after);
+					}
+				}
+			},
+			expect,
+			it: (description: string, fn: () => Promise<void> | void) => {
+				expect(typeof description === "string");
+				expect(typeof fn === "function");
+				tasks.push([description, fn]);
+			},
+			beforeEach: (fn: () => Promise<void> | void) => {
+				expect(typeof fn === "function");
+				beforeTasks.push(fn);
+			},
+			afterEach: (fn: () => Promise<void> | void) => {
+				expect(typeof fn === "function");
+				afterTasks.push(fn);
+			},
+			jest
+		};
+	}
+
 	protected createEnv(testConfig: TTestConfig<T>): ITestEnv {
 		if (typeof this._options.runner === "function" && !testConfig.noTest) {
 			return createLazyTestEnv(10000);
@@ -178,4 +336,30 @@ export class BasicCaseCreator<T extends ECompilerType> {
 			})
 		});
 	}
+
+	protected tryRunTask() {
+		while (
+			this.tasks.length !== 0 &&
+			this.currentConcurrent < this.getMaxConcurrent()
+		) {
+			const [_name, starter] = this.tasks.shift()!;
+			this.currentConcurrent++;
+			starter();
+		}
+	}
+
+	protected getMaxConcurrent() {
+		return typeof this._options.concurrent === "number"
+			? this._options.concurrent
+			: DEFAULT_MAX_CONCURRENT;
+	}
+
+	protected registerConcurrentTask(name: string, starter: () => void) {
+		this.tasks.push([name, starter]);
+		this.tryRunTask();
+		return () => {
+			this.currentConcurrent--;
+			this.tryRunTask();
+		};
+	}
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -50,7 +50,8 @@ const creator = new BasicCaseCreator({ @@
     						watchState
     					)
     		);
-    	}
+    	},
+    	concurrent: true
     });
     export function createWatchCase(
@@ Expand Down @@
suite	result
modernjs	✅ success
rspress	✅ success
rslib	❌ failure
rsbuild	✅ success
rsdoctor	❌ failure
examples	✅ success
devserver	✅ success
nuxt	✅ success
Name	Base (2025-03-04 ec71bd0)	Current	Change
10000_big_production-mode_disable-minimize + exec	36.7 s ± 403 ms	36.5 s ± 220 ms	-0.61 %
10000_development-mode + exec	1.71 s ± 20 ms	1.69 s ± 187 ms	-1.18 %
10000_development-mode_hmr + exec	681 ms ± 18 ms	669 ms ± 24 ms	-1.76 %
10000_production-mode + exec	2.21 s ± 117 ms	2.11 s ± 81 ms	-4.94 %
10000_production-mode_persistent-cold + exec	2.33 s ± 50 ms	2.21 s ± 43 ms	-5.28 %
10000_production-mode_persistent-hot + exec	1.64 s ± 26 ms	1.57 s ± 75 ms	-4.49 %
arco-pro_development-mode + exec	1.73 s ± 128 ms	1.77 s ± 161 ms	+2.64 %
arco-pro_development-mode_hmr + exec	375 ms ± 1.9 ms	375 ms ± 1.9 ms	+0.05 %
arco-pro_production-mode + exec	3.54 s ± 113 ms	3.46 s ± 227 ms	-2.29 %
arco-pro_production-mode_generate-package-json-webpack-plugin + exec	3.58 s ± 191 ms	3.56 s ± 143 ms	-0.77 %
arco-pro_production-mode_persistent-cold + exec	3.56 s ± 148 ms	3.49 s ± 80 ms	-2.03 %
arco-pro_production-mode_persistent-hot + exec	2.25 s ± 115 ms	2.28 s ± 109 ms	+1.54 %
arco-pro_production-mode_traverse-chunk-modules + exec	3.53 s ± 177 ms	3.48 s ± 155 ms	-1.32 %
large-dyn-imports_development-mode + exec	1.95 s ± 58 ms	1.91 s ± 9.6 ms	-2.47 %
large-dyn-imports_production-mode + exec	2 s ± 57 ms	1.97 s ± 11 ms	-1.66 %
threejs_development-mode_10x + exec	1.43 s ± 35 ms	1.44 s ± 36 ms	+0.25 %
threejs_development-mode_10x_hmr + exec	786 ms ± 10 ms	785 ms ± 28 ms	-0.11 %
threejs_production-mode_10x + exec	5.03 s ± 77 ms	4.97 s ± 333 ms	-1.19 %
threejs_production-mode_10x_persistent-cold + exec	5.13 s ± 357 ms	4.98 s ± 45 ms	-2.88 %
threejs_production-mode_10x_persistent-hot + exec	4.47 s ± 335 ms	4.28 s ± 94 ms	-4.19 %
10000_big_production-mode_disable-minimize + rss memory	8698 MiB ± 78.5 MiB	9597 MiB ± 414 MiB	+10.33 %
10000_development-mode + rss memory	678 MiB ± 11.8 MiB	691 MiB ± 47.3 MiB	+1.99 %
10000_development-mode_hmr + rss memory	1247 MiB ± 285 MiB	1514 MiB ± 263 MiB	+21.43 %
10000_production-mode + rss memory	650 MiB ± 26.5 MiB	693 MiB ± 20.4 MiB	+6.72 %
10000_production-mode_persistent-cold + rss memory	762 MiB ± 20 MiB	816 MiB ± 63.6 MiB	+7.12 %
10000_production-mode_persistent-hot + rss memory	735 MiB ± 11.5 MiB	749 MiB ± 41 MiB	+1.89 %
arco-pro_development-mode + rss memory	636 MiB ± 21.9 MiB	593 MiB ± 62.7 MiB	-6.73 %
arco-pro_development-mode_hmr + rss memory	709 MiB ± 21.4 MiB	638 MiB ± 57.9 MiB	-10.04 %
arco-pro_production-mode + rss memory	776 MiB ± 38.8 MiB	746 MiB ± 94.7 MiB	-3.95 %
arco-pro_production-mode_generate-package-json-webpack-plugin + rss memory	792 MiB ± 25.6 MiB	718 MiB ± 47.4 MiB	-9.36 %
arco-pro_production-mode_persistent-cold + rss memory	863 MiB ± 20.6 MiB	812 MiB ± 58 MiB	-5.97 %
arco-pro_production-mode_persistent-hot + rss memory	832 MiB ± 32 MiB	771 MiB ± 55.5 MiB	-7.32 %
arco-pro_production-mode_traverse-chunk-modules + rss memory	800 MiB ± 35.8 MiB	739 MiB ± 54 MiB	-7.66 %
large-dyn-imports_development-mode + rss memory	697 MiB ± 17 MiB	671 MiB ± 18.5 MiB	-3.74 %
large-dyn-imports_production-mode + rss memory	576 MiB ± 3.62 MiB	560 MiB ± 6.97 MiB	-2.81 %
threejs_development-mode_10x + rss memory	651 MiB ± 59.5 MiB	659 MiB ± 42.5 MiB	+1.30 %
threejs_development-mode_10x_hmr + rss memory	974 MiB ± 301 MiB	916 MiB ± 173 MiB	-5.97 %
threejs_production-mode_10x + rss memory	919 MiB ± 56.1 MiB	962 MiB ± 44.7 MiB	+4.63 %
threejs_production-mode_10x_persistent-cold + rss memory	1050 MiB ± 71.5 MiB	1033 MiB ± 72.6 MiB	-1.60 %
threejs_production-mode_10x_persistent-hot + rss memory	916 MiB ± 60.7 MiB	898 MiB ± 46.9 MiB	-1.98 %