feat(runner): project.stage (#17971)

docs/src/test-api/class-testproject.md

@@ -259,6 +259,14 @@ The maximum number of retry attempts given to failed tests. Learn more about [te
 
 Use [`property: TestConfig.retries`] to change this option for all projects.
 
+## property: TestProject.stage
+* since: v1.28
+- type: ?<[int]>
+
+An integer number that defines when the project should run relative to other projects. Each project runs in exactly
+one stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in
+each stage. Exeution order between projecs in the same stage is undefined.
+
 ## property: TestProject.testDir
 * since: v1.10
 - type: ?<[string]>

packages/playwright-test/src/loader.ts

@@ -277,6 +277,7 @@ export class Loader {
     const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results'));
     const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir);
     const name = takeFirst(projectConfig.name, config.name, '');
+    const stage =  takeFirst(projectConfig.stage, 0);
 
     let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
     if (process.env.PLAYWRIGHT_DOCKER) {
@@ -296,6 +297,7 @@ export class Loader {
       metadata: takeFirst(projectConfig.metadata, config.metadata, undefined),
       name,
       testDir,
+      stage,
       _respectGitIgnore: respectGitIgnore,
       snapshotDir,
       _screenshotsDir: screenshotsDir,

packages/playwright-test/src/runner.ts

@@ -52,47 +52,8 @@ const readDirAsync = promisify(fs.readdir);
 const readFileAsync = promisify(fs.readFile);
 export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
 
-type ProjectConstraints = {
-  projectName: string;
-  testFileMatcher: Matcher;
-  testTitleMatcher: Matcher;
-};
-
-// Project group is a sequence of run phases.
-class RunPhase {
-  static collectRunPhases(options: RunOptions, config: FullConfigInternal): RunPhase[] {
-    const phases: RunPhase[] = [];
-    const testFileMatcher = fileMatcherFrom(options.testFileFilters);
-    const testTitleMatcher = options.testTitleMatcher;
-    const projects = options.projectFilter ?? config.projects.map(p => p.name);
-    phases.push(new RunPhase(projects.map(projectName => ({
-      projectName,
-      testFileMatcher,
-      testTitleMatcher
-    }))));
-    return phases;
-  }
-
-  constructor(private _projectWithConstraints: ProjectConstraints[]) {
-  }
-
-  projectNames(): string[] {
-    return this._projectWithConstraints.map(p => p.projectName);
-  }
-
-  testFileMatcher(projectName: string) {
-    return this._projectEntry(projectName).testFileMatcher;
-  }
-
-  testTitleMatcher(projectName: string) {
-    return this._projectEntry(projectName).testTitleMatcher;
-  }
-
-  private _projectEntry(projectName: string) {
-    projectName = projectName.toLocaleLowerCase();
-    return this._projectWithConstraints.find(p => p.projectName.toLocaleLowerCase() === projectName)!;
-  }
-}
+// Test run is a sequence of run phases aka stages.
+type RunStage = FullProjectInternal[];
 
 type RunOptions = {
   listOnly?: boolean;
@@ -238,13 +199,8 @@ export class Runner {
   }
 
   async listTestFiles(configFile: string, projectNames: string[] | undefined): Promise<any> {
-    const projects = projectNames ?? this._loader.fullConfig().projects.map(p => p.name);
-    const phase = new RunPhase(projects.map(projectName => ({
-      projectName,
-      testFileMatcher: () => true,
-      testTitleMatcher: () => true,
-    })));
-    const filesByProject = await this._collectFiles(phase);
+    const projects = this._collectProjects(projectNames);
+    const filesByProject = await this._collectFiles(projects, () => true);
     const report: any = {
       projects: []
     };
@@ -259,15 +215,17 @@ export class Runner {
     return report;
   }
 
-  private _collectProjects(projectNames: string[]): FullProjectInternal[] {
+  private _collectProjects(projectNames?: string[]): FullProjectInternal[] {
+    const fullConfig = this._loader.fullConfig();
+    if (!projectNames)
+      return [...fullConfig.projects];
     const projectsToFind = new Set<string>();
     const unknownProjects = new Map<string, string>();
     projectNames.forEach(n => {
       const name = n.toLocaleLowerCase();
       projectsToFind.add(name);
       unknownProjects.set(name, n);
     });
-    const fullConfig = this._loader.fullConfig();
     const projects = fullConfig.projects.filter(project => {
       const name = project.name.toLocaleLowerCase();
       unknownProjects.delete(name);
@@ -283,16 +241,14 @@ export class Runner {
     return projects;
   }
 
-  private async _collectFiles(runPhase: RunPhase): Promise<Map<FullProjectInternal, string[]>> {
-    const projects = this._collectProjects(runPhase.projectNames());
+  private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<Map<FullProjectInternal, string[]>> {
     const files = new Map<FullProjectInternal, string[]>();
     for (const project of projects) {
       const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
       const testMatch = createFileMatcher(project.testMatch);
       const testIgnore = createFileMatcher(project.testIgnore);
       const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
       const testFileExtension = (file: string) => extensions.includes(path.extname(file));
-      const testFileFilter = runPhase.testFileMatcher(project.name);
       const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file) && testFileExtension(file));
       files.set(project, testFiles);
     }
@@ -305,11 +261,12 @@ export class Runner {
     // test groups from the previos entries must finish before entry starts.
     const concurrentTestGroups = [];
     const rootSuite = new Suite('', 'root');
-    const runPhases = RunPhase.collectRunPhases(options, config);
-    assert(runPhases.length > 0);
-    for (const phase of runPhases) {
+    const projects = this._collectProjects(options.projectFilter);
+    const runStages = collectRunStages(projects);
+    assert(runStages.length > 0);
+    for (const stage of runStages) {
       // TODO: do not collect files for each project multiple times.
-      const filesByProject = await this._collectFiles(phase);
+      const filesByProject = await this._collectFiles(stage, fileMatcherFrom(options.testFileFilters));
 
       const allTestFiles = new Set<string>();
       for (const files of filesByProject.values())
@@ -354,8 +311,6 @@ export class Runner {
       for (const [project, files] of filesByProject) {
         const grepMatcher = createTitleMatcher(project.grep);
         const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
-        // TODO: also apply title matcher from options.
-        const groupTitleMatcher = phase.testTitleMatcher(project.name);
 
         const projectSuite = new Suite(project.name, 'project');
         projectSuite._projectConfig = project;
@@ -371,7 +326,7 @@ export class Runner {
               const grepTitle = test.titlePath().join(' ');
               if (grepInvertMatcher?.(grepTitle))
                 return false;
-              return grepMatcher(grepTitle) && groupTitleMatcher(grepTitle);
+              return grepMatcher(grepTitle) && options.testTitleMatcher(grepTitle);
             });
             if (builtSuite)
               projectSuite._addSuite(builtSuite);
@@ -745,6 +700,20 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) {
   return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`;
 }
 
+function collectRunStages(projects: FullProjectInternal[]): RunStage[] {
+  const stages: RunStage[] = [];
+  const stageToProjects = new MultiMap<number, FullProjectInternal>();
+  for (const p of projects)
+    stageToProjects.set(p.stage, p);
+  const stageIds = Array.from(stageToProjects.keys());
+  stageIds.sort((a, b) => a - b);
+  for (const stage of stageIds) {
+    const projects = stageToProjects.get(stage);
+    stages.push(projects);
+  }
+  return stages;
+}
+
 function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] {
   // This function groups tests that can be run together.
   // Tests cannot be run together when:

packages/playwright-test/types/test.d.ts

@@ -256,6 +256,12 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * all projects.
    */
   retries: number;
+  /**
+   * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one
+   * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each
+   * stage. Exeution order between projecs in the same stage is undefined.
+   */
+  stage: number;
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *
@@ -4458,6 +4464,13 @@ interface TestProject {
    */
   retries?: number;
 
+  /**
+   * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one
+   * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each
+   * stage. Exeution order between projecs in the same stage is undefined.
+   */
+  stage?: number;
+
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *