feat(runner): project.stage (#17971)

docs/src/test-api/class-testproject.md

@@ -259,6 +259,14 @@ The maximum number of retry attempts given to failed tests. Learn more about [te
 
 Use [`property: TestConfig.retries`] to change this option for all projects.
 
+## property: TestProject.stage
+* since: v1.28
+- type: ?<[int]>
+
+An integer number that defines when the project should run relative to other projects. Each project runs in exactly
+one stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in
+each stage. Exeution order between projecs in the same stage is undefined.
+
 ## property: TestProject.testDir
 * since: v1.10
 - type: ?<[string]>

packages/playwright-test/src/loader.ts

@@ -277,6 +277,7 @@ export class Loader {
     const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results'));
     const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir);
     const name = takeFirst(projectConfig.name, config.name, '');
+    const stage =  takeFirst(projectConfig.stage, 0);
 
     let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
     if (process.env.PLAYWRIGHT_DOCKER) {
@@ -296,6 +297,7 @@ export class Loader {
       metadata: takeFirst(projectConfig.metadata, config.metadata, undefined),
       name,
       testDir,
+      stage,
       _respectGitIgnore: respectGitIgnore,
       snapshotDir,
       _screenshotsDir: screenshotsDir,

packages/playwright-test/src/runner.ts

@@ -52,47 +52,8 @@ const readDirAsync = promisify(fs.readdir);
 const readFileAsync = promisify(fs.readFile);
 export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
 
-type ProjectConstraints = {
-  projectName: string;
-  testFileMatcher: Matcher;
-  testTitleMatcher: Matcher;
-};
-
-// Project group is a sequence of run phases.
-class RunPhase {
-  static collectRunPhases(options: RunOptions, config: FullConfigInternal): RunPhase[] {
-    const phases: RunPhase[] = [];
-    const testFileMatcher = fileMatcherFrom(options.testFileFilters);
-    const testTitleMatcher = options.testTitleMatcher;
-    const projects = options.projectFilter ?? config.projects.map(p => p.name);
-    phases.push(new RunPhase(projects.map(projectName => ({
-      projectName,
-      testFileMatcher,
-      testTitleMatcher
-    }))));
-    return phases;
-  }
-
-  constructor(private _projectWithConstraints: ProjectConstraints[]) {
-  }
-
-  projectNames(): string[] {
-    return this._projectWithConstraints.map(p => p.projectName);
-  }
-
-  testFileMatcher(projectName: string) {
-    return this._projectEntry(projectName).testFileMatcher;
-  }
-
-  testTitleMatcher(projectName: string) {
-    return this._projectEntry(projectName).testTitleMatcher;
-  }
-
-  private _projectEntry(projectName: string) {
-    projectName = projectName.toLocaleLowerCase();
-    return this._projectWithConstraints.find(p => p.projectName.toLocaleLowerCase() === projectName)!;
-  }
-}
+// Test run is a sequence of run phases aka stages.
+type RunStage = FullProjectInternal[];
 
 type RunOptions = {
   listOnly?: boolean;
@@ -238,13 +199,8 @@ export class Runner {
   }
 
   async listTestFiles(configFile: string, projectNames: string[] | undefined): Promise<any> {
-    const projects = projectNames ?? this._loader.fullConfig().projects.map(p => p.name);
-    const phase = new RunPhase(projects.map(projectName => ({
-      projectName,
-      testFileMatcher: () => true,
-      testTitleMatcher: () => true,
-    })));
-    const filesByProject = await this._collectFiles(phase);
+    const projects = this._collectProjects(projectNames);
+    const filesByProject = await this._collectFiles(projects, () => true);
     const report: any = {
       projects: []
     };
@@ -259,7 +215,10 @@ export class Runner {
     return report;
   }
 
-  private _collectProjects(projectNames: string[]): FullProjectInternal[] {
+  private _collectProjects(projectNames?: string[]): FullProjectInternal[] {
+    const fullConfig = this._loader.fullConfig();
+    if (!projectNames)
+      return [...fullConfig.projects];
     const projectsToFind = new Set<string>();
     const unknownProjects = new Map<string, string>();
     projectNames.forEach(n => {
@@ -267,7 +226,6 @@ export class Runner {
       projectsToFind.add(name);
       unknownProjects.set(name, n);
     });
-    const fullConfig = this._loader.fullConfig();
     const projects = fullConfig.projects.filter(project => {
       const name = project.name.toLocaleLowerCase();
       unknownProjects.delete(name);
@@ -283,8 +241,7 @@ export class Runner {
     return projects;
   }
 
-  private async _collectFiles(runPhase: RunPhase): Promise<Map<FullProjectInternal, string[]>> {
-    const projects = this._collectProjects(runPhase.projectNames());
+  private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<Map<FullProjectInternal, string[]>> {
     const files = new Map<FullProjectInternal, string[]>();
     for (const project of projects) {
       const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
@@ -292,7 +249,6 @@ export class Runner {
       const testIgnore = createFileMatcher(project.testIgnore);
       const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
       const testFileExtension = (file: string) => extensions.includes(path.extname(file));
-      const testFileFilter = runPhase.testFileMatcher(project.name);
       const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file) && testFileExtension(file));
       files.set(project, testFiles);
     }
@@ -305,11 +261,12 @@ export class Runner {
     // test groups from the previos entries must finish before entry starts.
     const concurrentTestGroups = [];
     const rootSuite = new Suite('', 'root');
-    const runPhases = RunPhase.collectRunPhases(options, config);
-    assert(runPhases.length > 0);
-    for (const phase of runPhases) {
+    const projects = this._collectProjects(options.projectFilter);
+    const runStages = collectRunStages(projects);
+    assert(runStages.length > 0);
+    for (const stage of runStages) {
       // TODO: do not collect files for each project multiple times.
-      const filesByProject = await this._collectFiles(phase);
+      const filesByProject = await this._collectFiles(stage, fileMatcherFrom(options.testFileFilters));
 
       const allTestFiles = new Set<string>();
       for (const files of filesByProject.values())
@@ -354,8 +311,6 @@ export class Runner {
       for (const [project, files] of filesByProject) {
         const grepMatcher = createTitleMatcher(project.grep);
         const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
-        // TODO: also apply title matcher from options.
-        const groupTitleMatcher = phase.testTitleMatcher(project.name);
 
         const projectSuite = new Suite(project.name, 'project');
         projectSuite._projectConfig = project;
@@ -371,7 +326,7 @@ export class Runner {
               const grepTitle = test.titlePath().join(' ');
               if (grepInvertMatcher?.(grepTitle))
                 return false;
-              return grepMatcher(grepTitle) && groupTitleMatcher(grepTitle);
+              return grepMatcher(grepTitle) && options.testTitleMatcher(grepTitle);
             });
             if (builtSuite)
               projectSuite._addSuite(builtSuite);
@@ -745,6 +700,20 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) {
   return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`;
 }
 
+function collectRunStages(projects: FullProjectInternal[]): RunStage[] {
+  const stages: RunStage[] = [];
+  const stageToProjects = new MultiMap<number, FullProjectInternal>();
+  for (const p of projects)
+    stageToProjects.set(p.stage, p);
+  const stageIds = Array.from(stageToProjects.keys());
+  stageIds.sort((a, b) => a - b);
+  for (const stage of stageIds) {
+    const projects = stageToProjects.get(stage);
+    stages.push(projects);
+  }
+  return stages;
+}
+
 function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] {
   // This function groups tests that can be run together.
   // Tests cannot be run together when:

packages/playwright-test/types/test.d.ts

@@ -256,6 +256,12 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * all projects.
    */
   retries: number;
+  /**
+   * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one
+   * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each
+   * stage. Exeution order between projecs in the same stage is undefined.
+   */
+  stage: number;
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *
@@ -4458,6 +4464,13 @@ interface TestProject {
    */
   retries?: number;
 
+  /**
+   * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one
+   * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each
+   * stage. Exeution order between projecs in the same stage is undefined.
+   */
+  stage?: number;
+
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *

tests/playwright-test/stages.spec.ts

@@ -0,0 +1,209 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test';
+import path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+function createConfigWithProjects(names: string[], testInfo: TestInfo, projectTemplates?: { [name: string]: PlaywrightTestProject }): Record<string, string> {
+  const config: PlaywrightTestConfig = {
+    projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })),
+  };
+  const files = {};
+  for (const name of names) {
+    files[`${name}/${name}.spec.ts`] = `
+      const { test } = pwt;
+      test('${name} test', async () => {
+        await new Promise(f => setTimeout(f, 100));
+      });`;
+  }
+  function replacer(key, value) {
+    if (value instanceof RegExp)
+      return `RegExp(${value.toString()})`;
+    else
+      return value;
+  }
+  files['playwright.config.ts'] = `
+    import * as path from 'path';
+    module.exports = ${JSON.stringify(config, replacer, 2)};
+    `.replace(/"RegExp\((.*)\)"/g, '$1');
+  return files;
+}
+
+type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
+
+function formatTimeline(timeline: Timeline) {
+  return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
+}
+
+function projectNames(timeline: Timeline) {
+  const projectNames = Array.from(new Set(timeline.map(({ titlePath }) => titlePath[1])).keys());
+  projectNames.sort();
+  return projectNames;
+}
+
+function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
+  const begin = new Map<string, number>();
+  const end = new Map<string, number>();
+  for (let i = 0; i < timeline.length; i++) {
+    const projectName = timeline[i].titlePath[1];
+    const map = timeline[i].event === 'begin' ? begin : end;
+    const oldIndex = map.get(projectName) ?? i;
+    const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex);
+    map.set(projectName, newIndex);
+  }
+  for (const b of before) {
+    for (const a of after) {
+      const bEnd = end.get(b) as number;
+      expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy();
+      const aBegin = begin.get(a) as number;
+      expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy();
+      if (bEnd < aBegin)
+        continue;
+      throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`);
+    }
+  }
+}
+
+test('should work for two projects', async ({ runGroups }, testInfo) => {
+  await test.step(`order a then b`, async () => {
+    const projectTemplates = {
+      'a': {
+        stage: 10
+      },
+      'b': {
+        stage: 20
+      },
+    };
+    const configWithFiles = createConfigWithProjects(['a', 'b'], testInfo, projectTemplates);
+    const { exitCode, passed, timeline } = await runGroups(configWithFiles);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
+a > a${path.sep}a.spec.ts > a test [end]
+b > b${path.sep}b.spec.ts > b test [begin]
+b > b${path.sep}b.spec.ts > b test [end]`);
+  });
+  await test.step(`order b then a`, async () => {
+    const projectTemplates = {
+      'a': {
+        stage: 20
+      },
+      'b': {
+        stage: 10
+      },
+    };
+    const configWithFiles = createConfigWithProjects(['a', 'b'], testInfo, projectTemplates);
+    const { exitCode, passed, timeline } = await runGroups(configWithFiles);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin]
+b > b${path.sep}b.spec.ts > b test [end]
+a > a${path.sep}a.spec.ts > a test [begin]
+a > a${path.sep}a.spec.ts > a test [end]`);
+  });
+});
+
+
+test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'e': {
+      stage: -100
+    },
+    'a': {
+      stage: 100
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expectRunBefore(timeline, ['e'], ['d', 'c', 'b']);
+  expectRunBefore(timeline, ['d', 'c', 'b'], ['a']);
+  expect(passed).toBe(5);
+});
+
+test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      stage: -30
+    },
+    'b': {
+      stage: -30
+    },
+    'e': {
+      stage: 40
+    },
+    'f': {
+      stage: 40
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expectRunBefore(timeline, ['a', 'b'], ['c', 'd']);
+  expectRunBefore(timeline, ['c', 'd'], ['e', 'f']);
+  expect(passed).toBe(6);
+});
+
+test('should order project according to stage 1-1-2-2', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      stage: 10
+    },
+    'b': {
+      stage: 10
+    },
+    'd': {
+      stage: -10
+    },
+    'e': {
+      stage: -20
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(6);
+  expect(projectNames(timeline)).toEqual(['a', 'b', 'c', 'd', 'e', 'f']);
+  expectRunBefore(timeline, ['e'], ['a', 'b', 'c', 'd', 'f']); // -20
+  expectRunBefore(timeline, ['d'], ['a', 'b', 'c', 'f']); // -10
+  expectRunBefore(timeline, ['c', 'f'], ['a', 'b']); // 0
+  expect(passed).toBe(6);
+});
+
+test('should work with project filter', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      stage: 10
+    },
+    'b': {
+      stage: 10
+    },
+    'e': {
+      stage: -10
+    },
+    'f': {
+      stage: -10
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { project: ['b', 'c', 'e'] });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(3);
+  expect(projectNames(timeline)).toEqual(['b', 'c', 'e']);
+  expectRunBefore(timeline, ['e'], ['b', 'c']); // -10 < 0
+  expectRunBefore(timeline, ['c'], ['b']); // 0 < 10
+  expect(passed).toBe(3);
+});

utils/generate_types/overrides-test.d.ts

@@ -46,6 +46,7 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   outputDir: string;
   repeatEach: number;
   retries: number;
+  stage: number;
   testDir: string;
   testIgnore: string | RegExp | (string | RegExp)[];
   testMatch: string | RegExp | (string | RegExp)[];