diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 8642b7f435..444a72e0df 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -259,6 +259,14 @@ The maximum number of retry attempts given to failed tests. Learn more about [te Use [`property: TestConfig.retries`] to change this option for all projects. +## property: TestProject.stage +* since: v1.28 +- type: ?<[int]> + +An integer number that defines when the project should run relative to other projects. Each project runs in exactly +one stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in +each stage. Exeution order between projecs in the same stage is undefined. + ## property: TestProject.testDir * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 908f72bc48..5459cc4ab1 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -277,6 +277,7 @@ export class Loader { const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results')); const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir); const name = takeFirst(projectConfig.name, config.name, ''); + const stage = takeFirst(projectConfig.stage, 0); let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name)); if (process.env.PLAYWRIGHT_DOCKER) { @@ -296,6 +297,7 @@ export class Loader { metadata: takeFirst(projectConfig.metadata, config.metadata, undefined), name, testDir, + stage, _respectGitIgnore: respectGitIgnore, snapshotDir, _screenshotsDir: screenshotsDir, diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 2f093f54db..ea4a6e7767 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -52,47 +52,8 @@ const readDirAsync = promisify(fs.readdir); const readFileAsync = promisify(fs.readFile); export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs']; -type ProjectConstraints = { - projectName: string; - testFileMatcher: Matcher; - testTitleMatcher: Matcher; -}; - -// Project group is a sequence of run phases. -class RunPhase { - static collectRunPhases(options: RunOptions, config: FullConfigInternal): RunPhase[] { - const phases: RunPhase[] = []; - const testFileMatcher = fileMatcherFrom(options.testFileFilters); - const testTitleMatcher = options.testTitleMatcher; - const projects = options.projectFilter ?? config.projects.map(p => p.name); - phases.push(new RunPhase(projects.map(projectName => ({ - projectName, - testFileMatcher, - testTitleMatcher - })))); - return phases; - } - - constructor(private _projectWithConstraints: ProjectConstraints[]) { - } - - projectNames(): string[] { - return this._projectWithConstraints.map(p => p.projectName); - } - - testFileMatcher(projectName: string) { - return this._projectEntry(projectName).testFileMatcher; - } - - testTitleMatcher(projectName: string) { - return this._projectEntry(projectName).testTitleMatcher; - } - - private _projectEntry(projectName: string) { - projectName = projectName.toLocaleLowerCase(); - return this._projectWithConstraints.find(p => p.projectName.toLocaleLowerCase() === projectName)!; - } -} +// Test run is a sequence of run phases aka stages. +type RunStage = FullProjectInternal[]; type RunOptions = { listOnly?: boolean; @@ -238,13 +199,8 @@ export class Runner { } async listTestFiles(configFile: string, projectNames: string[] | undefined): Promise { - const projects = projectNames ?? this._loader.fullConfig().projects.map(p => p.name); - const phase = new RunPhase(projects.map(projectName => ({ - projectName, - testFileMatcher: () => true, - testTitleMatcher: () => true, - }))); - const filesByProject = await this._collectFiles(phase); + const projects = this._collectProjects(projectNames); + const filesByProject = await this._collectFiles(projects, () => true); const report: any = { projects: [] }; @@ -259,7 +215,10 @@ export class Runner { return report; } - private _collectProjects(projectNames: string[]): FullProjectInternal[] { + private _collectProjects(projectNames?: string[]): FullProjectInternal[] { + const fullConfig = this._loader.fullConfig(); + if (!projectNames) + return [...fullConfig.projects]; const projectsToFind = new Set(); const unknownProjects = new Map(); projectNames.forEach(n => { @@ -267,7 +226,6 @@ export class Runner { projectsToFind.add(name); unknownProjects.set(name, n); }); - const fullConfig = this._loader.fullConfig(); const projects = fullConfig.projects.filter(project => { const name = project.name.toLocaleLowerCase(); unknownProjects.delete(name); @@ -283,8 +241,7 @@ export class Runner { return projects; } - private async _collectFiles(runPhase: RunPhase): Promise> { - const projects = this._collectProjects(runPhase.projectNames()); + private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise> { const files = new Map(); for (const project of projects) { const allFiles = await collectFiles(project.testDir, project._respectGitIgnore); @@ -292,7 +249,6 @@ export class Runner { const testIgnore = createFileMatcher(project.testIgnore); const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); - const testFileFilter = runPhase.testFileMatcher(project.name); const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file) && testFileExtension(file)); files.set(project, testFiles); } @@ -305,11 +261,12 @@ export class Runner { // test groups from the previos entries must finish before entry starts. const concurrentTestGroups = []; const rootSuite = new Suite('', 'root'); - const runPhases = RunPhase.collectRunPhases(options, config); - assert(runPhases.length > 0); - for (const phase of runPhases) { + const projects = this._collectProjects(options.projectFilter); + const runStages = collectRunStages(projects); + assert(runStages.length > 0); + for (const stage of runStages) { // TODO: do not collect files for each project multiple times. - const filesByProject = await this._collectFiles(phase); + const filesByProject = await this._collectFiles(stage, fileMatcherFrom(options.testFileFilters)); const allTestFiles = new Set(); for (const files of filesByProject.values()) @@ -354,8 +311,6 @@ export class Runner { for (const [project, files] of filesByProject) { const grepMatcher = createTitleMatcher(project.grep); const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null; - // TODO: also apply title matcher from options. - const groupTitleMatcher = phase.testTitleMatcher(project.name); const projectSuite = new Suite(project.name, 'project'); projectSuite._projectConfig = project; @@ -371,7 +326,7 @@ export class Runner { const grepTitle = test.titlePath().join(' '); if (grepInvertMatcher?.(grepTitle)) return false; - return grepMatcher(grepTitle) && groupTitleMatcher(grepTitle); + return grepMatcher(grepTitle) && options.testTitleMatcher(grepTitle); }); if (builtSuite) projectSuite._addSuite(builtSuite); @@ -745,6 +700,20 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) { return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`; } +function collectRunStages(projects: FullProjectInternal[]): RunStage[] { + const stages: RunStage[] = []; + const stageToProjects = new MultiMap(); + for (const p of projects) + stageToProjects.set(p.stage, p); + const stageIds = Array.from(stageToProjects.keys()); + stageIds.sort((a, b) => a - b); + for (const stage of stageIds) { + const projects = stageToProjects.get(stage); + stages.push(projects); + } + return stages; +} + function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] { // This function groups tests that can be run together. // Tests cannot be run together when: diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 108c3a205f..bb1d6c72b0 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -256,6 +256,12 @@ export interface FullProject { * all projects. */ retries: number; + /** + * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one + * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each + * stage. Exeution order between projecs in the same stage is undefined. + */ + stage: number; /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * @@ -4458,6 +4464,13 @@ interface TestProject { */ retries?: number; + /** + * An integer number that defines when the project should run relative to other projects. Each project runs in exactly one + * stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each + * stage. Exeution order between projecs in the same stage is undefined. + */ + stage?: number; + /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * diff --git a/tests/playwright-test/stages.spec.ts b/tests/playwright-test/stages.spec.ts new file mode 100644 index 0000000000..569d529a67 --- /dev/null +++ b/tests/playwright-test/stages.spec.ts @@ -0,0 +1,209 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test'; +import path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +function createConfigWithProjects(names: string[], testInfo: TestInfo, projectTemplates?: { [name: string]: PlaywrightTestProject }): Record { + const config: PlaywrightTestConfig = { + projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })), + }; + const files = {}; + for (const name of names) { + files[`${name}/${name}.spec.ts`] = ` + const { test } = pwt; + test('${name} test', async () => { + await new Promise(f => setTimeout(f, 100)); + });`; + } + function replacer(key, value) { + if (value instanceof RegExp) + return `RegExp(${value.toString()})`; + else + return value; + } + files['playwright.config.ts'] = ` + import * as path from 'path'; + module.exports = ${JSON.stringify(config, replacer, 2)}; + `.replace(/"RegExp\((.*)\)"/g, '$1'); + return files; +} + +type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; + +function formatTimeline(timeline: Timeline) { + return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n'); +} + +function projectNames(timeline: Timeline) { + const projectNames = Array.from(new Set(timeline.map(({ titlePath }) => titlePath[1])).keys()); + projectNames.sort(); + return projectNames; +} + +function expectRunBefore(timeline: Timeline, before: string[], after: string[]) { + const begin = new Map(); + const end = new Map(); + for (let i = 0; i < timeline.length; i++) { + const projectName = timeline[i].titlePath[1]; + const map = timeline[i].event === 'begin' ? begin : end; + const oldIndex = map.get(projectName) ?? i; + const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex); + map.set(projectName, newIndex); + } + for (const b of before) { + for (const a of after) { + const bEnd = end.get(b) as number; + expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy(); + const aBegin = begin.get(a) as number; + expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy(); + if (bEnd < aBegin) + continue; + throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`); + } + } +} + +test('should work for two projects', async ({ runGroups }, testInfo) => { + await test.step(`order a then b`, async () => { + const projectTemplates = { + 'a': { + stage: 10 + }, + 'b': { + stage: 20 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin] +a > a${path.sep}a.spec.ts > a test [end] +b > b${path.sep}b.spec.ts > b test [begin] +b > b${path.sep}b.spec.ts > b test [end]`); + }); + await test.step(`order b then a`, async () => { + const projectTemplates = { + 'a': { + stage: 20 + }, + 'b': { + stage: 10 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin] +b > b${path.sep}b.spec.ts > b test [end] +a > a${path.sep}a.spec.ts > a test [begin] +a > a${path.sep}a.spec.ts > a test [end]`); + }); +}); + + +test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'e': { + stage: -100 + }, + 'a': { + stage: 100 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expectRunBefore(timeline, ['e'], ['d', 'c', 'b']); + expectRunBefore(timeline, ['d', 'c', 'b'], ['a']); + expect(passed).toBe(5); +}); + +test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'a': { + stage: -30 + }, + 'b': { + stage: -30 + }, + 'e': { + stage: 40 + }, + 'f': { + stage: 40 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expectRunBefore(timeline, ['a', 'b'], ['c', 'd']); + expectRunBefore(timeline, ['c', 'd'], ['e', 'f']); + expect(passed).toBe(6); +}); + +test('should order project according to stage 1-1-2-2', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'a': { + stage: 10 + }, + 'b': { + stage: 10 + }, + 'd': { + stage: -10 + }, + 'e': { + stage: -20 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(6); + expect(projectNames(timeline)).toEqual(['a', 'b', 'c', 'd', 'e', 'f']); + expectRunBefore(timeline, ['e'], ['a', 'b', 'c', 'd', 'f']); // -20 + expectRunBefore(timeline, ['d'], ['a', 'b', 'c', 'f']); // -10 + expectRunBefore(timeline, ['c', 'f'], ['a', 'b']); // 0 + expect(passed).toBe(6); +}); + +test('should work with project filter', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'a': { + stage: 10 + }, + 'b': { + stage: 10 + }, + 'e': { + stage: -10 + }, + 'f': { + stage: -10 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, projectTemplates); + const { exitCode, passed, timeline } = await runGroups(configWithFiles, { project: ['b', 'c', 'e'] }); + expect(exitCode).toBe(0); + expect(passed).toBe(3); + expect(projectNames(timeline)).toEqual(['b', 'c', 'e']); + expectRunBefore(timeline, ['e'], ['b', 'c']); // -10 < 0 + expectRunBefore(timeline, ['c'], ['b']); // 0 < 10 + expect(passed).toBe(3); +}); \ No newline at end of file diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 3e1f8aae6a..c6a4886a73 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -46,6 +46,7 @@ export interface FullProject { outputDir: string; repeatEach: number; retries: number; + stage: number; testDir: string; testIgnore: string | RegExp | (string | RegExp)[]; testMatch: string | RegExp | (string | RegExp)[];