diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 816060c394..9432217e8d 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -25,25 +25,16 @@ module.exports = defineConfig({ // Options specific to each project. projects: [ { - name: 'Desktop Chromium', - use: { - browserName: 'chromium', - viewport: { width: 1280, height: 720 }, - }, + name: 'chromium', + use: devices['Desktop Chrome'], }, { - name: 'Desktop Safari', - use: { - browserName: 'webkit', - viewport: { width: 1280, height: 720 }, - } + name: 'firefox', + use: devices['Desktop Firefox'], }, { - name: 'Desktop Firefox', - use: { - browserName: 'firefox', - viewport: { width: 1280, height: 720 }, - } + name: 'webkit', + use: devices['Desktop Safari'], }, { name: 'Mobile Chrome', @@ -71,25 +62,16 @@ export default defineConfig({ // Options specific to each project. projects: [ { - name: 'Desktop Chromium', - use: { - browserName: 'chromium', - viewport: { width: 1280, height: 720 }, - }, + name: 'chromium', + use: devices['Desktop Chrome'], }, { - name: 'Desktop Safari', - use: { - browserName: 'webkit', - viewport: { width: 1280, height: 720 }, - } + name: 'firefox', + use: devices['Desktop Firefox'], }, { - name: 'Desktop Firefox', - use: { - browserName: 'firefox', - viewport: { width: 1280, height: 720 }, - } + name: 'webkit', + use: devices['Desktop Safari'], }, { name: 'Mobile Chrome', @@ -103,6 +85,44 @@ export default defineConfig({ }); ``` +## property: TestProject.dependencies +* since: v1.31 +- type: ?<[Array]<[string]>> + +List of projects that need to run before any test in this project runs. Dependencies can +be useful for configuring the global setup actions in a way that every action is a test. +For example: + +```js +// playwright.config.ts +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + projects: [ + { + name: 'setup', + testMatch: /global.setup\.ts/, + dependencies: ['setup'], + }, + { + name: 'chromium', + use: devices['Desktop Chrome'], + dependencies: ['setup'], + }, + { + name: 'firefox', + use: devices['Desktop Firefox'], + dependencies: ['setup'], + }, + { + name: 'webkit', + use: devices['Desktop Safari'], + dependencies: ['setup'], + }, + ], +}); +``` + ## property: TestProject.expect * since: v1.10 - type: ?<[Object]> diff --git a/packages/playwright-test/src/common/configLoader.ts b/packages/playwright-test/src/common/configLoader.ts index 52ea10d7e1..9687d71942 100644 --- a/packages/playwright-test/src/common/configLoader.ts +++ b/packages/playwright-test/src/common/configLoader.ts @@ -203,6 +203,7 @@ export class ConfigLoader { _fullConfig: fullConfig, _fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined), _expect: takeFirst(projectConfig.expect, config.expect, {}), + _deps: [], grep: takeFirst(projectConfig.grep, config.grep, baseFullConfig.grep), grepInvert: takeFirst(projectConfig.grepInvert, config.grepInvert, baseFullConfig.grepInvert), outputDir, @@ -218,8 +219,7 @@ export class ConfigLoader { testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/?(*.)@(spec|test).*'), timeout: takeFirst(projectConfig.timeout, config.timeout, defaultTimeout), use: mergeObjects(config.use, projectConfig.use), - _deps: (projectConfig as any)._deps || [], - _depProjects: [], + dependencies: projectConfig.dependencies || [], }; } } @@ -460,13 +460,13 @@ function resolveScript(id: string, rootDir: string) { function resolveProjectDependencies(projects: FullProjectInternal[]) { for (const project of projects) { - for (const dependencyName of project._deps) { + for (const dependencyName of project.dependencies) { const dependencies = projects.filter(p => p.name === dependencyName); if (!dependencies.length) throw new Error(`Project '${project.name}' depends on unknown project '${dependencyName}'`); if (dependencies.length > 1) throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`); - project._depProjects.push(...dependencies); + project._deps.push(...dependencies); } } } diff --git a/packages/playwright-test/src/common/types.ts b/packages/playwright-test/src/common/types.ts index 1f4d5db087..cc76a038d0 100644 --- a/packages/playwright-test/src/common/types.ts +++ b/packages/playwright-test/src/common/types.ts @@ -72,8 +72,7 @@ export interface FullProjectInternal extends FullProjectPublic { _fullyParallel: boolean; _expect: Project['expect']; _respectGitIgnore: boolean; - _deps: string[]; - _depProjects: FullProjectInternal[]; + _deps: FullProjectInternal[]; snapshotPathTemplate: string; } diff --git a/packages/playwright-test/src/runner/projectUtils.ts b/packages/playwright-test/src/runner/projectUtils.ts index 331c9a541d..ad0cabf792 100644 --- a/packages/playwright-test/src/runner/projectUtils.ts +++ b/packages/playwright-test/src/runner/projectUtils.ts @@ -59,8 +59,8 @@ export function projectsThatAreDependencies(projects: FullProjectInternal[]): Fu } if (result.has(project)) return; - project._depProjects.map(visit.bind(undefined, depth + 1)); - project._depProjects.forEach(dep => result.add(dep)); + project._deps.map(visit.bind(undefined, depth + 1)); + project._deps.forEach(dep => result.add(dep)); }; projects.forEach(visit.bind(undefined, 0)); return [...result]; diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index deeaa5256f..3156728837 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -191,7 +191,7 @@ function createRunTestsTask(): Task { // that depend on the projects that failed previously. const phaseTestGroups: TestGroup[] = []; for (const { project, testGroups } of projects) { - const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p)); + const hasFailedDeps = project._deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps) { phaseTestGroups.push(...testGroups); } else { @@ -211,7 +211,7 @@ function createRunTestsTask(): Task { // projects failed. if (!dispatcher.hasWorkerErrors()) { for (const { project, projectSuite } of projects) { - const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p)); + const hasFailedDeps = project._deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok())) successfulProjects.add(project); } @@ -228,7 +228,7 @@ function buildPhases(projectSuites: Suite[]): Suite[][] { for (const projectSuite of projectSuites) { if (processed.has(projectSuite._projectConfig!)) continue; - if (projectSuite._projectConfig!._depProjects.find(p => !processed.has(p))) + if (projectSuite._projectConfig!._deps.find(p => !processed.has(p))) continue; phase.push(projectSuite); } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index f597c69c6d..d20382ef7b 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -57,25 +57,16 @@ type UseOptions = { [K in keyof WorkerArgs]?: WorkerArgs[K * // Options specific to each project. * projects: [ * { - * name: 'Desktop Chromium', - * use: { - * browserName: 'chromium', - * viewport: { width: 1280, height: 720 }, - * }, + * name: 'chromium', + * use: devices['Desktop Chrome'], * }, * { - * name: 'Desktop Safari', - * use: { - * browserName: 'webkit', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'firefox', + * use: devices['Desktop Firefox'], * }, * { - * name: 'Desktop Firefox', - * use: { - * browserName: 'firefox', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'webkit', + * use: devices['Desktop Safari'], * }, * { * name: 'Mobile Chrome', @@ -144,25 +135,16 @@ export interface Project extends TestProject { * // Options specific to each project. * projects: [ * { - * name: 'Desktop Chromium', - * use: { - * browserName: 'chromium', - * viewport: { width: 1280, height: 720 }, - * }, + * name: 'chromium', + * use: devices['Desktop Chrome'], * }, * { - * name: 'Desktop Safari', - * use: { - * browserName: 'webkit', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'firefox', + * use: devices['Desktop Firefox'], * }, * { - * name: 'Desktop Firefox', - * use: { - * browserName: 'firefox', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'webkit', + * use: devices['Desktop Safari'], * }, * { * name: 'Mobile Chrome', @@ -203,6 +185,42 @@ export interface FullProject { * Project name is visible in the report and during test execution. */ name: string; + /** + * List of projects that need to run before any test in this project runs. Dependencies can be useful for configuring + * the global setup actions in a way that every action is a test. For example: + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'setup', + * testMatch: /global.setup\.ts/, + * dependencies: ['setup'], + * }, + * { + * name: 'chromium', + * use: devices['Desktop Chrome'], + * dependencies: ['setup'], + * }, + * { + * name: 'firefox', + * use: devices['Desktop Firefox'], + * dependencies: ['setup'], + * }, + * { + * name: 'webkit', + * use: devices['Desktop Safari'], + * dependencies: ['setup'], + * }, + * ], + * }); + * ``` + * + */ + dependencies: string[]; /** * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir). @@ -4991,25 +5009,16 @@ export interface TestInfoError { * // Options specific to each project. * projects: [ * { - * name: 'Desktop Chromium', - * use: { - * browserName: 'chromium', - * viewport: { width: 1280, height: 720 }, - * }, + * name: 'chromium', + * use: devices['Desktop Chrome'], * }, * { - * name: 'Desktop Safari', - * use: { - * browserName: 'webkit', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'firefox', + * use: devices['Desktop Firefox'], * }, * { - * name: 'Desktop Firefox', - * use: { - * browserName: 'firefox', - * viewport: { width: 1280, height: 720 }, - * } + * name: 'webkit', + * use: devices['Desktop Safari'], * }, * { * name: 'Mobile Chrome', @@ -5025,6 +5034,43 @@ export interface TestInfoError { * */ interface TestProject { + /** + * List of projects that need to run before any test in this project runs. Dependencies can be useful for configuring + * the global setup actions in a way that every action is a test. For example: + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'setup', + * testMatch: /global.setup\.ts/, + * dependencies: ['setup'], + * }, + * { + * name: 'chromium', + * use: devices['Desktop Chrome'], + * dependencies: ['setup'], + * }, + * { + * name: 'firefox', + * use: devices['Desktop Firefox'], + * dependencies: ['setup'], + * }, + * { + * name: 'webkit', + * use: devices['Desktop Safari'], + * dependencies: ['setup'], + * }, + * ], + * }); + * ``` + * + */ + dependencies?: Array; + /** * Configuration for the `expect` assertion library. * diff --git a/tests/playwright-test/deps.spec.ts b/tests/playwright-test/deps.spec.ts index cce7569d97..6310a68ef4 100644 --- a/tests/playwright-test/deps.spec.ts +++ b/tests/playwright-test/deps.spec.ts @@ -22,8 +22,8 @@ test('should run projects with dependencies', async ({ runInlineTest }) => { module.exports = { projects: [ { name: 'A' }, - { name: 'B', _deps: ['A'] }, - { name: 'C', _deps: ['A'] }, + { name: 'B', dependencies: ['A'] }, + { name: 'C', dependencies: ['A'] }, ], };`, 'test.spec.ts': ` @@ -44,8 +44,8 @@ test('should not run project if dependency failed', async ({ runInlineTest }) => module.exports = { projects: [ { name: 'A' }, - { name: 'B', _deps: ['A'] }, - { name: 'C', _deps: ['B'] }, + { name: 'B', dependencies: ['A'] }, + { name: 'C', dependencies: ['B'] }, ], };`, 'test.spec.ts': ` @@ -71,11 +71,11 @@ test('should not run project if dependency failed (2)', async ({ runInlineTest } module.exports = { projects: [ { name: 'A1' }, - { name: 'A2', _deps: ['A1'] }, - { name: 'A3', _deps: ['A2'] }, + { name: 'A2', dependencies: ['A1'] }, + { name: 'A3', dependencies: ['A2'] }, { name: 'B1' }, - { name: 'B2', _deps: ['B1'] }, - { name: 'B3', _deps: ['B2'] }, + { name: 'B2', dependencies: ['B1'] }, + { name: 'B3', dependencies: ['B2'] }, ], };`, 'test.spec.ts': ` @@ -97,7 +97,7 @@ test('should filter by project list, but run deps', async ({ runInlineTest }) => module.exports = { projects: [ { name: 'A' }, { name: 'B' }, - { name: 'C', _deps: ['A'] }, + { name: 'C', dependencies: ['A'] }, { name: 'D' }, ] }; `, @@ -120,7 +120,7 @@ test('should not filter dependency by file name', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { projects: [ { name: 'A' }, - { name: 'B', _deps: ['A'] }, + { name: 'B', dependencies: ['A'] }, ] }; `, 'one.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, @@ -136,7 +136,7 @@ test('should not filter dependency by only', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { projects: [ { name: 'setup', testMatch: /setup.ts/ }, - { name: 'browser', _deps: ['setup'] }, + { name: 'browser', dependencies: ['setup'] }, ] }; `, 'setup.ts': ` @@ -160,7 +160,7 @@ test('should not filter dependency by only 2', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { projects: [ { name: 'setup', testMatch: /setup.ts/ }, - { name: 'browser', _deps: ['setup'] }, + { name: 'browser', dependencies: ['setup'] }, ] }; `, 'setup.ts': ` @@ -184,7 +184,7 @@ test('should not filter dependency by only 3', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { projects: [ { name: 'setup', testMatch: /setup.*.ts/ }, - { name: 'browser', _deps: ['setup'] }, + { name: 'browser', dependencies: ['setup'] }, ] }; `, 'setup-1.ts': ` @@ -210,7 +210,7 @@ test('should report skipped dependent tests', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { projects: [ { name: 'setup', testMatch: /setup.ts/ }, - { name: 'browser', _deps: ['setup'] }, + { name: 'browser', dependencies: ['setup'] }, ] }; `, 'setup.ts': ` @@ -230,8 +230,8 @@ test('should report circular dependencies', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` module.exports = { projects: [ - { name: 'A', _deps: ['B'] }, - { name: 'B', _deps: ['A'] }, + { name: 'A', dependencies: ['B'] }, + { name: 'B', dependencies: ['A'] }, ] }; `, 'test.spec.ts': `pwt.test('pass', () => {});`, diff --git a/tests/playwright-test/global-scripts.spec.ts b/tests/playwright-test/global-scripts.spec.ts index 8775e4a208..4051170c00 100644 --- a/tests/playwright-test/global-scripts.spec.ts +++ b/tests/playwright-test/global-scripts.spec.ts @@ -76,7 +76,7 @@ test('should work for one project', async ({ runGroups }, testInfo) => { { name: 'p1', testMatch: /.*.test.ts/, - _deps: ['setup'], + dependencies: ['setup'], }, ] };`, @@ -116,12 +116,12 @@ test('should work for several projects', async ({ runGroups }, testInfo) => { { name: 'p1', testMatch: /.*a.test.ts/, - _deps: ['setup'], + dependencies: ['setup'], }, { name: 'p2', testMatch: /.*b.test.ts/, - _deps: ['setup'], + dependencies: ['setup'], }, ] };`, @@ -159,12 +159,12 @@ test('should skip tests if global setup fails', async ({ runGroups }, testInfo) { name: 'p1', testMatch: /.*a.test.ts/, - _deps: ['setup'], + dependencies: ['setup'], }, { name: 'p2', testMatch: /.*b.test.ts/, - _deps: ['setup'], + dependencies: ['setup'], }, ] };`, @@ -200,7 +200,7 @@ test('should run setup in each project shard', async ({ runGroups }, testInfo) = }, { name: 'p1', - _deps: ['setup'], + dependencies: ['setup'], }, ] };`, diff --git a/tests/playwright-test/project-setup.spec.ts b/tests/playwright-test/project-setup.spec.ts deleted file mode 100644 index 70558cdd19..0000000000 --- a/tests/playwright-test/project-setup.spec.ts +++ /dev/null @@ -1,942 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test'; -import path from 'path'; -import { test, expect } from './playwright-test-fixtures'; - -test.fixme(true, 'Restore this'); - -function createConfigWithProjects(names: string[], testInfo: TestInfo, projectTemplates?: { [name: string]: PlaywrightTestProject }): Record { - const config: PlaywrightTestConfig = { - projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })), - }; - const files = {}; - for (const name of names) { - files[`${name}/${name}.spec.ts`] = ` - const { test } = pwt; - test('${name} test', async () => { - await new Promise(f => setTimeout(f, 100)); - });`; - files[`${name}/${name}.setup.ts`] = ` - const { test } = pwt; - test.projectSetup('${name} setup', async () => { - await new Promise(f => setTimeout(f, 100)); - });`; - } - function replacer(key, value) { - if (value instanceof RegExp) - return `RegExp(${value.toString()})`; - else - return value; - } - files['playwright.config.ts'] = ` - import * as path from 'path'; - module.exports = ${JSON.stringify(config, replacer, 2)}; - `.replace(/"RegExp\((.*)\)"/g, '$1'); - return files; -} - -type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; - -function formatTimeline(timeline: Timeline) { - return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n'); -} - -function formatFileNames(timeline: Timeline) { - return timeline.map(e => e.titlePath[2]).join('\n'); -} - -function fileNames(timeline: Timeline) { - const fileNames = Array.from(new Set(timeline.map(({ titlePath }) => { - const name = titlePath[2]; - const index = name.lastIndexOf(path.sep); - if (index === -1) - return name; - return name.slice(index + 1); - })).keys()); - fileNames.sort(); - return fileNames; -} - -function expectFilesRunBefore(timeline: Timeline, before: string[], after: string[]) { - const fileBegin = name => { - const index = timeline.findIndex(({ titlePath }) => titlePath[2] === name); - expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); - return index; - }; - const fileEnd = name => { - // There is no Array.findLastIndex in Node < 18. - let index = -1; - for (index = timeline.length - 1; index >= 0; index--) { - if (timeline[index].titlePath[2] === name) - break; - } - expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); - return index; - }; - - for (const b of before) { - const bEnd = fileEnd(b); - for (const a of after) { - const aBegin = fileBegin(a); - expect(bEnd < aBegin, `'${b}' expected to finish before ${a}, actual order:\n${formatTimeline(timeline)}`).toBeTruthy(); - } - } -} - -test('should work for one project', async ({ runGroups }, testInfo) => { - const projectTemplates = { - 'a': { - setupMatch: ['**/*.setup.ts'] - }, - }; - const configWithFiles = createConfigWithProjects(['a'], testInfo, projectTemplates); - const { exitCode, passed, timeline } = await runGroups(configWithFiles); - expect(exitCode).toBe(0); - expect(passed).toBe(2); - expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.setup.ts > a setup [begin] -a > a${path.sep}a.setup.ts > a setup [end] -a > a${path.sep}a.spec.ts > a test [begin] -a > a${path.sep}a.spec.ts > a test [end]`); -}); - -test('should work for several projects', async ({ runGroups }, testInfo) => { - const projectTemplates = { - 'a': { - setupMatch: ['**/*.setup.ts'] - }, - 'b': { - setupMatch: /.*b.setup.ts/ - }, - 'c': { - setupMatch: '**/c.setup.ts' - }, - }; - const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates); - const { exitCode, passed, timeline } = await runGroups(configWithFiles); - expect(exitCode).toBe(0); - expect(passed).toBe(6); - for (const name of ['a', 'b', 'c']) - expectFilesRunBefore(timeline, [`${name}${path.sep}${name}.setup.ts`], [`${name}${path.sep}${name}.spec.ts`]); -}); - -test('should stop project if setup fails', async ({ runGroups }, testInfo) => { - const projectTemplates = { - 'a': { - setupMatch: ['**/*.setup.ts'] - }, - 'b': { - setupMatch: /.*b.setup.ts/ - }, - }; - const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates); - configWithFiles[`a/a.setup.ts`] = ` - const { test, expect } = pwt; - test.projectSetup('a setup', async () => { - expect(1).toBe(2); - });`; - - const { exitCode, passed, skipped, timeline } = await runGroups(configWithFiles); - expect(exitCode).toBe(1); - expect(passed).toBe(3); - expect(skipped).toBe(1); // 1 test from project 'a' - for (const name of ['a', 'b']) - expectFilesRunBefore(timeline, [`${name}${path.sep}${name}.setup.ts`], [`${name}${path.sep}${name}.spec.ts`]); -}); - -test('should run setup in each project shard', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - `, - 'c.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - }; - - { // Shard 1/2 - const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' }); - expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2'); - expect(fileNames(timeline)).toEqual(['a.test.ts', 'c.setup.ts']); - expectFilesRunBefore(timeline, [`c.setup.ts`], [`a.test.ts`]); - expect(exitCode).toBe(0); - expect(passed).toBe(6); - } - { // Shard 2/2 - const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' }); - expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2'); - expect(fileNames(timeline)).toEqual(['b.test.ts', 'c.setup.ts']); - expectFilesRunBefore(timeline, [`c.setup.ts`], [`b.test.ts`]); - expect(exitCode).toBe(0); - expect(passed).toBe(4); - } -}); - -test('should run setup only for projects that have tests in the shard', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*p1.setup.ts$/, - testMatch: /.*a.test.ts/, - }, - { - name: 'p2', - setupMatch: /.*p2.setup.ts$/, - testMatch: /.*b.test.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - `, - 'p1.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'p2.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup3', async () => { }); - test.projectSetup('setup4', async () => { }); - `, - }; - - { // Shard 1/2 - const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' }); - expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2'); - expect(fileNames(timeline)).toEqual(['a.test.ts', 'p1.setup.ts']); - expectFilesRunBefore(timeline, [`p1.setup.ts`], [`a.test.ts`]); - expect(exitCode).toBe(0); - expect(passed).toBe(6); - } - { // Shard 2/2 - const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' }); - expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2'); - expect(fileNames(timeline)).toEqual(['b.test.ts', 'p2.setup.ts']); - expectFilesRunBefore(timeline, [`p2.setup.ts`], [`b.test.ts`]); - expect(exitCode).toBe(0); - expect(passed).toBe(4); - } -}); - -test('--project only runs setup from that project;', async ({ runGroups }, testInfo) => { - const projectTemplates = { - 'a': { - setupMatch: /.*a.setup.ts/ - }, - 'b': { - setupMatch: /.*b.setup.ts/ - }, - }; - const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates); - const { exitCode, passed, timeline } = await runGroups(configWithFiles, { project: ['a', 'c'] }); - expect(exitCode).toBe(0); - expect(passed).toBe(3); - expect(fileNames(timeline)).toEqual(['a.setup.ts', 'a.spec.ts', 'c.spec.ts']); -}); - -test('same file cannot be a setup and a test in the same project', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*a.test.ts$/, - testMatch: /.*a.test.ts$/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain(`a.test.ts" matches both 'setup' and 'testMatch' filters in project "p1"`); -}); - -test('same file cannot be a setup and a test in different projects', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*a.test.ts$/, - testMatch: /.*noMatch.test.ts$/, - }, - { - name: 'p2', - setupMatch: /.*noMatch.test.ts$/, - testMatch: /.*a.test.ts$/ - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain(`a.test.ts" matches 'setup' filter in project "p1" and 'testMatch' filter in project "p2"`); -}); - -test('list-files should enumerate setup files in same group', async ({ runCommand }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*a..setup.ts$/, - testMatch: /.*a.test.ts$/, - }, - { - name: 'p2', - setupMatch: /.*b.setup.ts$/, - testMatch: /.*b.test.ts$/ - }, - ] - };`, - 'a1.setup.ts': ` - const { test } = pwt; - test.projectSetup('test1', async () => { }); - `, - 'a2.setup.ts': ` - const { test } = pwt; - test.projectSetup('test1', async () => { }); - `, - 'a.test.ts': ` - const { test } = pwt; - test('test2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('test3', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test4', async () => { }); - `, - }; - - const { exitCode, output } = await runCommand(files, ['list-files']); - expect(exitCode).toBe(0); - const json = JSON.parse(output); - expect(json.projects.map(p => p.name)).toEqual(['p1', 'p2']); - expect(json.projects[0].files.map(f => path.basename(f))).toEqual(['a.test.ts', 'a1.setup.ts', 'a2.setup.ts']); - expect(json.projects[1].files.map(f => path.basename(f))).toEqual(['b.setup.ts', 'b.test.ts']); -}); - -test('test --list should enumerate setup tests as regular ones', async ({ runCommand }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*a..setup.ts$/, - testMatch: /.*a.test.ts$/, - }, - { - name: 'p2', - setupMatch: /.*b.setup.ts$/, - testMatch: /.*b.test.ts$/ - }, - ] - };`, - 'a1.setup.ts': ` - const { test } = pwt; - test.projectSetup('test1', async () => { }); - `, - 'a2.setup.ts': ` - const { test } = pwt; - test.projectSetup('test1', async () => { }); - `, - 'a.test.ts': ` - const { test } = pwt; - test('test2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('test3', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test4', async () => { }); - `, - }; - - const { exitCode, output } = await runCommand(files, ['test', '--list']); - expect(exitCode).toBe(0); - expect(output).toContain(`Listing tests: - [p1] › a.test.ts:6:7 › test2 - [p1] › a1.setup.ts:5:12 › test1 - [p1] › a2.setup.ts:5:12 › test1 - [p2] › b.setup.ts:5:12 › test3 - [p2] › b.test.ts:6:7 › test4 -Total: 5 tests in 5 files`); -}); - -test('should allow .only in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup.only('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - test.projectSetup.only('setup3', async () => { }); - `, - }; - - const { exitCode, passed, timeline, output } = await runGroups(files); - expect(output).toContain('Running 2 tests using 1 worker'); - expect(output).toContain('[p1] › a.setup.ts:5:25 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:7:25 › setup3'); - expect(fileNames(timeline)).toEqual(['a.setup.ts']); - expect(exitCode).toBe(0); - expect(passed).toBe(2); -}); - -test('should allow describe.only in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.describe.only('main', () => { - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - }); - test.projectSetup('setup3', async () => { }); - `, - }; - - const { exitCode, passed, timeline, output } = await runGroups(files); - expect(output).toContain('Running 2 tests using 1 worker'); - expect(output).toContain('[p1] › a.setup.ts:6:14 › main › setup1'); - expect(output).toContain('[p1] › a.setup.ts:7:14 › main › setup2'); - expect(fileNames(timeline)).toEqual(['a.setup.ts']); - expect(exitCode).toBe(0); - expect(passed).toBe(2); -}); - -test('should filter describe line in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.describe('main', () => { - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - }); - test.projectSetup('setup3', async () => { }); - `, - }; - - const { exitCode, passed, timeline, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['a.setup.ts:5'] }); - expect(output).toContain('Running 2 tests using 1 worker'); - expect(output).toContain('[p1] › a.setup.ts:6:14 › main › setup1'); - expect(output).toContain('[p1] › a.setup.ts:7:14 › main › setup2'); - expect(fileNames(timeline)).toEqual(['a.setup.ts']); - expect(exitCode).toBe(0); - expect(passed).toBe(2); -}); - -test('should allow .only in both setup and test files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test.only('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup.only('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - test.projectSetup('setup3', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(0); - expect(output).toContain('[p1] › a.setup.ts:5:25 › setup1'); - expect(output).toContain('[p1] › a.test.ts:7:12 › test2'); -}); - -test('should run full setup when there is test.only', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test.only('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - test('test4', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup3', async () => { }); - `, - }; - - const { exitCode, passed, output } = await runGroups(files); - expect(exitCode).toBe(0); - expect(passed).toBe(4); - expect(output).toContain('Running 4 tests using 2 workers'); - expect(output).toContain('[p1] › b.setup.ts:5:12 › setup3'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p1] › a.test.ts:6:12 › test1'); -}); - -test('should allow filtering setup by file:line', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*a.setup.ts/, - }, - { - name: 'p2', - setupMatch: /.*b.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - `, - }; - - { - const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*setup.ts$'] }); - expect(output).toContain('Running 3 tests using 2 workers'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p2] › b.setup.ts:5:12 › setup1'); - expect(exitCode).toBe(0); - expect(passed).toBe(3); - } - { - const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.setup.ts:5'] }); - expect(output).toContain('Running 1 test using 1 worker'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(exitCode).toBe(0); - expect(passed).toBe(1); - } -}); - -test('should support filters matching both setup and test', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.(setup|test).ts$'] }); - expect(exitCode).toBe(0); - expect(output).toContain('Running 5 tests using 1 worker'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p1] › a.test.ts:6:7 › test1'); - expect(output).toContain('[p1] › a.test.ts:7:7 › test2'); - expect(output).toContain('[p1] › a.test.ts:8:7 › test3'); -}); - -test('should run setup for a project if tests match only in another project', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - testMatch: /.*a.test.ts/, - setupMatch: /.*a.setup.ts/, - }, - { - name: 'p2', - testMatch: /.*b.test.ts/, - setupMatch: /.*b.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - 'b.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.test.ts$'] }); - expect(exitCode).toBe(0); - expect(output).toContain('Running 3 tests using 2 workers'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.test.ts:6:7 › test1'); - expect(output).toContain('[p2] › b.setup.ts:5:12 › setup1'); -}); - -test('should run all setup files if only tests match filter', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['a.test.ts:7'] }); - expect(exitCode).toBe(0); - expect(output).toContain('Running 4 tests using 2 workers'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p1] › b.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.test.ts:7:7 › test2'); -}); - -test('should run all setup files if only tests match grep filter', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('test3', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['--grep', '.*test2$'] }); - expect(exitCode).toBe(0); - expect(output).toContain('Running 4 tests using 2 workers'); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p1] › b.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.test.ts:7:7 › test2'); -}); - -test('should apply project.grep filter to both setup and tests', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - grep: /a.(test|setup).ts.*(test|setup)/, - }, - ] - };`, - 'a.test.ts': ` - const { test } = pwt; - test('test1', async () => { }); - test('test2', async () => { }); - test('foo', async () => { }); - `, - 'a.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('setup2', async () => { }); - `, - 'b.setup.ts': ` - const { test } = pwt; - test.projectSetup('setup1', async () => { }); - test.projectSetup('foo', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(0); - expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1'); - expect(output).toContain('[p1] › a.setup.ts:6:12 › setup2'); - expect(output).toContain('[p1] › a.test.ts:6:7 › test1'); - expect(output).toContain('[p1] › a.test.ts:7:7 › test2'); -}); - -test('should prohibit setup in test files', async ({ runGroups }, testInfo) => { - const files = { - 'a.test.ts': ` - const { test } = pwt; - test.projectSetup('test1', async () => { }); - test('test2', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain('test.projectSetup() is only allowed in a project setup file.'); -}); - -test('should prohibit beforeAll hooks in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.setup.ts': ` - const { test } = pwt; - test.beforeAll(async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain('test.beforeAll() is not allowed in a setup file'); -}); - -test('should prohibit test in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.setup.ts': ` - const { test } = pwt; - test('test1', async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain('test() is not allowed in a setup file'); -}); - -test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo) => { - const files = { - 'playwright.config.ts': ` - module.exports = { - projects: [ - { - name: 'p1', - setupMatch: /.*.setup.ts/, - }, - ] - };`, - 'a.setup.ts': ` - const { test } = pwt; - test.beforeEach(async () => { }); - `, - }; - - const { exitCode, output } = await runGroups(files); - expect(exitCode).toBe(1); - expect(output).toContain('test.beforeEach() is not allowed in a setup file'); -}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index b145a04532..756728c30f 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -42,6 +42,7 @@ export interface FullProject { grepInvert: RegExp | RegExp[] | null; metadata: Metadata; name: string; + dependencies: string[]; snapshotDir: string; outputDir: string; repeatEach: number;