microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-10-04 16:37:49 +03:00
Code Issues Projects Releases Wiki Activity

feat(runner): project execution schedule (#17548)

Browse Source
This commit is contained in:
Yury Semikhatsky 2022-09-23 20:01:27 -07:00 committed by GitHub
parent 080cf29191
commit 10d7c60abf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 596 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/src/test-api/class-testconfig.md
View File

@ -232,6 +232,13 @@ Filter to only run tests with a title **not** matching one of the patterns. This
`grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests).
## property: TestConfig.groups
* since: v1.27
- type: ?<[Object]<[string],[Array]<[string]|[Array]<[string]|[Object]>>>>
- `project` <[string]|[Array]<[string]>> Project name(s).
Project groups that control project execution order.
## property: TestConfig.ignoreSnapshots
* since: v1.26
- type: ?<[boolean]>

2
packages/playwright-test/src/cli.ts
View File

@ -58,6 +58,7 @@ function addTestCommand(program: Command) {
command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: run all projects)`);
command.option('--group <project-group-name>', `Only run tests from the specified project group (default: run all projects from the 'default' group or just all projects if 'default' group is not defined).`);
command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
command.option('--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`);
command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
@ -166,6 +167,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
listOnly: !!opts.list,
testFileFilters,
projectFilter: opts.project || undefined,
projectGroup: opts.group,
watchMode: !!process.env.PW_TEST_WATCH,
passWithNoTests: opts.passWithNoTests,
});

55
packages/playwright-test/src/loader.ts
View File

@ -27,7 +27,7 @@ import * as os from 'os';
import type { BuiltInReporter, ConfigCLIOverrides } from './runner';
import type { Reporter } from '../types/testReporter';
import { builtInReporters } from './runner';
import { isRegExp, calculateSha1 } from 'playwright-core/lib/utils';
import { isRegExp, calculateSha1, isString, isObject } from 'playwright-core/lib/utils';
import { serializeError } from './util';
import { FixturePool, isFixtureOption } from './fixtures';
import type { TestTypeImpl } from './testType';
@ -167,6 +167,7 @@ export class Loader {
this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
this._assignUniqueProjectIds(this._fullConfig.projects);
this._fullConfig.groups = config.groups;
}
private _assignUniqueProjectIds(projects: FullProjectInternal[]) {
@ -538,6 +539,8 @@ function validateConfig(file: string, config: Config) {
});
}
validateProjectGroups(file, config);
if ('quiet' in config && config.quiet !== undefined) {
if (typeof config.quiet !== 'boolean')
throw errorWithFile(file, `config.quiet must be a boolean`);
@ -644,6 +647,54 @@ function validateProject(file: string, project: Project, title: string) {
}
}
function validateProjectGroups(file: string, config: Config) {
if (config.groups === undefined)
return;
const projectNames = new Set(config.projects?.filter(p => !!p.name).map(p => p.name));
for (const [groupName, group] of Object.entries(config.groups)) {
function validateProjectReference(projectName: string) {
if (projectName.trim() === '')
throw errorWithFile(file, `config.groups.${groupName} refers to an empty project name`);
if (!projectNames.has(projectName))
throw errorWithFile(file, `config.groups.${groupName} refers to an unknown project '${projectName}'`);
}
for (const step of group) {
if (isString(step)) {
validateProjectReference(step);
} else if (Array.isArray(step)) {
const parallelProjectNames = new Set();
for (const item of step) {
let projectName;
if (isString(item)) {
validateProjectReference(item);
projectName = item;
} else if (isObject(item)) {
const project = (item as any).project;
if (isString(project)) {
validateProjectReference(project);
} else if (Array.isArray(project)) {
project.forEach(name => {
if (!isString(name))
throw errorWithFile(file, `config.groups.${groupName}[*].project contains non string value.`);
validateProjectReference(name);
});
}
projectName = project;
} else {
throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
}
// We can relax this later.
if (parallelProjectNames.has(projectName))
throw errorWithFile(file, `config.groups.${groupName} group mentions project '${projectName}' twice in one parallel group`);
parallelProjectNames.add(projectName);
}
} else {
throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
}
}
}
}
export const baseFullConfig: FullConfigInternal = {
forbidOnly: false,
fullyParallel: false,
@ -670,7 +721,7 @@ export const baseFullConfig: FullConfigInternal = {
_webServers: [],
_globalOutputDir: path.resolve(process.cwd()),
_configDir: '',
_testGroupsCount: 0,
_maxConcurrentTestGroups: 0,
_ignoreSnapshots: false,
_workerIsolation: 'isolate-pools',
};

2
packages/playwright-test/src/reporters/base.ts
View File

@ -121,7 +121,7 @@ export class BaseReporter implements ReporterInternal {
}
protected generateStartingMessage() {
const jobs = Math.min(this.config.workers, this.config._testGroupsCount);
const jobs = Math.min(this.config.workers, this.config._maxConcurrentTestGroups);
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
if (this.config._watchMode)
return `\nRunning tests in the --watch mode`;

253
packages/playwright-test/src/runner.ts
View File

@ -47,16 +47,24 @@ import { setRunnerToAddPluginsTo } from './plugins';
import { webServerPluginsForConfig } from './plugins/webServerPlugin';
import { dockerPlugin } from './plugins/dockerPlugin';
import { MultiMap } from 'playwright-core/lib/utils/multimap';
import { isString, assert } from 'playwright-core/lib/utils';
const removeFolderAsync = promisify(rimraf);
const readDirAsync = promisify(fs.readdir);
const readFileAsync = promisify(fs.readFile);
export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
// Project group is a sequence of run phases.
type RunPhase = {
testFileFilters: TestFileFilter[];
projectFilter?: string[];
};
type RunOptions = {
listOnly?: boolean;
testFileFilters?: TestFileFilter[];
projectFilter?: string[];
projectGroup?: string;
watchMode?: boolean;
passWithNoTests?: boolean;
};
@ -227,6 +235,58 @@ export class Runner {
return report;
}
private _collectRunPhases(options: RunOptions) {
const config = this._loader.fullConfig();
let projectGroup = options.projectGroup;
if (options.projectFilter) {
if (projectGroup)
throw new Error('--group option can not be combined with --project');
} else {
if (!projectGroup && config.groups?.default && !options.testFileFilters?.length)
projectGroup = 'default';
if (projectGroup) {
if (config.shard)
throw new Error(`Project group '${projectGroup}' cannot be combined with --shard`);
}
}
const phases: RunPhase[] = [];
if (projectGroup) {
const group = config.groups?.[projectGroup];
if (!group)
throw new Error(`Cannot find project group '${projectGroup}' in the config`);
for (const entry of group) {
const projectFilter: string[] = [];
const testFileFilters: TestFileFilter[] = [];
if (isString(entry)) {
projectFilter.push(entry);
} else {
for (const p of entry) {
if (isString(p))
projectFilter.push(p);
else if (isString(p.project))
projectFilter.push(p.project);
else
projectFilter.push(...p.project);
}
}
// TODO: filter per project set.
phases.push({
testFileFilters,
projectFilter
});
}
} else {
phases.push({
projectFilter: options.projectFilter,
testFileFilters: options.testFileFilters || [],
});
}
return phases;
}
private async _collectFiles(testFileFilters: TestFileFilter[], projectNames?: string[]): Promise<Map<FullProjectInternal, string[]>> {
const testFileFilter = testFileFilters.length ? createFileMatcher(testFileFilters.map(e => e.re || e.exact || '')) : () => true;
let projectsToFind: Set<string> | undefined;
@ -270,74 +330,86 @@ export class Runner {
}
private async _run(options: RunOptions): Promise<FullResult> {
const testFileFilters = options.testFileFilters || [];
const filesByProject = await this._collectFiles(testFileFilters, options.projectFilter);
const allTestFiles = new Set<string>();
for (const files of filesByProject.values())
files.forEach(file => allTestFiles.add(file));
const config = this._loader.fullConfig();
const fatalErrors: TestError[] = [];
// 1. Add all tests.
const preprocessRoot = new Suite('', 'root');
for (const file of allTestFiles) {
const fileSuite = await this._loader.loadTestFile(file, 'runner');
if (fileSuite._loadError)
fatalErrors.push(fileSuite._loadError);
preprocessRoot._addSuite(fileSuite);
}
// 2. Complain about duplicate titles.
const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot);
if (duplicateTitlesError)
fatalErrors.push(duplicateTitlesError);
// 3. Filter tests to respect line/column filter.
filterByFocusedLine(preprocessRoot, testFileFilters);
// 4. Complain about only.
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites));
}
// 5. Filter only.
if (!options.listOnly)
filterOnly(preprocessRoot);
// 6. Generate projects.
const fileSuites = new Map<string, Suite>();
for (const fileSuite of preprocessRoot.suites)
fileSuites.set(fileSuite._requireFile, fileSuite);
// Each entry is an array of test groups that can be run concurrently. All
// test groups from the previos entries must finish before entry starts.
const concurrentTestGroups = [];
const rootSuite = new Suite('', 'root');
for (const [project, files] of filesByProject) {
const grepMatcher = createTitleMatcher(project.grep);
const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
const projectSuite = new Suite(project.name, 'project');
projectSuite._projectConfig = project;
if (project._fullyParallel)
projectSuite._parallelMode = 'parallel';
rootSuite._addSuite(projectSuite);
for (const file of files) {
const fileSuite = fileSuites.get(file);
if (!fileSuite)
continue;
for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => {
const grepTitle = test.titlePath().join(' ');
if (grepInvertMatcher?.(grepTitle))
return false;
return grepMatcher(grepTitle);
});
if (builtSuite)
projectSuite._addSuite(builtSuite);
const runPhases = this._collectRunPhases(options);
assert(runPhases.length > 0);
for (const { projectFilter, testFileFilters } of runPhases) {
// TODO: do not collect files for each project multiple times.
const filesByProject = await this._collectFiles(testFileFilters, projectFilter);
const allTestFiles = new Set<string>();
for (const files of filesByProject.values())
files.forEach(file => allTestFiles.add(file));
// 1. Add all tests.
const preprocessRoot = new Suite('', 'root');
for (const file of allTestFiles) {
const fileSuite = await this._loader.loadTestFile(file, 'runner');
if (fileSuite._loadError)
fatalErrors.push(fileSuite._loadError);
preprocessRoot._addSuite(fileSuite);
}
// 2. Complain about duplicate titles.
const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot);
if (duplicateTitlesError)
fatalErrors.push(duplicateTitlesError);
// 3. Filter tests to respect line/column filter.
// TODO: figure out how this is supposed to work with groups.
filterByFocusedLine(preprocessRoot, testFileFilters);
// 4. Complain about only.
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites));
}
// 5. Filter only.
if (!options.listOnly)
filterOnly(preprocessRoot);
// 6. Generate projects.
const fileSuites = new Map<string, Suite>();
for (const fileSuite of preprocessRoot.suites)
fileSuites.set(fileSuite._requireFile, fileSuite);
const firstProjectSuiteIndex = rootSuite.suites.length;
for (const [project, files] of filesByProject) {
const grepMatcher = createTitleMatcher(project.grep);
const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
const projectSuite = new Suite(project.name, 'project');
projectSuite._projectConfig = project;
if (project._fullyParallel)
projectSuite._parallelMode = 'parallel';
rootSuite._addSuite(projectSuite);
for (const file of files) {
const fileSuite = fileSuites.get(file);
if (!fileSuite)
continue;
for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => {
const grepTitle = test.titlePath().join(' ');
if (grepInvertMatcher?.(grepTitle))
return false;
return grepMatcher(grepTitle);
});
if (builtSuite)
projectSuite._addSuite(builtSuite);
}
}
}
const projectSuites = rootSuite.suites.slice(firstProjectSuiteIndex);
const testGroups = createTestGroups(projectSuites, config.workers);
concurrentTestGroups.push(testGroups);
}
// 7. Fail when no tests.
@ -346,10 +418,10 @@ export class Runner {
fatalErrors.push(createNoTestsError());
// 8. Compute shards.
let testGroups = createTestGroups(rootSuite, config.workers);
const shard = config.shard;
if (shard) {
assert(!options.projectGroup);
assert(concurrentTestGroups.length === 1);
const shardGroups: TestGroup[] = [];
const shardTests = new Set<TestCase>();
@ -362,7 +434,7 @@ export class Runner {
const from = shardSize * currentShard + Math.min(extraOne, currentShard);
const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
let current = 0;
for (const group of testGroups) {
for (const group of concurrentTestGroups[0]) {
// Any test group goes to the shard that contains the first test of this group.
// So, this shard gets any group that starts at [from; to)
if (current >= from && current < to) {
@ -373,11 +445,12 @@ export class Runner {
current += group.tests.length;
}
testGroups = shardGroups;
concurrentTestGroups[0] = shardGroups;
filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
total = rootSuite.allTests().length;
}
config._testGroupsCount = testGroups.length;
config._maxConcurrentTestGroups = Math.max(...concurrentTestGroups.map(g => g.length));
// 9. Report begin
this._reporter.onBegin?.(config, rootSuite);
@ -399,7 +472,7 @@ export class Runner {
// 13. Run Global setup.
const result: FullResult = { status: 'passed' };
const globalTearDown = await this._performGlobalSetup(config, rootSuite, [...filesByProject.keys()], result);
const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
if (result.status !== 'passed')
return result;
@ -414,24 +487,32 @@ export class Runner {
// 14. Run tests.
try {
const sigintWatcher = new SigIntWatcher();
let sigintWatcher;
let hasWorkerErrors = false;
const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter);
await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
if (!sigintWatcher.hadSignal()) {
// We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
// as soon as we can.
sigintWatcher.disarm();
for (const testGroups of concurrentTestGroups) {
const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter);
sigintWatcher = new SigIntWatcher();
await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
if (!sigintWatcher.hadSignal()) {
// We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
// as soon as we can.
sigintWatcher.disarm();
}
await dispatcher.stop();
hasWorkerErrors = dispatcher.hasWorkerErrors();
if (hasWorkerErrors)
break;
if (testGroups.some(testGroup => testGroup.tests.some(test => !test.ok())))
break;
if (sigintWatcher.hadSignal())
break;
}
await dispatcher.stop();
hasWorkerErrors = dispatcher.hasWorkerErrors();
if (!sigintWatcher.hadSignal()) {
if (sigintWatcher?.hadSignal()) {
result.status = 'interrupted';
} else {
const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok());
result.status = failed ? 'failed' : 'passed';
} else {
result.status = 'interrupted';
}
} catch (e) {
this._reporter.onError?.(serializeError(e));
@ -457,7 +538,7 @@ export class Runner {
// 4. Run Global setup.
const result: FullResult = { status: 'passed' };
const globalTearDown = await this._performGlobalSetup(config, rootSuite, config.projects.filter(p => !options.projectFilter || options.projectFilter.includes(p.name)), result);
const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
if (result.status !== 'passed')
return result;
@ -590,7 +671,7 @@ export class Runner {
return true;
}
private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, projects: FullProjectInternal[], result: FullResult): Promise<(() => Promise<void>) | undefined> {
private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise<void>) | undefined> {
let globalSetupResult: any = undefined;
const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
@ -805,7 +886,7 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) {
return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`;
}
function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] {
function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] {
// This function groups tests that can be run together.
// Tests cannot be run together when:
// - They belong to different projects - requires different workers.
@ -843,7 +924,7 @@ function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] {
};
};
for (const projectSuite of rootSuite.suites) {
for (const projectSuite of projectSuites) {
for (const test of projectSuite.allTests()) {
let withWorkerHash = groups.get(test._workerHash);
if (!withWorkerHash) {

10
packages/playwright-test/src/types.ts
View File

@ -44,7 +44,7 @@ export interface TestStepInternal {
export interface FullConfigInternal extends FullConfigPublic {
_globalOutputDir: string;
_configDir: string;
_testGroupsCount: number;
_maxConcurrentTestGroups: number;
_watchMode: boolean;
_ignoreSnapshots: boolean;
_workerIsolation: WorkerIsolation;
@ -56,6 +56,14 @@ export interface FullConfigInternal extends FullConfigPublic {
// Overrides the public field.
projects: FullProjectInternal[];
groups?: { [key: string]: Array<string | Array<string | {
project: string | string[],
grep?: RegExp | RegExp[],
grepInvert?: RegExp | RegExp[],
testMatch?: string | RegExp | Array<string | RegExp>,
testIgnore?: string | RegExp | Array<string | RegExp>
}>> };
}
/**

10
packages/playwright-test/types/test.d.ts vendored
View File

@ -687,6 +687,16 @@ interface TestConfig {
*/
grepInvert?: RegExp|Array<RegExp>;
/**
* Project groups that control project execution order.
*/
groups?: { [key: string]: Array<string|Array<string|{
/**
* Project name(s).
*/
project: string|Array<string>;
}>>; };
/**
* Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await
* expect(page).toHaveScreenshot()`.

48
tests/playwright-test/config.spec.ts
View File

@ -480,3 +480,51 @@ test('should have correct types for the config', async ({ runTSC }) => {
});
expect(result.exitCode).toBe(0);
});
test('should throw when group has duplicate project references', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'a' },
],
groups: {
default: [
['a', 'a']
]
}
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async () => {});
`
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`config.groups.default group mentions project 'a' twice in one parallel group`);
});
test('should throw when group has unknown project reference', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'a' },
],
groups: {
default: [
[{project: 'b'}]
]
}
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async () => {});
`
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`config.groups.default refers to an unknown project 'b'`);
});

260
tests/playwright-test/groups.spec.ts Normal file
View File

@ -0,0 +1,260 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { PlaywrightTestConfig, TestInfo } from '@playwright/test';
import path from 'path';
import { test, expect } from './playwright-test-fixtures';
function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups']): Record<string, string> {
const config: PlaywrightTestConfig = {
projects: names.map(name => ({ name, testDir: testInfo.outputPath(name) })),
groups
};
const files = {};
for (const name of names) {
files[`${name}/${name}.spec.ts`] = `
const { test } = pwt;
test('${name} test', async () => {
await new Promise(f => setTimeout(f, 100));
});`;
}
files['playwright.config.ts'] = `
import * as path from 'path';
module.exports = ${JSON.stringify(config)};
`;
return files;
}
type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
function formatTimeline(timeline: Timeline) {
return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
}
function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
const begin = new Map<string, number>();
const end = new Map<string, number>();
for (let i = 0; i < timeline.length; i++) {
const projectName = timeline[i].titlePath[1];
const map = timeline[i].event === 'begin' ? begin : end;
const oldIndex = map.get(projectName) ?? i;
const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex);
map.set(projectName, newIndex);
}
for (const b of before) {
for (const a of after) {
const bEnd = end.get(b) as number;
expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy();
const aBegin = begin.get(a) as number;
expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy();
if (bEnd < aBegin)
continue;
throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`);
}
}
}
test('should work', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: ['a']
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(1);
expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
a > a${path.sep}a.spec.ts > a test [end]`);
});
test('should order two projects', async ({ runGroups }, testInfo) => {
await test.step(`order a then b`, async () => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'a',
'b'
]
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
a > a${path.sep}a.spec.ts > a test [end]
b > b${path.sep}b.spec.ts > b test [begin]
b > b${path.sep}b.spec.ts > b test [end]`);
});
await test.step(`order b then a`, async () => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'b',
'a'
]
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin]
b > b${path.sep}b.spec.ts > b test [end]
a > a${path.sep}a.spec.ts > a test [begin]
a > a${path.sep}a.spec.ts > a test [end]`);
});
});
test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'e',
['d', 'c', 'b'],
'a',
]
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expectRunBefore(timeline, ['e'], ['d', 'c', 'b']);
expectRunBefore(timeline, ['d', 'c', 'b'], ['a']);
expect(passed).toBe(5);
});
test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
['a', 'b'],
['d', 'c'],
['e', 'f'],
]
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expectRunBefore(timeline, ['a', 'b'], ['c', 'd']);
expectRunBefore(timeline, ['c', 'd'], ['e', 'f']);
expect(passed).toBe(6);
});
test('should run parallel groups sequentially without overlaps', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
['a', 'b', 'c', 'd'],
['a', 'b', 'c', 'd'],
['a', 'b', 'c', 'd'],
]
});
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
const expectedEndOfFirstPhase = events => {
const firstProjectEndIndex = project => events.findIndex(e => e.event === 'end' && e.titlePath[1] === project);
return Math.max(...['a', 'b', 'c', 'd'].map(firstProjectEndIndex));
};
const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
let remainingTimeline = timeline;
for (let i = 0; i < 3; i++) {
const phaseEndIndex = expectedEndOfFirstPhase(remainingTimeline);
const firstPhase = formatPhaseEvents(remainingTimeline.slice(0, phaseEndIndex + 1));
firstPhase.sort();
expect(firstPhase, `check phase ${i}`).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end', 'c:begin', 'c:end', 'd:begin', 'd:end']);
remainingTimeline = remainingTimeline.slice(phaseEndIndex + 1);
}
expect(remainingTimeline.length).toBe(0);
expect(passed).toBe(12);
});
test('should support phase with multiple project names', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
[
{ project: ['a', 'b', 'c'] }
],
[
{ project: ['d'] },
{ project: ['e', 'f'] }
],
]
});
const { exitCode, passed } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(6);
});
test('should support varios syntax', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'a',
['a', 'b'],
[
{ project: ['a', 'b'] }
],
[
{ project: ['a', 'b'] },
'c',
{ project: 'd' },
],
[{ project: 'e' }],
'f'
]
});
const { exitCode, passed } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(11);
});
test('should support --group option', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'a', 'b'
],
foo: [
['b', 'c']
],
bar: [
'd', 'e'
]
});
const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
{
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'default' });
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(formatPhaseEvents(timeline)).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end']);
}
{
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'foo' });
expect(exitCode).toBe(0);
expect(passed).toBe(2);
const formatted = formatPhaseEvents(timeline);
formatted.sort();
expect(formatted).toEqual(['b:begin', 'b:end', 'c:begin', 'c:end']);
}
{
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'bar' });
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(formatPhaseEvents(timeline)).toEqual(['d:begin', 'd:end', 'e:begin', 'e:end']);
}
});
test('should throw when unknown --group is passed', async ({ runGroups }, testInfo) => {
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
default: [
'a', 'b'
],
foo: [
['b', 'c']
]
});
const { exitCode, output } = await runGroups(configWithFiles, { group: 'bar' });
expect(exitCode).toBe(1);
expect(output).toContain(`Cannot find project group 'bar' in the config`);
});

40
tests/playwright-test/playwright-test-fixtures.ts
View File

@ -224,7 +224,8 @@ type Fixtures = {
writeFiles: (files: Files) => Promise<string>;
runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise<void>) => Promise<RunResult>;
runTSC: (files: Files) => Promise<TSCResult>;
nodeVersion: { major: number, minor: number, patch: number },
nodeVersion: { major: number, minor: number, patch: number };
runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
};
export const test = base
@ -261,6 +262,43 @@ export const test = base
const [major, minor, patch] = process.versions.node.split('.');
await use({ major: +major, minor: +minor, patch: +patch });
},
runGroups: async ({ runInlineTest }, use, testInfo) => {
const timelinePath = testInfo.outputPath('timeline.json');
await use(async (files, params, env, options) => {
const result = await runInlineTest({
...files,
'reporter.ts': `
import { Reporter, TestCase } from '@playwright/test/reporter';
import fs from 'fs';
import path from 'path';
class TimelineReporter implements Reporter {
private _timeline: {titlePath: string, event: 'begin' | 'end'}[] = [];
onTestBegin(test: TestCase) {
this._timeline.push({ titlePath: test.titlePath(), event: 'begin' });
}
onTestEnd(test: TestCase) {
this._timeline.push({ titlePath: test.titlePath(), event: 'end' });
}
onEnd() {
fs.writeFileSync(path.join(${JSON.stringify(timelinePath)}), JSON.stringify(this._timeline, null, 2));
}
}
export default TimelineReporter;
`
}, { ...params, reporter: 'list,json,./reporter.ts', workers: 2 }, env, options);
let timeline;
try {
timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8'));
} catch (e) {
}
return {
...result,
timeline
};
});
},
});
const TSCONFIG = {