feat(runner): allow filtering setup and tests (#18900)

Running `npx playwright test file:123` will have the following behavior
- if only test files match then only matching subset of tests will run
but all setup files will run as well
- if only setup files match the filter then only those setup tests will
run
- if both setup and test files match an error will be thrown
This commit is contained in:
Yury Semikhatsky 2022-11-18 11:35:29 -08:00 committed by GitHub
parent 7c5e4241b1
commit c0d0f54a12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 194 additions and 9 deletions

View File

@ -344,7 +344,7 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result:
return errorDetails;
}
function relativeFilePath(config: FullConfig, file: string): string {
export function relativeFilePath(config: FullConfig, file: string): string {
return path.relative(config.rootDir, file) || path.basename(file);
}

View File

@ -29,7 +29,7 @@ import type { TestRunnerPlugin } from './plugins';
import { setRunnerToAddPluginsTo } from './plugins';
import { dockerPlugin } from './plugins/dockerPlugin';
import { webServerPluginsForConfig } from './plugins/webServerPlugin';
import { formatError } from './reporters/base';
import { formatError, relativeFilePath } from './reporters/base';
import DotReporter from './reporters/dot';
import EmptyReporter from './reporters/empty';
import GitHubReporter from './reporters/github';
@ -196,7 +196,7 @@ export class Runner {
async listTestFiles(projectNames: string[] | undefined): Promise<any> {
const projects = this._collectProjects(projectNames);
const { filesByProject } = await this._collectFiles(projects, () => true);
const { filesByProject } = await this._collectFiles(projects, []);
const report: any = {
projects: []
};
@ -235,12 +235,13 @@ export class Runner {
return projects;
}
private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>}> {
private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>, applyFilterToSetup: boolean}> {
const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
const testFileExtension = (file: string) => extensions.includes(path.extname(file));
const filesByProject = new Map<FullProjectInternal, string[]>();
const setupFiles = new Set<string>();
const fileToProjectName = new Map<string, string>();
const commandLineFileMatcher = fileMatcherFrom(commandLineFileFilters);
for (const project of projects) {
const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
const setupMatch = createFileMatcher(project._setup);
@ -250,7 +251,7 @@ export class Runner {
if (!testFileExtension(file))
return false;
const isSetup = setupMatch(file);
const isTest = !testIgnore(file) && testMatch(file) && testFileFilter(file);
const isTest = !testIgnore(file) && testMatch(file) && commandLineFileMatcher(file);
if (!isTest && !isSetup)
return false;
if (isSetup && isTest)
@ -270,13 +271,40 @@ export class Runner {
});
filesByProject.set(project, testFiles);
}
return { filesByProject, setupFiles };
// If the filter didn't match any tests, apply it to the setup files.
const applyFilterToSetup = setupFiles.size === fileToProjectName.size;
if (applyFilterToSetup) {
// We now have only setup files in filesByProject, because all test files were filtered out.
for (const [project, setupFiles] of filesByProject) {
const filteredFiles = setupFiles.filter(commandLineFileMatcher);
if (filteredFiles.length)
filesByProject.set(project, filteredFiles);
else
filesByProject.delete(project);
}
for (const file of setupFiles) {
if (!commandLineFileMatcher(file))
setupFiles.delete(file);
}
} else if (commandLineFileFilters.length) {
const setupFile = [...setupFiles].find(commandLineFileMatcher);
// If the filter is not empty and it matches both setup and tests then it's an error: we allow
// to run either subset of tests with full setup or partial setup without any tests.
if (setupFile) {
const testFile = Array.from(fileToProjectName.keys()).find(f => !setupFiles.has(f));
const config = this._loader.fullConfig();
throw new Error(`Both setup and test files match command line filter.\n Setup file: ${relativeFilePath(config, setupFile)}\n Test file: ${relativeFilePath(config, testFile!)}`);
}
}
return { filesByProject, setupFiles, applyFilterToSetup };
}
private async _collectTestGroups(options: RunOptions, fatalErrors: TestError[]): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
const config = this._loader.fullConfig();
const projects = this._collectProjects(options.projectFilter);
const { filesByProject, setupFiles } = await this._collectFiles(projects, fileMatcherFrom(options.testFileFilters));
const { filesByProject, setupFiles, applyFilterToSetup } = await this._collectFiles(projects, options.testFileFilters);
const allTestFiles = new Set<string>();
for (const files of filesByProject.values())
@ -298,7 +326,7 @@ export class Runner {
// Filter tests to respect line/column filter.
if (options.testFileFilters.length)
filterByFocusedLine(preprocessRoot, options.testFileFilters, setupFiles);
filterByFocusedLine(preprocessRoot, options.testFileFilters, applyFilterToSetup ? new Set() : setupFiles);
// Complain about only.
// TODO: check in project setup.

View File

@ -316,7 +316,7 @@ export const test = base
let timeline;
try {
timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8'));
timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString());
} catch (e) {
}
return {

View File

@ -425,3 +425,160 @@ test('test --list should enumerate setup tests as regular ones', async ({ runCom
[p2] b.test.ts:6:7 test4
Total: 5 tests in 5 files`);
});
test('should allow .only in setup files', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*.setup.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
test('test4', async () => { });
`,
'a.setup.ts': `
const { test } = pwt;
test.only('setup1', async () => { });
test('setup2', async () => { });
test.only('setup3', async () => { });
`,
};
const { exitCode, passed, timeline, output } = await runGroups(files);
expect(output).toContain('Running 2 tests using 1 worker');
expect(output).toContain('[p1] a.setup.ts:5:12 setup1');
expect(output).toContain('[p1] a.setup.ts:7:12 setup3');
expect(fileNames(timeline)).toEqual(['a.setup.ts']);
expect(exitCode).toBe(0);
expect(passed).toBe(2);
});
test('should allow filtering setup by file:line', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*a.setup.ts/,
},
{
name: 'p2',
setup: /.*b.setup.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
`,
'a.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
test('setup2', async () => { });
`,
'b.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
`,
'b.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
`,
};
{
const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*setup.ts$'] });
expect(output).toContain('Running 3 tests using 2 workers');
expect(output).toContain('[p1] a.setup.ts:5:7 setup1');
expect(output).toContain('[p1] a.setup.ts:6:7 setup2');
expect(output).toContain('[p2] b.setup.ts:5:7 setup1');
expect(exitCode).toBe(0);
expect(passed).toBe(3);
}
{
const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.setup.ts:5'] });
expect(output).toContain('Running 1 test using 1 worker');
expect(output).toContain('[p1] a.setup.ts:5:7 setup1');
expect(exitCode).toBe(0);
expect(passed).toBe(1);
}
});
test('should prohibit filters matching both setup and test', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*.setup.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
`,
'a.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
test('setup2', async () => { });
`,
};
const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*ts$'] });
expect(output).toContain('Error: Both setup and test files match command line filter.');
expect(exitCode).toBe(1);
});
test('should run all setup files if only tests match filter', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*.setup.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
`,
'a.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
test('setup2', async () => { });
`,
'b.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
`,
};
const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['a.test.ts:7'] });
expect(exitCode).toBe(0);
expect(output).toContain('Running 4 tests using 2 workers');
expect(output).toContain('[p1] a.setup.ts:5:7 setup1');
expect(output).toContain('[p1] a.setup.ts:6:7 setup2');
expect(output).toContain('[p1] b.setup.ts:5:7 setup1');
expect(output).toContain('[p1] a.test.ts:7:7 test2');
});