mirror of
https://github.com/microsoft/playwright.git
synced 2024-12-14 05:37:20 +03:00
feat(runner): allow filtering setup and tests (#18900)
Running `npx playwright test file:123` will have the following behavior - if only test files match then only matching subset of tests will run but all setup files will run as well - if only setup files match the filter then only those setup tests will run - if both setup and test files match an error will be thrown
This commit is contained in:
parent
7c5e4241b1
commit
c0d0f54a12
@ -344,7 +344,7 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result:
|
||||
return errorDetails;
|
||||
}
|
||||
|
||||
function relativeFilePath(config: FullConfig, file: string): string {
|
||||
export function relativeFilePath(config: FullConfig, file: string): string {
|
||||
return path.relative(config.rootDir, file) || path.basename(file);
|
||||
}
|
||||
|
||||
|
@ -29,7 +29,7 @@ import type { TestRunnerPlugin } from './plugins';
|
||||
import { setRunnerToAddPluginsTo } from './plugins';
|
||||
import { dockerPlugin } from './plugins/dockerPlugin';
|
||||
import { webServerPluginsForConfig } from './plugins/webServerPlugin';
|
||||
import { formatError } from './reporters/base';
|
||||
import { formatError, relativeFilePath } from './reporters/base';
|
||||
import DotReporter from './reporters/dot';
|
||||
import EmptyReporter from './reporters/empty';
|
||||
import GitHubReporter from './reporters/github';
|
||||
@ -196,7 +196,7 @@ export class Runner {
|
||||
|
||||
async listTestFiles(projectNames: string[] | undefined): Promise<any> {
|
||||
const projects = this._collectProjects(projectNames);
|
||||
const { filesByProject } = await this._collectFiles(projects, () => true);
|
||||
const { filesByProject } = await this._collectFiles(projects, []);
|
||||
const report: any = {
|
||||
projects: []
|
||||
};
|
||||
@ -235,12 +235,13 @@ export class Runner {
|
||||
return projects;
|
||||
}
|
||||
|
||||
private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>}> {
|
||||
private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>, applyFilterToSetup: boolean}> {
|
||||
const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
|
||||
const testFileExtension = (file: string) => extensions.includes(path.extname(file));
|
||||
const filesByProject = new Map<FullProjectInternal, string[]>();
|
||||
const setupFiles = new Set<string>();
|
||||
const fileToProjectName = new Map<string, string>();
|
||||
const commandLineFileMatcher = fileMatcherFrom(commandLineFileFilters);
|
||||
for (const project of projects) {
|
||||
const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
|
||||
const setupMatch = createFileMatcher(project._setup);
|
||||
@ -250,7 +251,7 @@ export class Runner {
|
||||
if (!testFileExtension(file))
|
||||
return false;
|
||||
const isSetup = setupMatch(file);
|
||||
const isTest = !testIgnore(file) && testMatch(file) && testFileFilter(file);
|
||||
const isTest = !testIgnore(file) && testMatch(file) && commandLineFileMatcher(file);
|
||||
if (!isTest && !isSetup)
|
||||
return false;
|
||||
if (isSetup && isTest)
|
||||
@ -270,13 +271,40 @@ export class Runner {
|
||||
});
|
||||
filesByProject.set(project, testFiles);
|
||||
}
|
||||
return { filesByProject, setupFiles };
|
||||
|
||||
// If the filter didn't match any tests, apply it to the setup files.
|
||||
const applyFilterToSetup = setupFiles.size === fileToProjectName.size;
|
||||
if (applyFilterToSetup) {
|
||||
// We now have only setup files in filesByProject, because all test files were filtered out.
|
||||
for (const [project, setupFiles] of filesByProject) {
|
||||
const filteredFiles = setupFiles.filter(commandLineFileMatcher);
|
||||
if (filteredFiles.length)
|
||||
filesByProject.set(project, filteredFiles);
|
||||
else
|
||||
filesByProject.delete(project);
|
||||
}
|
||||
for (const file of setupFiles) {
|
||||
if (!commandLineFileMatcher(file))
|
||||
setupFiles.delete(file);
|
||||
}
|
||||
} else if (commandLineFileFilters.length) {
|
||||
const setupFile = [...setupFiles].find(commandLineFileMatcher);
|
||||
// If the filter is not empty and it matches both setup and tests then it's an error: we allow
|
||||
// to run either subset of tests with full setup or partial setup without any tests.
|
||||
if (setupFile) {
|
||||
const testFile = Array.from(fileToProjectName.keys()).find(f => !setupFiles.has(f));
|
||||
const config = this._loader.fullConfig();
|
||||
throw new Error(`Both setup and test files match command line filter.\n Setup file: ${relativeFilePath(config, setupFile)}\n Test file: ${relativeFilePath(config, testFile!)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { filesByProject, setupFiles, applyFilterToSetup };
|
||||
}
|
||||
|
||||
private async _collectTestGroups(options: RunOptions, fatalErrors: TestError[]): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
|
||||
const config = this._loader.fullConfig();
|
||||
const projects = this._collectProjects(options.projectFilter);
|
||||
const { filesByProject, setupFiles } = await this._collectFiles(projects, fileMatcherFrom(options.testFileFilters));
|
||||
const { filesByProject, setupFiles, applyFilterToSetup } = await this._collectFiles(projects, options.testFileFilters);
|
||||
|
||||
const allTestFiles = new Set<string>();
|
||||
for (const files of filesByProject.values())
|
||||
@ -298,7 +326,7 @@ export class Runner {
|
||||
|
||||
// Filter tests to respect line/column filter.
|
||||
if (options.testFileFilters.length)
|
||||
filterByFocusedLine(preprocessRoot, options.testFileFilters, setupFiles);
|
||||
filterByFocusedLine(preprocessRoot, options.testFileFilters, applyFilterToSetup ? new Set() : setupFiles);
|
||||
|
||||
// Complain about only.
|
||||
// TODO: check in project setup.
|
||||
|
@ -316,7 +316,7 @@ export const test = base
|
||||
|
||||
let timeline;
|
||||
try {
|
||||
timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8'));
|
||||
timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString());
|
||||
} catch (e) {
|
||||
}
|
||||
return {
|
||||
|
@ -425,3 +425,160 @@ test('test --list should enumerate setup tests as regular ones', async ({ runCom
|
||||
[p2] › b.test.ts:6:7 › test4
|
||||
Total: 5 tests in 5 files`);
|
||||
});
|
||||
|
||||
test('should allow .only in setup files', async ({ runGroups }, testInfo) => {
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
projects: [
|
||||
{
|
||||
name: 'p1',
|
||||
setup: /.*.setup.ts/,
|
||||
},
|
||||
]
|
||||
};`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('test1', async () => { });
|
||||
test('test2', async () => { });
|
||||
test('test3', async () => { });
|
||||
test('test4', async () => { });
|
||||
`,
|
||||
'a.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test.only('setup1', async () => { });
|
||||
test('setup2', async () => { });
|
||||
test.only('setup3', async () => { });
|
||||
`,
|
||||
};
|
||||
|
||||
const { exitCode, passed, timeline, output } = await runGroups(files);
|
||||
expect(output).toContain('Running 2 tests using 1 worker');
|
||||
expect(output).toContain('[p1] › a.setup.ts:5:12 › setup1');
|
||||
expect(output).toContain('[p1] › a.setup.ts:7:12 › setup3');
|
||||
expect(fileNames(timeline)).toEqual(['a.setup.ts']);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(passed).toBe(2);
|
||||
});
|
||||
|
||||
test('should allow filtering setup by file:line', async ({ runGroups }, testInfo) => {
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
projects: [
|
||||
{
|
||||
name: 'p1',
|
||||
setup: /.*a.setup.ts/,
|
||||
},
|
||||
{
|
||||
name: 'p2',
|
||||
setup: /.*b.setup.ts/,
|
||||
},
|
||||
]
|
||||
};`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('test1', async () => { });
|
||||
test('test2', async () => { });
|
||||
test('test3', async () => { });
|
||||
`,
|
||||
'a.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test('setup1', async () => { });
|
||||
test('setup2', async () => { });
|
||||
`,
|
||||
'b.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test('setup1', async () => { });
|
||||
`,
|
||||
'b.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('test1', async () => { });
|
||||
test('test2', async () => { });
|
||||
test('test3', async () => { });
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*setup.ts$'] });
|
||||
expect(output).toContain('Running 3 tests using 2 workers');
|
||||
expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
|
||||
expect(output).toContain('[p1] › a.setup.ts:6:7 › setup2');
|
||||
expect(output).toContain('[p2] › b.setup.ts:5:7 › setup1');
|
||||
expect(exitCode).toBe(0);
|
||||
expect(passed).toBe(3);
|
||||
}
|
||||
{
|
||||
const { exitCode, passed, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*a.setup.ts:5'] });
|
||||
expect(output).toContain('Running 1 test using 1 worker');
|
||||
expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
|
||||
expect(exitCode).toBe(0);
|
||||
expect(passed).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
test('should prohibit filters matching both setup and test', async ({ runGroups }, testInfo) => {
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
projects: [
|
||||
{
|
||||
name: 'p1',
|
||||
setup: /.*.setup.ts/,
|
||||
},
|
||||
]
|
||||
};`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('test1', async () => { });
|
||||
test('test2', async () => { });
|
||||
test('test3', async () => { });
|
||||
`,
|
||||
'a.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test('setup1', async () => { });
|
||||
test('setup2', async () => { });
|
||||
`,
|
||||
};
|
||||
|
||||
const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['.*ts$'] });
|
||||
expect(output).toContain('Error: Both setup and test files match command line filter.');
|
||||
expect(exitCode).toBe(1);
|
||||
});
|
||||
|
||||
test('should run all setup files if only tests match filter', async ({ runGroups }, testInfo) => {
|
||||
const files = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
projects: [
|
||||
{
|
||||
name: 'p1',
|
||||
setup: /.*.setup.ts/,
|
||||
},
|
||||
]
|
||||
};`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('test1', async () => { });
|
||||
test('test2', async () => { });
|
||||
test('test3', async () => { });
|
||||
`,
|
||||
'a.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test('setup1', async () => { });
|
||||
test('setup2', async () => { });
|
||||
`,
|
||||
'b.setup.ts': `
|
||||
const { test } = pwt;
|
||||
test('setup1', async () => { });
|
||||
`,
|
||||
};
|
||||
|
||||
const { exitCode, output } = await runGroups(files, undefined, undefined, { additionalArgs: ['a.test.ts:7'] });
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output).toContain('Running 4 tests using 2 workers');
|
||||
expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
|
||||
expect(output).toContain('[p1] › a.setup.ts:6:7 › setup2');
|
||||
expect(output).toContain('[p1] › b.setup.ts:5:7 › setup1');
|
||||
expect(output).toContain('[p1] › a.test.ts:7:7 › test2');
|
||||
});
|
||||
|
Loading…
Reference in New Issue
Block a user