microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-12-13 17:14:02 +03:00
Code Issues Projects Releases Wiki Activity

test(cli): setup files are in list-files and test --list output (#18890)

Browse Source
This commit is contained in:
Yury Semikhatsky 2022-11-17 16:31:04 -08:00 committed by GitHub
parent c2e3704f86
commit 0f4b67bc6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 178 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/playwright-test/src/runner.ts
View File

@ -203,7 +203,7 @@ export class Runner {
for (const [project, files] of filesByProject) {
report.projects.push({
...sanitizeConfigForJSON(project, new Set()),
files: files,
files
});
}
return report;

134
tests/playwright-test/playwright-test-fixtures.ts
View File

@ -30,6 +30,11 @@ import { test as base } from './stable-test-runner';
const removeFolderAsync = promisify(rimraf);
export type CliRunResult = {
exitCode: number,
output: string,
};
export type RunResult = {
exitCode: number,
output: string,
@ -113,7 +118,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
}
const outputDir = path.join(baseDir, 'test-results');
const reportFile = path.join(outputDir, 'report.json');
const args = ['node', cliEntrypoint, 'test'];
const args = ['test'];
if (!options.usesCustomOutputDir)
args.push('--output=' + outputDir);
if (!options.usesCustomReporters)
@ -124,12 +129,71 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
);
if (options.additionalArgs)
args.push(...options.additionalArgs);
const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'playwright-test-cache-'));
const cwd = options.cwd ? path.resolve(baseDir, options.cwd) : baseDir;
// eslint-disable-next-line prefer-const
let { exitCode, output } = await runPlaywrightCommand(childProcess, cwd, args, {
PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile,
...env,
}, options.sendSIGINTAfter);
const summary = (re: RegExp) => {
let result = 0;
let match = re.exec(output);
while (match) {
result += (+match[1]);
match = re.exec(output);
}
return result;
};
const passed = summary(/(\d+) passed/g);
const failed = summary(/(\d+) failed/g);
const flaky = summary(/(\d+) flaky/g);
const skipped = summary(/(\d+) skipped/g);
const interrupted = summary(/(\d+) interrupted/g);
let report;
try {
report = JSON.parse(fs.readFileSync(reportFile).toString());
} catch (e) {
output += '\n' + e.toString();
}
const results: JSONReportTestResult[] = [];
function visitSuites(suites?: JSONReportSuite[]) {
if (!suites)
return;
for (const suite of suites) {
for (const spec of suite.specs) {
for (const test of spec.tests)
results.push(...test.results);
}
visitSuites(suite.suites);
}
}
if (report)
visitSuites(report.suites);
return {
exitCode,
output,
passed,
failed,
flaky,
skipped,
interrupted,
report,
results,
};
}
async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: Env, sendSIGINTAfter?: number): Promise<CliRunResult> {
const command = ['node', cliEntrypoint];
command.push(...commandWithArguments);
const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
const testProcess = childProcess({
command: args,
command,
env: {
...process.env,
PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile,
PWTEST_CACHE_DIR: cacheDir,
// BEGIN: Reserved CI
CI: undefined,
@ -151,11 +215,11 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
NODE_OPTIONS: undefined,
...env,
},
cwd: options.cwd ? path.resolve(baseDir, options.cwd) : baseDir,
cwd,
});
let didSendSigint = false;
testProcess.onOutput = () => {
if (options.sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= options.sendSIGINTAfter) {
if (sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= sendSIGINTAfter) {
didSendSigint = true;
process.kill(testProcess.process.pid!, 'SIGINT');
}
@ -163,56 +227,10 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
const { exitCode } = await testProcess.exited;
await removeFolderAsync(cacheDir);
const outputString = testProcess.output.toString();
const summary = (re: RegExp) => {
let result = 0;
let match = re.exec(outputString);
while (match) {
result += (+match[1]);
match = re.exec(outputString);
}
return result;
};
const passed = summary(/(\d+) passed/g);
const failed = summary(/(\d+) failed/g);
const flaky = summary(/(\d+) flaky/g);
const skipped = summary(/(\d+) skipped/g);
const interrupted = summary(/(\d+) interrupted/g);
let report;
try {
report = JSON.parse(fs.readFileSync(reportFile).toString());
} catch (e) {
testProcess.output += '\n' + e.toString();
}
const results: JSONReportTestResult[] = [];
function visitSuites(suites?: JSONReportSuite[]) {
if (!suites)
return;
for (const suite of suites) {
for (const spec of suite.specs) {
for (const test of spec.tests)
results.push(...test.results);
}
visitSuites(suite.suites);
}
}
if (report)
visitSuites(report.suites);
return {
exitCode,
output: testProcess.output,
passed,
failed,
flaky,
skipped,
interrupted,
report,
results,
};
return { exitCode, output: testProcess.output.toString() };
}
type RunOptions = {
sendSIGINTAfter?: number;
usesCustomOutputDir?: boolean;
@ -226,6 +244,7 @@ type Fixtures = {
runTSC: (files: Files) => Promise<TSCResult>;
nodeVersion: { major: number, minor: number, patch: number };
runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
runCommand: (files: Files, args: string[]) => Promise<CliRunResult>;
};
export const test = base
@ -245,6 +264,13 @@ export const test = base
});
},
runCommand: async ({ childProcess }, use, testInfo: TestInfo) => {
await use(async (files: Files, args: string[]) => {
const baseDir = await writeFiles(testInfo, files);
return await runPlaywrightCommand(childProcess, baseDir, args, { });
});
},
runTSC: async ({ childProcess }, use, testInfo) => {
await use(async files => {
const baseDir = await writeFiles(testInfo, { 'tsconfig.json': JSON.stringify(TSCONFIG), ...files });

97
tests/playwright-test/project-setup.spec.ts
View File

@ -328,3 +328,100 @@ test('same file cannot be a setup and a test in different projects', async ({ ru
expect(exitCode).toBe(1);
expect(output).toContain(`a.test.ts" matches 'setup' filter in project "p1" and 'testMatch' filter in project "p2"`);
});
test('list-files should enumerate setup files in same group', async ({ runCommand }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*a..setup.ts$/,
testMatch: /.*a.test.ts$/,
},
{
name: 'p2',
setup: /.*b.setup.ts$/,
testMatch: /.*b.test.ts$/
},
]
};`,
'a1.setup.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
'a2.setup.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
'a.test.ts': `
const { test } = pwt;
test('test2', async () => { });
`,
'b.setup.ts': `
const { test } = pwt;
test('test3', async () => { });
`,
'b.test.ts': `
const { test } = pwt;
test('test4', async () => { });
`,
};
const { exitCode, output } = await runCommand(files, ['list-files']);
expect(exitCode).toBe(0);
const json = JSON.parse(output);
expect(json.projects.map(p => p.name)).toEqual(['p1', 'p2']);
expect(json.projects[0].files.map(f => path.basename(f))).toEqual(['a.test.ts', 'a1.setup.ts', 'a2.setup.ts']);
expect(json.projects[1].files.map(f => path.basename(f))).toEqual(['b.setup.ts', 'b.test.ts']);
});
test('test --list should enumerate setup tests as regular ones', async ({ runCommand }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*a..setup.ts$/,
testMatch: /.*a.test.ts$/,
},
{
name: 'p2',
setup: /.*b.setup.ts$/,
testMatch: /.*b.test.ts$/
},
]
};`,
'a1.setup.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
'a2.setup.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
'a.test.ts': `
const { test } = pwt;
test('test2', async () => { });
`,
'b.setup.ts': `
const { test } = pwt;
test('test3', async () => { });
`,
'b.test.ts': `
const { test } = pwt;
test('test4', async () => { });
`,
};
const { exitCode, output } = await runCommand(files, ['test', '--list']);
expect(exitCode).toBe(0);
expect(output).toContain(`Listing tests:
[p1] › a.test.ts:6:7 › test2
[p1] › a1.setup.ts:5:7 › test1
[p1] › a2.setup.ts:5:7 › test1
[p2] › b.setup.ts:5:7 › test3
[p2] › b.test.ts:6:7 › test4
Total: 5 tests in 5 files`);
});