microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-12-14 05:37:20 +03:00
Code Issues Projects Releases Wiki Activity

chore(test runner): route runner errors through Reporter.onError (#10257)

Browse Source
This commit is contained in:
Dmitry Gozman 2021-11-11 16:48:08 -08:00 committed by GitHub
parent b76daf361d
commit 7eec66d0f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 79 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
packages/playwright-test/src/cli.ts
View File

@ -186,9 +186,15 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
const result = await runner.run(!!opts.list, filePatternFilters, opts.project || undefined);
await stopProfiling(undefined);
if (result === 'sigint')
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
// See https://github.com/nodejs/node/issues/12921
await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
if (result.status === 'interrupted')
process.exit(130);
process.exit(result === 'passed' ? 0 : 1);
process.exit(result.status === 'passed' ? 0 : 1);
}
function forceRegExp(pattern: string): RegExp {

130
packages/playwright-test/src/runner.ts
View File

@ -15,7 +15,6 @@
* limitations under the License.
*/
/* eslint-disable no-console */
import rimraf from 'rimraf';
import * as fs from 'fs';
import * as path from 'path';
@ -24,7 +23,7 @@ import { Dispatcher, TestGroup } from './dispatcher';
import { createFileMatcher, createTitleMatcher, FilePatternFilter, monotonicTime } from './util';
import { TestCase, Suite } from './test';
import { Loader } from './loader';
import { Reporter } from '../types/testReporter';
import { FullResult, Reporter, TestError } from '../types/testReporter';
import { Multiplexer } from './reporters/multiplexer';
import DotReporter from './reporters/dot';
import GitHubReporter from './reporters/github';
@ -44,18 +43,6 @@ const removeFolderAsync = promisify(rimraf);
const readDirAsync = promisify(fs.readdir);
const readFileAsync = promisify(fs.readFile);
type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-test-titles' | 'no-tests' | 'timedout';
type RunResult = {
status: Exclude<RunResultStatus, 'forbid-only' | 'clashing-test-titles'>;
} | {
status: 'forbid-only',
locations: string[]
} | {
status: 'clashing-test-titles',
clashingTests: Map<string, TestCase[]>
};
type InternalGlobalSetupFunction = () => Promise<() => Promise<void>>;
export class Runner {
@ -108,51 +95,23 @@ export class Runner {
this._internalGlobalSetups.push(internalGlobalSetup);
}
async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise<RunResultStatus> {
async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise<FullResult> {
this._reporter = await this._createReporter(list);
const config = this._loader.fullConfig();
const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0;
const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline);
if (timedOut) {
if (!this._didBegin)
this._reporter.onBegin?.(config, new Suite(''));
await this._reporter.onEnd?.({ status: 'timedout' });
await this._flushOutput();
return 'failed';
const actualResult: FullResult = { status: 'timedout' };
if (this._didBegin)
await this._reporter.onEnd?.(actualResult);
else
this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
return actualResult;
}
if (result?.status === 'forbid-only') {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
for (const location of result?.locations)
console.error(` - ${location}`);
console.error('=====================================');
} else if (result!.status === 'no-tests') {
console.error('=================');
console.error(' no tests found.');
console.error('=================');
} else if (result?.status === 'clashing-test-titles') {
console.error('=================');
console.error(' duplicate test titles are not allowed.');
for (const [title, tests] of result?.clashingTests.entries()) {
console.error(` - title: ${title}`);
for (const test of tests)
console.error(` - ${buildItemLocation(config.rootDir, test)}`);
console.error('=================');
}
}
await this._flushOutput();
return result!.status!;
return result!;
}
async _flushOutput() {
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
// See https://github.com/nodejs/node/issues/12921
await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
}
async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise<RunResult> {
async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise<FullResult> {
const testFileFilter = testFileReFilters.length ? createFileMatcher(testFileReFilters.map(e => e.re)) : () => true;
const config = this._loader.fullConfig();
@ -218,17 +177,15 @@ export class Runner {
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0) {
const locations = onlyTestsAndSuites.map(testOrSuite => {
// Skip root and file.
const title = testOrSuite.titlePath().slice(2).join(' ');
return `${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`;
});
return { status: 'forbid-only', locations };
this._reporter.onError?.(createForbidOnlyError(config, onlyTestsAndSuites));
return { status: 'failed' };
}
}
const clashingTests = getClashingTestsPerSuite(preprocessRoot);
if (clashingTests.size > 0)
return { status: 'clashing-test-titles', clashingTests: clashingTests };
if (clashingTests.size > 0) {
this._reporter.onError?.(createDuplicateTitlesError(config, clashingTests));
return { status: 'failed' };
}
filterOnly(preprocessRoot);
filterByFocusedLine(preprocessRoot, testFileReFilters);
@ -263,8 +220,10 @@ export class Runner {
}
let total = rootSuite.allTests().length;
if (!total)
return { status: 'no-tests' };
if (!total) {
this._reporter.onError?.(createNoTestsError());
return { status: 'failed' };
}
await Promise.all(Array.from(outputDirs).map(outputDir => removeFolderAsync(outputDir).catch(e => {})));
@ -336,13 +295,15 @@ export class Runner {
}
if (sigint) {
await this._reporter.onEnd?.({ status: 'interrupted' });
return { status: 'sigint' };
const result: FullResult = { status: 'interrupted' };
await this._reporter.onEnd?.(result);
return result;
}
const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok());
await this._reporter.onEnd?.({ status: failed ? 'failed' : 'passed' });
return { status: failed ? 'failed' : 'passed' };
const result: FullResult = { status: failed ? 'failed' : 'passed' };
await this._reporter.onEnd?.(result);
return result;
} finally {
if (globalSetupResult && typeof globalSetupResult === 'function')
await globalSetupResult(this._loader.fullConfig());
@ -548,6 +509,7 @@ function createTestGroups(rootSuite: Suite): TestGroup[] {
class ListModeReporter implements Reporter {
onBegin(config: FullConfig, suite: Suite): void {
// eslint-disable-next-line no-console
console.log(`Listing tests:`);
const tests = suite.allTests();
const files = new Set<string>();
@ -556,12 +518,50 @@ class ListModeReporter implements Reporter {
const [, projectName, , ...titles] = test.titlePath();
const location = `${path.relative(config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;
const projectTitle = projectName ? `[${projectName}] › ` : '';
// eslint-disable-next-line no-console
console.log(` ${projectTitle}${location} › ${titles.join(' ')}`);
files.add(test.location.file);
}
// eslint-disable-next-line no-console
console.log(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);
}
}
function createForbidOnlyError(config: FullConfig, onlyTestsAndSuites: (TestCase | Suite)[]): TestError {
const errorMessage = [
'=====================================',
' --forbid-only found a focused test.',
];
for (const testOrSuite of onlyTestsAndSuites) {
// Skip root and file.
const title = testOrSuite.titlePath().slice(2).join(' ');
errorMessage.push(` - ${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`);
}
errorMessage.push('=====================================');
return createStacklessError(errorMessage.join('\n'));
}
function createDuplicateTitlesError(config: FullConfig, clashingTests: Map<string, TestCase[]>): TestError {
const errorMessage = [
'========================================',
' duplicate test titles are not allowed.',
];
for (const [title, tests] of clashingTests.entries()) {
errorMessage.push(` - title: ${title}`);
for (const test of tests)
errorMessage.push(` - ${buildItemLocation(config.rootDir, test)}`);
}
errorMessage.push('========================================');
return createStacklessError(errorMessage.join('\n'));
}
function createNoTestsError(): TestError {
return createStacklessError(`=================\n no tests found.\n=================`);
}
function createStacklessError(message: string): TestError {
return { message };
}
export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html'] as const;
export type BuiltInReporter = typeof builtInReporters[number];

6
tests/playwright-test/reporter-base.spec.ts
View File

@ -248,3 +248,9 @@ test('should print errors with inconsistent message/stack', async ({ runInlineTe
expect(result.output).toContain('hi!Error: Hello');
expect(result.output).toContain('at myTest');
});
test('should print "no tests found" error', async ({ runInlineTest }) => {
const result = await runInlineTest({ });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('no tests found.');
});