diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 5d286ef4b6..2cb060fe45 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -186,9 +186,15 @@ async function runTests(args: string[], opts: { [key: string]: any }) { const result = await runner.run(!!opts.list, filePatternFilters, opts.project || undefined); await stopProfiling(undefined); - if (result === 'sigint') + // Calling process.exit() might truncate large stdout/stderr output. + // See https://github.com/nodejs/node/issues/6456. + // See https://github.com/nodejs/node/issues/12921 + await new Promise(resolve => process.stdout.write('', () => resolve())); + await new Promise(resolve => process.stderr.write('', () => resolve())); + + if (result.status === 'interrupted') process.exit(130); - process.exit(result === 'passed' ? 0 : 1); + process.exit(result.status === 'passed' ? 0 : 1); } function forceRegExp(pattern: string): RegExp { diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index cf8b4ade71..927016ef9b 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -15,7 +15,6 @@ * limitations under the License. */ -/* eslint-disable no-console */ import rimraf from 'rimraf'; import * as fs from 'fs'; import * as path from 'path'; @@ -24,7 +23,7 @@ import { Dispatcher, TestGroup } from './dispatcher'; import { createFileMatcher, createTitleMatcher, FilePatternFilter, monotonicTime } from './util'; import { TestCase, Suite } from './test'; import { Loader } from './loader'; -import { Reporter } from '../types/testReporter'; +import { FullResult, Reporter, TestError } from '../types/testReporter'; import { Multiplexer } from './reporters/multiplexer'; import DotReporter from './reporters/dot'; import GitHubReporter from './reporters/github'; @@ -44,18 +43,6 @@ const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); const readFileAsync = promisify(fs.readFile); -type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-test-titles' | 'no-tests' | 'timedout'; - -type RunResult = { - status: Exclude; -} | { - status: 'forbid-only', - locations: string[] -} | { - status: 'clashing-test-titles', - clashingTests: Map -}; - type InternalGlobalSetupFunction = () => Promise<() => Promise>; export class Runner { @@ -108,51 +95,23 @@ export class Runner { this._internalGlobalSetups.push(internalGlobalSetup); } - async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise { + async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise { this._reporter = await this._createReporter(list); const config = this._loader.fullConfig(); const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : 0; const { result, timedOut } = await raceAgainstDeadline(this._run(list, filePatternFilters, projectNames), globalDeadline); if (timedOut) { - if (!this._didBegin) - this._reporter.onBegin?.(config, new Suite('')); - await this._reporter.onEnd?.({ status: 'timedout' }); - await this._flushOutput(); - return 'failed'; + const actualResult: FullResult = { status: 'timedout' }; + if (this._didBegin) + await this._reporter.onEnd?.(actualResult); + else + this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); + return actualResult; } - if (result?.status === 'forbid-only') { - console.error('====================================='); - console.error(' --forbid-only found a focused test.'); - for (const location of result?.locations) - console.error(` - ${location}`); - console.error('====================================='); - } else if (result!.status === 'no-tests') { - console.error('================='); - console.error(' no tests found.'); - console.error('================='); - } else if (result?.status === 'clashing-test-titles') { - console.error('================='); - console.error(' duplicate test titles are not allowed.'); - for (const [title, tests] of result?.clashingTests.entries()) { - console.error(` - title: ${title}`); - for (const test of tests) - console.error(` - ${buildItemLocation(config.rootDir, test)}`); - console.error('================='); - } - } - await this._flushOutput(); - return result!.status!; + return result!; } - async _flushOutput() { - // Calling process.exit() might truncate large stdout/stderr output. - // See https://github.com/nodejs/node/issues/6456. - // See https://github.com/nodejs/node/issues/12921 - await new Promise(resolve => process.stdout.write('', () => resolve())); - await new Promise(resolve => process.stderr.write('', () => resolve())); - } - - async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise { + async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise { const testFileFilter = testFileReFilters.length ? createFileMatcher(testFileReFilters.map(e => e.re)) : () => true; const config = this._loader.fullConfig(); @@ -218,17 +177,15 @@ export class Runner { if (config.forbidOnly) { const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); if (onlyTestsAndSuites.length > 0) { - const locations = onlyTestsAndSuites.map(testOrSuite => { - // Skip root and file. - const title = testOrSuite.titlePath().slice(2).join(' '); - return `${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`; - }); - return { status: 'forbid-only', locations }; + this._reporter.onError?.(createForbidOnlyError(config, onlyTestsAndSuites)); + return { status: 'failed' }; } } const clashingTests = getClashingTestsPerSuite(preprocessRoot); - if (clashingTests.size > 0) - return { status: 'clashing-test-titles', clashingTests: clashingTests }; + if (clashingTests.size > 0) { + this._reporter.onError?.(createDuplicateTitlesError(config, clashingTests)); + return { status: 'failed' }; + } filterOnly(preprocessRoot); filterByFocusedLine(preprocessRoot, testFileReFilters); @@ -263,8 +220,10 @@ export class Runner { } let total = rootSuite.allTests().length; - if (!total) - return { status: 'no-tests' }; + if (!total) { + this._reporter.onError?.(createNoTestsError()); + return { status: 'failed' }; + } await Promise.all(Array.from(outputDirs).map(outputDir => removeFolderAsync(outputDir).catch(e => {}))); @@ -336,13 +295,15 @@ export class Runner { } if (sigint) { - await this._reporter.onEnd?.({ status: 'interrupted' }); - return { status: 'sigint' }; + const result: FullResult = { status: 'interrupted' }; + await this._reporter.onEnd?.(result); + return result; } const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok()); - await this._reporter.onEnd?.({ status: failed ? 'failed' : 'passed' }); - return { status: failed ? 'failed' : 'passed' }; + const result: FullResult = { status: failed ? 'failed' : 'passed' }; + await this._reporter.onEnd?.(result); + return result; } finally { if (globalSetupResult && typeof globalSetupResult === 'function') await globalSetupResult(this._loader.fullConfig()); @@ -548,6 +509,7 @@ function createTestGroups(rootSuite: Suite): TestGroup[] { class ListModeReporter implements Reporter { onBegin(config: FullConfig, suite: Suite): void { + // eslint-disable-next-line no-console console.log(`Listing tests:`); const tests = suite.allTests(); const files = new Set(); @@ -556,12 +518,50 @@ class ListModeReporter implements Reporter { const [, projectName, , ...titles] = test.titlePath(); const location = `${path.relative(config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`; const projectTitle = projectName ? `[${projectName}] › ` : ''; + // eslint-disable-next-line no-console console.log(` ${projectTitle}${location} › ${titles.join(' ')}`); files.add(test.location.file); } + // eslint-disable-next-line no-console console.log(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`); } } +function createForbidOnlyError(config: FullConfig, onlyTestsAndSuites: (TestCase | Suite)[]): TestError { + const errorMessage = [ + '=====================================', + ' --forbid-only found a focused test.', + ]; + for (const testOrSuite of onlyTestsAndSuites) { + // Skip root and file. + const title = testOrSuite.titlePath().slice(2).join(' '); + errorMessage.push(` - ${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`); + } + errorMessage.push('====================================='); + return createStacklessError(errorMessage.join('\n')); +} + +function createDuplicateTitlesError(config: FullConfig, clashingTests: Map): TestError { + const errorMessage = [ + '========================================', + ' duplicate test titles are not allowed.', + ]; + for (const [title, tests] of clashingTests.entries()) { + errorMessage.push(` - title: ${title}`); + for (const test of tests) + errorMessage.push(` - ${buildItemLocation(config.rootDir, test)}`); + } + errorMessage.push('========================================'); + return createStacklessError(errorMessage.join('\n')); +} + +function createNoTestsError(): TestError { + return createStacklessError(`=================\n no tests found.\n=================`); +} + +function createStacklessError(message: string): TestError { + return { message }; +} + export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html'] as const; export type BuiltInReporter = typeof builtInReporters[number]; diff --git a/tests/playwright-test/reporter-base.spec.ts b/tests/playwright-test/reporter-base.spec.ts index 08223a59b8..6459b1a4ca 100644 --- a/tests/playwright-test/reporter-base.spec.ts +++ b/tests/playwright-test/reporter-base.spec.ts @@ -248,3 +248,9 @@ test('should print errors with inconsistent message/stack', async ({ runInlineTe expect(result.output).toContain('hi!Error: Hello'); expect(result.output).toContain('at myTest'); }); + +test('should print "no tests found" error', async ({ runInlineTest }) => { + const result = await runInlineTest({ }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('no tests found.'); +});