diff --git a/packages/playwright-test/src/globals.ts b/packages/playwright-test/src/globals.ts index 985d0b3930..6c0c0040c1 100644 --- a/packages/playwright-test/src/globals.ts +++ b/packages/playwright-test/src/globals.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { TestInfoImpl } from './types'; +import type { TestInfoImpl } from './testInfo'; import { Suite } from './test'; let currentTestInfoValue: TestInfoImpl | null = null; diff --git a/packages/playwright-test/src/matchers/golden.ts b/packages/playwright-test/src/matchers/golden.ts index ca499fe5a4..546d04bd68 100644 --- a/packages/playwright-test/src/matchers/golden.ts +++ b/packages/playwright-test/src/matchers/golden.ts @@ -22,10 +22,11 @@ import path from 'path'; import jpeg from 'jpeg-js'; import pixelmatch from 'pixelmatch'; import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch'; -import { TestInfoImpl, UpdateSnapshots } from '../types'; +import { UpdateSnapshots } from '../types'; import { addSuffixToFilePath } from '../util'; import BlinkDiff from '../third_party/blink-diff'; import PNGImage from '../third_party/png-js'; +import { TestInfoImpl } from '../testInfo'; // Note: we require the pngjs version of pixelmatch to avoid version mismatches. const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] })) as typeof import('pngjs'); @@ -128,12 +129,7 @@ export function compare( return { pass: true, message }; } if (updateSnapshots === 'missing') { - if (testInfo.status === 'passed') - testInfo.status = 'failed'; - if (!('error' in testInfo)) - testInfo.error = { value: 'Error: ' + message }; - else if (testInfo.error?.value) - testInfo.error.value += '\nError: ' + message; + testInfo._appendErrorMessage(message); return { pass: true, message }; } return { pass: false, message }; diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts new file mode 100644 index 0000000000..ba647139a1 --- /dev/null +++ b/packages/playwright-test/src/testInfo.ts @@ -0,0 +1,268 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import * as mime from 'mime'; +import path from 'path'; +import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/async'; +import { calculateSha1 } from 'playwright-core/lib/utils/utils'; +import type { FullConfig, FullProject, TestError, TestInfo, TestStatus } from '../types/test'; +import { WorkerInitParams } from './ipc'; +import { Loader } from './loader'; +import { ProjectImpl } from './project'; +import { TestCase } from './test'; +import { Annotations, TestStepInternal } from './types'; +import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util'; + +export class TestInfoImpl implements TestInfo { + private _projectImpl: ProjectImpl; + private _addStepImpl: (data: Omit) => TestStepInternal; + readonly _test: TestCase; + readonly _timeoutRunner: TimeoutRunner; + readonly _startTime: number; + readonly _startWallTime: number; + + // ------------ TestInfo fields ------------ + readonly repeatEachIndex: number; + readonly retry: number; + readonly workerIndex: number; + readonly parallelIndex: number; + readonly project: FullProject; + config: FullConfig; + readonly title: string; + readonly titlePath: string[]; + readonly file: string; + readonly line: number; + readonly column: number; + readonly fn: Function; + expectedStatus: TestStatus; + duration: number = 0; + readonly annotations: Annotations = []; + readonly attachments: TestInfo['attachments'] = []; + status: TestStatus = 'passed'; + readonly stdout: TestInfo['stdout'] = []; + readonly stderr: TestInfo['stderr'] = []; + timeout: number; + snapshotSuffix: string = ''; + readonly outputDir: string; + readonly snapshotDir: string; + error: TestError | undefined = undefined; + + constructor( + loader: Loader, + workerParams: WorkerInitParams, + test: TestCase, + retry: number, + addStepImpl: (data: Omit) => TestStepInternal, + ) { + this._projectImpl = loader.projects()[workerParams.projectIndex]; + this._test = test; + this._addStepImpl = addStepImpl; + this._startTime = monotonicTime(); + this._startWallTime = Date.now(); + + this.repeatEachIndex = workerParams.repeatEachIndex; + this.retry = retry; + this.workerIndex = workerParams.workerIndex; + this.parallelIndex = workerParams.parallelIndex; + this.project = this._projectImpl.config; + this.config = loader.fullConfig(); + this.title = test.title; + this.titlePath = test.titlePath(); + this.file = test.location.file; + this.line = test.location.line; + this.column = test.location.column; + this.fn = test.fn; + this.expectedStatus = test.expectedStatus; + this.timeout = this.project.timeout; + + this._timeoutRunner = new TimeoutRunner(this.timeout); + + this.outputDir = (() => { + const sameName = loader.projects().filter(project => project.config.name === this.project.name); + let uniqueProjectNamePathSegment: string; + if (sameName.length > 1) + uniqueProjectNamePathSegment = this.project.name + (sameName.indexOf(this._projectImpl) + 1); + else + uniqueProjectNamePathSegment = this.project.name; + + const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, '')); + const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-'); + const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this.workerIndex); + + let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec)); + if (uniqueProjectNamePathSegment) + testOutputDir += '-' + sanitizeForFilePath(uniqueProjectNamePathSegment); + if (this.retry) + testOutputDir += '-retry' + this.retry; + if (this.repeatEachIndex) + testOutputDir += '-repeat' + this.repeatEachIndex; + return path.join(this.project.outputDir, testOutputDir); + })(); + + this.snapshotDir = (() => { + const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile); + return path.join(this.project.snapshotDir, relativeTestFilePath + '-snapshots'); + })(); + } + + private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) { + if (typeof modifierArgs[1] === 'function') { + throw new Error([ + 'It looks like you are calling test.skip() inside the test and pass a callback.', + 'Pass a condition instead and optional description instead:', + `test('my test', async ({ page, isMobile }) => {`, + ` test.skip(isMobile, 'This test is not applicable on mobile');`, + `});`, + ].join('\n')); + } + + if (modifierArgs.length >= 1 && !modifierArgs[0]) + return; + + const description = modifierArgs[1]; + this.annotations.push({ type, description }); + if (type === 'slow') { + this.setTimeout(this.timeout * 3); + } else if (type === 'skip' || type === 'fixme') { + this.expectedStatus = 'skipped'; + throw new SkipError('Test is skipped: ' + (description || '')); + } else if (type === 'fail') { + if (this.expectedStatus !== 'skipped') + this.expectedStatus = 'failed'; + } + } + + async _runWithTimeout(cb: () => Promise): Promise { + try { + await this._timeoutRunner.run(cb); + } catch (error) { + if (!(error instanceof TimeoutRunnerError)) + throw error; + // Do not overwrite existing failure upon hook/teardown timeout. + if (this.status === 'passed') + this.status = 'timedOut'; + } + this.duration = monotonicTime() - this._startTime; + } + + async _runFn(fn: Function, skips?: 'allowSkips'): Promise { + try { + await fn(); + } catch (error) { + if (skips === 'allowSkips' && error instanceof SkipError) { + if (this.status === 'passed') + this.status = 'skipped'; + } else { + const serialized = serializeError(error); + this._failWithError(serialized); + return serialized; + } + } + } + + _addStep(data: Omit) { + return this._addStepImpl(data); + } + + _failWithError(error: TestError) { + // Do not overwrite any previous error and error status. + // Some (but not all) scenarios include: + // - expect() that fails after uncaught exception. + // - fail after the timeout, e.g. due to fixture teardown. + if (this.status === 'passed') + this.status = 'failed'; + if (this.error === undefined) + this.error = error; + } + + _appendErrorMessage(message: string) { + // Do not overwrite any previous error status. + if (this.status === 'passed') + this.status = 'failed'; + if (this.error === undefined) + this.error = { value: 'Error: ' + message }; + else if (this.error.value) + this.error.value += '\nError: ' + message; + } + + // ------------ TestInfo methods ------------ + + async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { + if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1) + throw new Error(`Exactly one of "path" and "body" must be specified`); + if (options.path !== undefined) { + const hash = calculateSha1(options.path); + const dest = this.outputPath('attachments', hash + path.extname(options.path)); + await fs.promises.mkdir(path.dirname(dest), { recursive: true }); + await fs.promises.copyFile(options.path, dest); + const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream'); + this.attachments.push({ name, contentType, path: dest }); + } else { + const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream'); + this.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body }); + } + } + + outputPath(...pathSegments: string[]){ + fs.mkdirSync(this.outputDir, { recursive: true }); + const joinedPath = path.join(...pathSegments); + const outputPath = getContainedPath(this.outputDir, joinedPath); + if (outputPath) + return outputPath; + throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`); + } + + snapshotPath(...pathSegments: string[]) { + let suffix = ''; + const projectNamePathSegment = sanitizeForFilePath(this.project.name); + if (projectNamePathSegment) + suffix += '-' + projectNamePathSegment; + if (this.snapshotSuffix) + suffix += '-' + this.snapshotSuffix; + const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix); + const snapshotPath = getContainedPath(this.snapshotDir, subPath); + if (snapshotPath) + return snapshotPath; + throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`); + } + + skip(...args: [arg?: any, description?: string]) { + this._modifier('skip', args); + } + + fixme(...args: [arg?: any, description?: string]) { + this._modifier('fixme', args); + } + + fail(...args: [arg?: any, description?: string]) { + this._modifier('fail', args); + } + + slow(...args: [arg?: any, description?: string]) { + this._modifier('slow', args); + } + + setTimeout(timeout: number) { + if (!this.timeout) + return; // Zero timeout means some debug mode - do not set a timeout. + this.timeout = timeout; + this._timeoutRunner.updateTimeout(timeout); + } +} + +class SkipError extends Error { +} diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 160e7fce34..bcb8a6f91a 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { Fixtures, TestError, TestInfo } from '../types/test'; +import type { Fixtures, TestError } from '../types/test'; import type { Location } from '../types/testReporter'; export * from '../types/test'; export { Location } from '../types/testReporter'; @@ -34,8 +34,4 @@ export interface TestStepInternal { location?: Location; } -export interface TestInfoImpl extends TestInfo { - _addStep: (data: Omit) => TestStepInternal; -} - export type TestCaseType = 'beforeAll' | 'afterAll' | 'test'; diff --git a/packages/playwright-test/src/worker.ts b/packages/playwright-test/src/worker.ts index 9d0e20d6be..fd887ff783 100644 --- a/packages/playwright-test/src/worker.ts +++ b/packages/playwright-test/src/worker.ts @@ -33,7 +33,7 @@ global.console = new Console({ process.stdout.write = (chunk: string | Buffer) => { const outPayload: TestOutputPayload = { - testId: workerRunner?._currentTest?.testId, + testId: workerRunner?._currentTest?._test._id, ...chunkToParams(chunk) }; sendMessageToParent('stdOut', outPayload); @@ -43,7 +43,7 @@ process.stdout.write = (chunk: string | Buffer) => { if (!process.env.PW_RUNNER_DEBUG) { process.stderr.write = (chunk: string | Buffer) => { const outPayload: TestOutputPayload = { - testId: workerRunner?._currentTest?.testId, + testId: workerRunner?._currentTest?._test._id, ...chunkToParams(chunk) }; sendMessageToParent('stdErr', outPayload); diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index b86e19030b..2ac6b1bb7d 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -14,44 +14,36 @@ * limitations under the License. */ -import fs from 'fs'; -import path from 'path'; import rimraf from 'rimraf'; -import * as mime from 'mime'; import util from 'util'; import colors from 'colors/safe'; import { EventEmitter } from 'events'; -import { monotonicTime, serializeError, sanitizeForFilePath, getContainedPath, addSuffixToFilePath, prependToTestError, trimLongString, formatLocation } from './util'; +import { serializeError, prependToTestError, formatLocation } from './util'; import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc'; import { setCurrentTestInfo } from './globals'; import { Loader } from './loader'; import { Modifier, Suite, TestCase } from './test'; -import { Annotations, TestCaseType, TestError, TestInfo, TestInfoImpl, TestStepInternal, WorkerInfo } from './types'; +import { Annotations, TestError, TestInfo, TestStepInternal, WorkerInfo } from './types'; import { ProjectImpl } from './project'; import { FixtureRunner } from './fixtures'; -import { TimeoutRunner, raceAgainstTimeout, TimeoutRunnerError } from 'playwright-core/lib/utils/async'; -import { calculateSha1 } from 'playwright-core/lib/utils/utils'; +import { raceAgainstTimeout } from 'playwright-core/lib/utils/async'; +import { TestInfoImpl } from './testInfo'; const removeFolderAsync = util.promisify(rimraf); -type TestData = { testId: string, testInfo: TestInfoImpl, type: TestCaseType }; - export class WorkerRunner extends EventEmitter { private _params: WorkerInitParams; private _loader!: Loader; private _project!: ProjectImpl; private _workerInfo!: WorkerInfo; - private _projectNamePathSegment = ''; - private _uniqueProjectNamePathSegment = ''; private _fixtureRunner: FixtureRunner; - private _failedTest: TestData | undefined; + private _failedTest: TestInfoImpl | undefined; private _fatalError: TestError | undefined; private _entries = new Map(); private _isStopped = false; private _runFinished = Promise.resolve(); - private _currentTimeoutRunner: TimeoutRunner | undefined; - _currentTest: TestData | null = null; + _currentTest: TestInfoImpl | null = null; constructor(params: WorkerInitParams) { super(); @@ -64,11 +56,11 @@ export class WorkerRunner extends EventEmitter { this._isStopped = true; // Interrupt current action. - this._currentTimeoutRunner?.interrupt(); + this._currentTest?._timeoutRunner.interrupt(); // TODO: mark test as 'interrupted' instead. - if (this._currentTest && this._currentTest.testInfo.status === 'passed') - this._currentTest.testInfo.status = 'skipped'; + if (this._currentTest && this._currentTest.status === 'passed') + this._currentTest.status = 'skipped'; } return this._runFinished; } @@ -102,11 +94,8 @@ export class WorkerRunner extends EventEmitter { // a test runner. In the latter case, the worker state could be messed up, // and continuing to run tests in the same worker is problematic. Therefore, // we turn this into a fatal error and restart the worker anyway. - if (this._currentTest && this._currentTest.type === 'test' && this._currentTest.testInfo.expectedStatus !== 'failed') { - if (!this._currentTest.testInfo.error) { - this._currentTest.testInfo.status = 'failed'; - this._currentTest.testInfo.error = serializeError(error); - } + if (this._currentTest && this._currentTest._test._type === 'test' && this._currentTest.expectedStatus !== 'failed') { + this._currentTest._failWithError(serializeError(error)); } else { // No current test - fatal error. if (!this._fatalError) @@ -122,15 +111,6 @@ export class WorkerRunner extends EventEmitter { this._loader = await Loader.deserialize(this._params.loader); this._project = this._loader.projects()[this._params.projectIndex]; - this._projectNamePathSegment = sanitizeForFilePath(this._project.config.name); - - const sameName = this._loader.projects().filter(project => project.config.name === this._project.config.name); - if (sameName.length > 1) - this._uniqueProjectNamePathSegment = this._project.config.name + (sameName.indexOf(this._project) + 1); - else - this._uniqueProjectNamePathSegment = this._project.config.name; - this._uniqueProjectNamePathSegment = sanitizeForFilePath(this._uniqueProjectNamePathSegment); - this._workerInfo = { workerIndex: this._params.workerIndex, parallelIndex: this._params.parallelIndex, @@ -218,134 +198,40 @@ export class WorkerRunner extends EventEmitter { } private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) { - const startTime = monotonicTime(); - const startWallTime = Date.now(); - let timeoutRunner: TimeoutRunner; - const testId = test._id; - - const baseOutputDir = (() => { - const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, '')); - const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-'); - const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this._params.workerIndex); - - let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec)); - if (this._uniqueProjectNamePathSegment) - testOutputDir += '-' + this._uniqueProjectNamePathSegment; - if (retry) - testOutputDir += '-retry' + retry; - if (this._params.repeatEachIndex) - testOutputDir += '-repeat' + this._params.repeatEachIndex; - return path.join(this._project.config.outputDir, testOutputDir); - })(); - - const snapshotDir = (() => { - const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile); - return path.join(this._project.config.snapshotDir, relativeTestFilePath + '-snapshots'); - })(); - let lastStepId = 0; - const testInfo: TestInfoImpl = { - workerIndex: this._params.workerIndex, - parallelIndex: this._params.parallelIndex, - project: this._project.config, - config: this._loader.fullConfig(), - title: test.title, - titlePath: test.titlePath(), - file: test.location.file, - line: test.location.line, - column: test.location.column, - fn: test.fn, - repeatEachIndex: this._params.repeatEachIndex, - retry, - expectedStatus: test.expectedStatus, - annotations: [], - attachments: [], - attach: async (name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) => { - if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1) - throw new Error(`Exactly one of "path" and "body" must be specified`); - if (options.path !== undefined) { - const hash = calculateSha1(options.path); - const dest = testInfo.outputPath('attachments', hash + path.extname(options.path)); - await fs.promises.mkdir(path.dirname(dest), { recursive: true }); - await fs.promises.copyFile(options.path, dest); - const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream'); - testInfo.attachments.push({ name, contentType, path: dest }); - } else { - const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream'); - testInfo.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body }); + const testInfo = new TestInfoImpl(this._loader, this._params, test, retry, data => { + const stepId = `${data.category}@${data.title}@${++lastStepId}`; + let callbackHandled = false; + const step: TestStepInternal = { + ...data, + complete: (error?: Error | TestError) => { + if (callbackHandled) + return; + callbackHandled = true; + if (error instanceof Error) + error = serializeError(error); + const payload: StepEndPayload = { + testId: test._id, + stepId, + wallTime: Date.now(), + error, + }; + this.emit('stepEnd', payload); } - }, - duration: 0, - status: 'passed', - stdout: [], - stderr: [], - timeout: this._project.config.timeout, - snapshotSuffix: '', - outputDir: baseOutputDir, - snapshotDir, - outputPath: (...pathSegments: string[]): string => { - fs.mkdirSync(baseOutputDir, { recursive: true }); - const joinedPath = path.join(...pathSegments); - const outputPath = getContainedPath(baseOutputDir, joinedPath); - if (outputPath) return outputPath; - throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`); - }, - snapshotPath: (...pathSegments: string[]): string => { - let suffix = ''; - if (this._projectNamePathSegment) - suffix += '-' + this._projectNamePathSegment; - if (testInfo.snapshotSuffix) - suffix += '-' + testInfo.snapshotSuffix; - const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix); - const snapshotPath = getContainedPath(snapshotDir, subPath); - if (snapshotPath) return snapshotPath; - throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`); - }, - skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args), - fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args), - fail: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fail', args), - slow: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'slow', args), - setTimeout: (timeout: number) => { - if (!testInfo.timeout) - return; // Zero timeout means some debug mode - do not set a timeout. - testInfo.timeout = timeout; - if (timeoutRunner) - timeoutRunner.updateTimeout(timeout); - }, - _addStep: data => { - const stepId = `${data.category}@${data.title}@${++lastStepId}`; - let callbackHandled = false; - const step: TestStepInternal = { - ...data, - complete: (error?: Error | TestError) => { - if (callbackHandled) - return; - callbackHandled = true; - if (error instanceof Error) - error = serializeError(error); - const payload: StepEndPayload = { - testId, - stepId, - wallTime: Date.now(), - error, - }; - this.emit('stepEnd', payload); - } - }; - const hasLocation = data.location && !data.location.file.includes('@playwright'); - // Sanitize location that comes from user land, it might have extra properties. - const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined; - const payload: StepBeginPayload = { - testId, - stepId, - ...data, - location, - wallTime: Date.now(), - }; - this.emit('stepBegin', payload); - return step; - }, - }; + }; + const hasLocation = data.location && !data.location.file.includes('@playwright'); + // Sanitize location that comes from user land, it might have extra properties. + const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined; + const payload: StepBeginPayload = { + testId: test._id, + stepId, + ...data, + location, + wallTime: Date.now(), + }; + this.emit('stepBegin', payload); + return step; + }); // Inherit test.setTimeout() from parent suites. for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { @@ -373,40 +259,35 @@ export class WorkerRunner extends EventEmitter { } } - const testData: TestData = { testInfo, testId, type: test._type }; - this._currentTest = testData; + this._currentTest = testInfo; setCurrentTestInfo(testInfo); - this.emit('testBegin', buildTestBeginPayload(testData, startWallTime)); + this.emit('testBegin', buildTestBeginPayload(testInfo)); if (testInfo.expectedStatus === 'skipped') { testInfo.status = 'skipped'; - this.emit('testEnd', buildTestEndPayload(testData)); + this.emit('testEnd', buildTestEndPayload(testInfo)); return; } // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures. this._fixtureRunner.setPool(test._pool!); - this._currentTimeoutRunner = timeoutRunner = new TimeoutRunner(testInfo.timeout); - await this._runWithTimeout(timeoutRunner, () => this._runTestWithBeforeHooks(test, testInfo), testInfo); - testInfo.duration = monotonicTime() - startTime; + await testInfo._runWithTimeout(() => this._runTestWithBeforeHooks(test, testInfo)); if (testInfo.status === 'timedOut') { // A timed-out test gets a full additional timeout to run after hooks. - timeoutRunner.resetTimeout(testInfo.timeout); + testInfo._timeoutRunner.resetTimeout(testInfo.timeout); } - await this._runWithTimeout(timeoutRunner, () => this._runAfterHooks(test, testInfo), testInfo); + await testInfo._runWithTimeout(() => this._runAfterHooks(test, testInfo)); - testInfo.duration = monotonicTime() - startTime; - this._currentTimeoutRunner = undefined; this._currentTest = null; setCurrentTestInfo(null); const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus; if (isFailure) { // Delay reporting testEnd result until after teardownScopes is done. - this._failedTest = testData; + this._failedTest = testInfo; if (test._type !== 'test') { // beforeAll/afterAll hook failure skips any remaining tests in the worker. if (!this._fatalError) @@ -417,7 +298,7 @@ export class WorkerRunner extends EventEmitter { } this.stop(); } else { - this.emit('testEnd', buildTestEndPayload(testData)); + this.emit('testEnd', buildTestEndPayload(testInfo)); } const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' || @@ -433,7 +314,7 @@ export class WorkerRunner extends EventEmitter { canHaveChildren: true, forceNoParent: true }); - const maybeError = await this._runFn(async () => { + const maybeError = await testInfo._runFn(async () => { if (test._type === 'test') { const beforeEachModifiers: Modifier[] = []; for (let s: Suite | undefined = test.parent; s; s = s.parent) { @@ -452,7 +333,7 @@ export class WorkerRunner extends EventEmitter { step.complete(); // Report fixture hooks step as completed. const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]"). await fn(params, testInfo); - }, testInfo, 'allowSkips'); + }, 'allowSkips'); step.complete(maybeError); // Second complete is a no-op. } @@ -466,10 +347,10 @@ export class WorkerRunner extends EventEmitter { let teardownError1: TestError | undefined; if (test._type === 'test') - teardownError1 = await this._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo), testInfo); + teardownError1 = await testInfo._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo)); // Continue teardown even after the failure. - const teardownError2 = await this._runFn(() => this._fixtureRunner.teardownScope('test'), testInfo); + const teardownError2 = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); step.complete(teardownError1 || teardownError2); } @@ -494,40 +375,6 @@ export class WorkerRunner extends EventEmitter { throw error; } - private async _runWithTimeout(timeoutRunner: TimeoutRunner, cb: () => Promise, testInfo: TestInfoImpl): Promise { - try { - await timeoutRunner.run(cb); - } catch (error) { - if (!(error instanceof TimeoutRunnerError)) - throw error; - // Do not overwrite existing failure upon hook/teardown timeout. - if (testInfo.status === 'passed') - testInfo.status = 'timedOut'; - } - } - - private async _runFn(fn: Function, testInfo: TestInfoImpl, skips?: 'allowSkips'): Promise { - try { - await fn(); - } catch (error) { - if (skips === 'allowSkips' && error instanceof SkipError) { - if (testInfo.status === 'passed') - testInfo.status = 'skipped'; - } else { - const serialized = serializeError(error); - // Do not overwrite any previous error and error status. - // Some (but not all) scenarios include: - // - expect() that fails after uncaught exception. - // - fail after the timeout, e.g. due to fixture teardown. - if (testInfo.status === 'passed') - testInfo.status = 'failed'; - if (!('error' in testInfo)) - testInfo.error = serialized; - return serialized; - } - } - } - private _reportDone() { const donePayload: DonePayload = { fatalError: this._fatalError }; this.emit('done', donePayload); @@ -536,17 +383,16 @@ export class WorkerRunner extends EventEmitter { } } -function buildTestBeginPayload(testData: TestData, startWallTime: number): TestBeginPayload { +function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload { return { - testId: testData.testId, - startWallTime, + testId: testInfo._test._id, + startWallTime: testInfo._startWallTime, }; } -function buildTestEndPayload(testData: TestData): TestEndPayload { - const { testId, testInfo } = testData; +function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { return { - testId, + testId: testInfo._test._id, duration: testInfo.duration, status: testInfo.status!, error: testInfo.error, @@ -561,33 +407,3 @@ function buildTestEndPayload(testData: TestData): TestEndPayload { })) }; } - -function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) { - if (typeof modifierArgs[1] === 'function') { - throw new Error([ - 'It looks like you are calling test.skip() inside the test and pass a callback.', - 'Pass a condition instead and optional description instead:', - `test('my test', async ({ page, isMobile }) => {`, - ` test.skip(isMobile, 'This test is not applicable on mobile');`, - `});`, - ].join('\n')); - } - - if (modifierArgs.length >= 1 && !modifierArgs[0]) - return; - - const description = modifierArgs[1]; - testInfo.annotations.push({ type, description }); - if (type === 'slow') { - testInfo.setTimeout(testInfo.timeout * 3); - } else if (type === 'skip' || type === 'fixme') { - testInfo.expectedStatus = 'skipped'; - throw new SkipError('Test is skipped: ' + (description || '')); - } else if (type === 'fail') { - if (testInfo.expectedStatus !== 'skipped') - testInfo.expectedStatus = 'failed'; - } -} - -class SkipError extends Error { -}