chore(test runner): extract TestInfoImpl (#11725)

This commit is contained in:
Dmitry Gozman 2022-01-28 17:39:42 -08:00 committed by GitHub
parent 401cd9c0ee
commit eeebcd53ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 335 additions and 259 deletions

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { TestInfoImpl } from './types';
import type { TestInfoImpl } from './testInfo';
import { Suite } from './test';
let currentTestInfoValue: TestInfoImpl | null = null;

View File

@ -22,10 +22,11 @@ import path from 'path';
import jpeg from 'jpeg-js';
import pixelmatch from 'pixelmatch';
import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
import { TestInfoImpl, UpdateSnapshots } from '../types';
import { UpdateSnapshots } from '../types';
import { addSuffixToFilePath } from '../util';
import BlinkDiff from '../third_party/blink-diff';
import PNGImage from '../third_party/png-js';
import { TestInfoImpl } from '../testInfo';
// Note: we require the pngjs version of pixelmatch to avoid version mismatches.
const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] })) as typeof import('pngjs');
@ -128,12 +129,7 @@ export function compare(
return { pass: true, message };
}
if (updateSnapshots === 'missing') {
if (testInfo.status === 'passed')
testInfo.status = 'failed';
if (!('error' in testInfo))
testInfo.error = { value: 'Error: ' + message };
else if (testInfo.error?.value)
testInfo.error.value += '\nError: ' + message;
testInfo._appendErrorMessage(message);
return { pass: true, message };
}
return { pass: false, message };

View File

@ -0,0 +1,268 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import * as mime from 'mime';
import path from 'path';
import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/async';
import { calculateSha1 } from 'playwright-core/lib/utils/utils';
import type { FullConfig, FullProject, TestError, TestInfo, TestStatus } from '../types/test';
import { WorkerInitParams } from './ipc';
import { Loader } from './loader';
import { ProjectImpl } from './project';
import { TestCase } from './test';
import { Annotations, TestStepInternal } from './types';
import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util';
export class TestInfoImpl implements TestInfo {
private _projectImpl: ProjectImpl;
private _addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal;
readonly _test: TestCase;
readonly _timeoutRunner: TimeoutRunner;
readonly _startTime: number;
readonly _startWallTime: number;
// ------------ TestInfo fields ------------
readonly repeatEachIndex: number;
readonly retry: number;
readonly workerIndex: number;
readonly parallelIndex: number;
readonly project: FullProject;
config: FullConfig;
readonly title: string;
readonly titlePath: string[];
readonly file: string;
readonly line: number;
readonly column: number;
readonly fn: Function;
expectedStatus: TestStatus;
duration: number = 0;
readonly annotations: Annotations = [];
readonly attachments: TestInfo['attachments'] = [];
status: TestStatus = 'passed';
readonly stdout: TestInfo['stdout'] = [];
readonly stderr: TestInfo['stderr'] = [];
timeout: number;
snapshotSuffix: string = '';
readonly outputDir: string;
readonly snapshotDir: string;
error: TestError | undefined = undefined;
constructor(
loader: Loader,
workerParams: WorkerInitParams,
test: TestCase,
retry: number,
addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal,
) {
this._projectImpl = loader.projects()[workerParams.projectIndex];
this._test = test;
this._addStepImpl = addStepImpl;
this._startTime = monotonicTime();
this._startWallTime = Date.now();
this.repeatEachIndex = workerParams.repeatEachIndex;
this.retry = retry;
this.workerIndex = workerParams.workerIndex;
this.parallelIndex = workerParams.parallelIndex;
this.project = this._projectImpl.config;
this.config = loader.fullConfig();
this.title = test.title;
this.titlePath = test.titlePath();
this.file = test.location.file;
this.line = test.location.line;
this.column = test.location.column;
this.fn = test.fn;
this.expectedStatus = test.expectedStatus;
this.timeout = this.project.timeout;
this._timeoutRunner = new TimeoutRunner(this.timeout);
this.outputDir = (() => {
const sameName = loader.projects().filter(project => project.config.name === this.project.name);
let uniqueProjectNamePathSegment: string;
if (sameName.length > 1)
uniqueProjectNamePathSegment = this.project.name + (sameName.indexOf(this._projectImpl) + 1);
else
uniqueProjectNamePathSegment = this.project.name;
const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this.workerIndex);
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec));
if (uniqueProjectNamePathSegment)
testOutputDir += '-' + sanitizeForFilePath(uniqueProjectNamePathSegment);
if (this.retry)
testOutputDir += '-retry' + this.retry;
if (this.repeatEachIndex)
testOutputDir += '-repeat' + this.repeatEachIndex;
return path.join(this.project.outputDir, testOutputDir);
})();
this.snapshotDir = (() => {
const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile);
return path.join(this.project.snapshotDir, relativeTestFilePath + '-snapshots');
})();
}
private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
if (typeof modifierArgs[1] === 'function') {
throw new Error([
'It looks like you are calling test.skip() inside the test and pass a callback.',
'Pass a condition instead and optional description instead:',
`test('my test', async ({ page, isMobile }) => {`,
` test.skip(isMobile, 'This test is not applicable on mobile');`,
`});`,
].join('\n'));
}
if (modifierArgs.length >= 1 && !modifierArgs[0])
return;
const description = modifierArgs[1];
this.annotations.push({ type, description });
if (type === 'slow') {
this.setTimeout(this.timeout * 3);
} else if (type === 'skip' || type === 'fixme') {
this.expectedStatus = 'skipped';
throw new SkipError('Test is skipped: ' + (description || ''));
} else if (type === 'fail') {
if (this.expectedStatus !== 'skipped')
this.expectedStatus = 'failed';
}
}
async _runWithTimeout(cb: () => Promise<any>): Promise<void> {
try {
await this._timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed')
this.status = 'timedOut';
}
this.duration = monotonicTime() - this._startTime;
}
async _runFn(fn: Function, skips?: 'allowSkips'): Promise<TestError | undefined> {
try {
await fn();
} catch (error) {
if (skips === 'allowSkips' && error instanceof SkipError) {
if (this.status === 'passed')
this.status = 'skipped';
} else {
const serialized = serializeError(error);
this._failWithError(serialized);
return serialized;
}
}
}
_addStep(data: Omit<TestStepInternal, 'complete'>) {
return this._addStepImpl(data);
}
_failWithError(error: TestError) {
// Do not overwrite any previous error and error status.
// Some (but not all) scenarios include:
// - expect() that fails after uncaught exception.
// - fail after the timeout, e.g. due to fixture teardown.
if (this.status === 'passed')
this.status = 'failed';
if (this.error === undefined)
this.error = error;
}
_appendErrorMessage(message: string) {
// Do not overwrite any previous error status.
if (this.status === 'passed')
this.status = 'failed';
if (this.error === undefined)
this.error = { value: 'Error: ' + message };
else if (this.error.value)
this.error.value += '\nError: ' + message;
}
// ------------ TestInfo methods ------------
async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {
if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1)
throw new Error(`Exactly one of "path" and "body" must be specified`);
if (options.path !== undefined) {
const hash = calculateSha1(options.path);
const dest = this.outputPath('attachments', hash + path.extname(options.path));
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.copyFile(options.path, dest);
const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream');
this.attachments.push({ name, contentType, path: dest });
} else {
const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream');
this.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body });
}
}
outputPath(...pathSegments: string[]){
fs.mkdirSync(this.outputDir, { recursive: true });
const joinedPath = path.join(...pathSegments);
const outputPath = getContainedPath(this.outputDir, joinedPath);
if (outputPath)
return outputPath;
throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`);
}
snapshotPath(...pathSegments: string[]) {
let suffix = '';
const projectNamePathSegment = sanitizeForFilePath(this.project.name);
if (projectNamePathSegment)
suffix += '-' + projectNamePathSegment;
if (this.snapshotSuffix)
suffix += '-' + this.snapshotSuffix;
const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix);
const snapshotPath = getContainedPath(this.snapshotDir, subPath);
if (snapshotPath)
return snapshotPath;
throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`);
}
skip(...args: [arg?: any, description?: string]) {
this._modifier('skip', args);
}
fixme(...args: [arg?: any, description?: string]) {
this._modifier('fixme', args);
}
fail(...args: [arg?: any, description?: string]) {
this._modifier('fail', args);
}
slow(...args: [arg?: any, description?: string]) {
this._modifier('slow', args);
}
setTimeout(timeout: number) {
if (!this.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
}
class SkipError extends Error {
}

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Fixtures, TestError, TestInfo } from '../types/test';
import type { Fixtures, TestError } from '../types/test';
import type { Location } from '../types/testReporter';
export * from '../types/test';
export { Location } from '../types/testReporter';
@ -34,8 +34,4 @@ export interface TestStepInternal {
location?: Location;
}
export interface TestInfoImpl extends TestInfo {
_addStep: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal;
}
export type TestCaseType = 'beforeAll' | 'afterAll' | 'test';

View File

@ -33,7 +33,7 @@ global.console = new Console({
process.stdout.write = (chunk: string | Buffer) => {
const outPayload: TestOutputPayload = {
testId: workerRunner?._currentTest?.testId,
testId: workerRunner?._currentTest?._test._id,
...chunkToParams(chunk)
};
sendMessageToParent('stdOut', outPayload);
@ -43,7 +43,7 @@ process.stdout.write = (chunk: string | Buffer) => {
if (!process.env.PW_RUNNER_DEBUG) {
process.stderr.write = (chunk: string | Buffer) => {
const outPayload: TestOutputPayload = {
testId: workerRunner?._currentTest?.testId,
testId: workerRunner?._currentTest?._test._id,
...chunkToParams(chunk)
};
sendMessageToParent('stdErr', outPayload);

View File

@ -14,44 +14,36 @@
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import rimraf from 'rimraf';
import * as mime from 'mime';
import util from 'util';
import colors from 'colors/safe';
import { EventEmitter } from 'events';
import { monotonicTime, serializeError, sanitizeForFilePath, getContainedPath, addSuffixToFilePath, prependToTestError, trimLongString, formatLocation } from './util';
import { serializeError, prependToTestError, formatLocation } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
import { setCurrentTestInfo } from './globals';
import { Loader } from './loader';
import { Modifier, Suite, TestCase } from './test';
import { Annotations, TestCaseType, TestError, TestInfo, TestInfoImpl, TestStepInternal, WorkerInfo } from './types';
import { Annotations, TestError, TestInfo, TestStepInternal, WorkerInfo } from './types';
import { ProjectImpl } from './project';
import { FixtureRunner } from './fixtures';
import { TimeoutRunner, raceAgainstTimeout, TimeoutRunnerError } from 'playwright-core/lib/utils/async';
import { calculateSha1 } from 'playwright-core/lib/utils/utils';
import { raceAgainstTimeout } from 'playwright-core/lib/utils/async';
import { TestInfoImpl } from './testInfo';
const removeFolderAsync = util.promisify(rimraf);
type TestData = { testId: string, testInfo: TestInfoImpl, type: TestCaseType };
export class WorkerRunner extends EventEmitter {
private _params: WorkerInitParams;
private _loader!: Loader;
private _project!: ProjectImpl;
private _workerInfo!: WorkerInfo;
private _projectNamePathSegment = '';
private _uniqueProjectNamePathSegment = '';
private _fixtureRunner: FixtureRunner;
private _failedTest: TestData | undefined;
private _failedTest: TestInfoImpl | undefined;
private _fatalError: TestError | undefined;
private _entries = new Map<string, TestEntry>();
private _isStopped = false;
private _runFinished = Promise.resolve();
private _currentTimeoutRunner: TimeoutRunner | undefined;
_currentTest: TestData | null = null;
_currentTest: TestInfoImpl | null = null;
constructor(params: WorkerInitParams) {
super();
@ -64,11 +56,11 @@ export class WorkerRunner extends EventEmitter {
this._isStopped = true;
// Interrupt current action.
this._currentTimeoutRunner?.interrupt();
this._currentTest?._timeoutRunner.interrupt();
// TODO: mark test as 'interrupted' instead.
if (this._currentTest && this._currentTest.testInfo.status === 'passed')
this._currentTest.testInfo.status = 'skipped';
if (this._currentTest && this._currentTest.status === 'passed')
this._currentTest.status = 'skipped';
}
return this._runFinished;
}
@ -102,11 +94,8 @@ export class WorkerRunner extends EventEmitter {
// a test runner. In the latter case, the worker state could be messed up,
// and continuing to run tests in the same worker is problematic. Therefore,
// we turn this into a fatal error and restart the worker anyway.
if (this._currentTest && this._currentTest.type === 'test' && this._currentTest.testInfo.expectedStatus !== 'failed') {
if (!this._currentTest.testInfo.error) {
this._currentTest.testInfo.status = 'failed';
this._currentTest.testInfo.error = serializeError(error);
}
if (this._currentTest && this._currentTest._test._type === 'test' && this._currentTest.expectedStatus !== 'failed') {
this._currentTest._failWithError(serializeError(error));
} else {
// No current test - fatal error.
if (!this._fatalError)
@ -122,15 +111,6 @@ export class WorkerRunner extends EventEmitter {
this._loader = await Loader.deserialize(this._params.loader);
this._project = this._loader.projects()[this._params.projectIndex];
this._projectNamePathSegment = sanitizeForFilePath(this._project.config.name);
const sameName = this._loader.projects().filter(project => project.config.name === this._project.config.name);
if (sameName.length > 1)
this._uniqueProjectNamePathSegment = this._project.config.name + (sameName.indexOf(this._project) + 1);
else
this._uniqueProjectNamePathSegment = this._project.config.name;
this._uniqueProjectNamePathSegment = sanitizeForFilePath(this._uniqueProjectNamePathSegment);
this._workerInfo = {
workerIndex: this._params.workerIndex,
parallelIndex: this._params.parallelIndex,
@ -218,134 +198,40 @@ export class WorkerRunner extends EventEmitter {
}
private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) {
const startTime = monotonicTime();
const startWallTime = Date.now();
let timeoutRunner: TimeoutRunner;
const testId = test._id;
const baseOutputDir = (() => {
const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this._params.workerIndex);
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec));
if (this._uniqueProjectNamePathSegment)
testOutputDir += '-' + this._uniqueProjectNamePathSegment;
if (retry)
testOutputDir += '-retry' + retry;
if (this._params.repeatEachIndex)
testOutputDir += '-repeat' + this._params.repeatEachIndex;
return path.join(this._project.config.outputDir, testOutputDir);
})();
const snapshotDir = (() => {
const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile);
return path.join(this._project.config.snapshotDir, relativeTestFilePath + '-snapshots');
})();
let lastStepId = 0;
const testInfo: TestInfoImpl = {
workerIndex: this._params.workerIndex,
parallelIndex: this._params.parallelIndex,
project: this._project.config,
config: this._loader.fullConfig(),
title: test.title,
titlePath: test.titlePath(),
file: test.location.file,
line: test.location.line,
column: test.location.column,
fn: test.fn,
repeatEachIndex: this._params.repeatEachIndex,
retry,
expectedStatus: test.expectedStatus,
annotations: [],
attachments: [],
attach: async (name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) => {
if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1)
throw new Error(`Exactly one of "path" and "body" must be specified`);
if (options.path !== undefined) {
const hash = calculateSha1(options.path);
const dest = testInfo.outputPath('attachments', hash + path.extname(options.path));
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.copyFile(options.path, dest);
const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream');
testInfo.attachments.push({ name, contentType, path: dest });
} else {
const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream');
testInfo.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body });
const testInfo = new TestInfoImpl(this._loader, this._params, test, retry, data => {
const stepId = `${data.category}@${data.title}@${++lastStepId}`;
let callbackHandled = false;
const step: TestStepInternal = {
...data,
complete: (error?: Error | TestError) => {
if (callbackHandled)
return;
callbackHandled = true;
if (error instanceof Error)
error = serializeError(error);
const payload: StepEndPayload = {
testId: test._id,
stepId,
wallTime: Date.now(),
error,
};
this.emit('stepEnd', payload);
}
},
duration: 0,
status: 'passed',
stdout: [],
stderr: [],
timeout: this._project.config.timeout,
snapshotSuffix: '',
outputDir: baseOutputDir,
snapshotDir,
outputPath: (...pathSegments: string[]): string => {
fs.mkdirSync(baseOutputDir, { recursive: true });
const joinedPath = path.join(...pathSegments);
const outputPath = getContainedPath(baseOutputDir, joinedPath);
if (outputPath) return outputPath;
throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`);
},
snapshotPath: (...pathSegments: string[]): string => {
let suffix = '';
if (this._projectNamePathSegment)
suffix += '-' + this._projectNamePathSegment;
if (testInfo.snapshotSuffix)
suffix += '-' + testInfo.snapshotSuffix;
const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix);
const snapshotPath = getContainedPath(snapshotDir, subPath);
if (snapshotPath) return snapshotPath;
throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`);
},
skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args),
fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args),
fail: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fail', args),
slow: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'slow', args),
setTimeout: (timeout: number) => {
if (!testInfo.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
testInfo.timeout = timeout;
if (timeoutRunner)
timeoutRunner.updateTimeout(timeout);
},
_addStep: data => {
const stepId = `${data.category}@${data.title}@${++lastStepId}`;
let callbackHandled = false;
const step: TestStepInternal = {
...data,
complete: (error?: Error | TestError) => {
if (callbackHandled)
return;
callbackHandled = true;
if (error instanceof Error)
error = serializeError(error);
const payload: StepEndPayload = {
testId,
stepId,
wallTime: Date.now(),
error,
};
this.emit('stepEnd', payload);
}
};
const hasLocation = data.location && !data.location.file.includes('@playwright');
// Sanitize location that comes from user land, it might have extra properties.
const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined;
const payload: StepBeginPayload = {
testId,
stepId,
...data,
location,
wallTime: Date.now(),
};
this.emit('stepBegin', payload);
return step;
},
};
};
const hasLocation = data.location && !data.location.file.includes('@playwright');
// Sanitize location that comes from user land, it might have extra properties.
const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined;
const payload: StepBeginPayload = {
testId: test._id,
stepId,
...data,
location,
wallTime: Date.now(),
};
this.emit('stepBegin', payload);
return step;
});
// Inherit test.setTimeout() from parent suites.
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
@ -373,40 +259,35 @@ export class WorkerRunner extends EventEmitter {
}
}
const testData: TestData = { testInfo, testId, type: test._type };
this._currentTest = testData;
this._currentTest = testInfo;
setCurrentTestInfo(testInfo);
this.emit('testBegin', buildTestBeginPayload(testData, startWallTime));
this.emit('testBegin', buildTestBeginPayload(testInfo));
if (testInfo.expectedStatus === 'skipped') {
testInfo.status = 'skipped';
this.emit('testEnd', buildTestEndPayload(testData));
this.emit('testEnd', buildTestEndPayload(testInfo));
return;
}
// Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
this._fixtureRunner.setPool(test._pool!);
this._currentTimeoutRunner = timeoutRunner = new TimeoutRunner(testInfo.timeout);
await this._runWithTimeout(timeoutRunner, () => this._runTestWithBeforeHooks(test, testInfo), testInfo);
testInfo.duration = monotonicTime() - startTime;
await testInfo._runWithTimeout(() => this._runTestWithBeforeHooks(test, testInfo));
if (testInfo.status === 'timedOut') {
// A timed-out test gets a full additional timeout to run after hooks.
timeoutRunner.resetTimeout(testInfo.timeout);
testInfo._timeoutRunner.resetTimeout(testInfo.timeout);
}
await this._runWithTimeout(timeoutRunner, () => this._runAfterHooks(test, testInfo), testInfo);
await testInfo._runWithTimeout(() => this._runAfterHooks(test, testInfo));
testInfo.duration = monotonicTime() - startTime;
this._currentTimeoutRunner = undefined;
this._currentTest = null;
setCurrentTestInfo(null);
const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus;
if (isFailure) {
// Delay reporting testEnd result until after teardownScopes is done.
this._failedTest = testData;
this._failedTest = testInfo;
if (test._type !== 'test') {
// beforeAll/afterAll hook failure skips any remaining tests in the worker.
if (!this._fatalError)
@ -417,7 +298,7 @@ export class WorkerRunner extends EventEmitter {
}
this.stop();
} else {
this.emit('testEnd', buildTestEndPayload(testData));
this.emit('testEnd', buildTestEndPayload(testInfo));
}
const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' ||
@ -433,7 +314,7 @@ export class WorkerRunner extends EventEmitter {
canHaveChildren: true,
forceNoParent: true
});
const maybeError = await this._runFn(async () => {
const maybeError = await testInfo._runFn(async () => {
if (test._type === 'test') {
const beforeEachModifiers: Modifier[] = [];
for (let s: Suite | undefined = test.parent; s; s = s.parent) {
@ -452,7 +333,7 @@ export class WorkerRunner extends EventEmitter {
step.complete(); // Report fixture hooks step as completed.
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
await fn(params, testInfo);
}, testInfo, 'allowSkips');
}, 'allowSkips');
step.complete(maybeError); // Second complete is a no-op.
}
@ -466,10 +347,10 @@ export class WorkerRunner extends EventEmitter {
let teardownError1: TestError | undefined;
if (test._type === 'test')
teardownError1 = await this._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo), testInfo);
teardownError1 = await testInfo._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo));
// Continue teardown even after the failure.
const teardownError2 = await this._runFn(() => this._fixtureRunner.teardownScope('test'), testInfo);
const teardownError2 = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test'));
step.complete(teardownError1 || teardownError2);
}
@ -494,40 +375,6 @@ export class WorkerRunner extends EventEmitter {
throw error;
}
private async _runWithTimeout(timeoutRunner: TimeoutRunner, cb: () => Promise<any>, testInfo: TestInfoImpl): Promise<void> {
try {
await timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
// Do not overwrite existing failure upon hook/teardown timeout.
if (testInfo.status === 'passed')
testInfo.status = 'timedOut';
}
}
private async _runFn(fn: Function, testInfo: TestInfoImpl, skips?: 'allowSkips'): Promise<TestError | undefined> {
try {
await fn();
} catch (error) {
if (skips === 'allowSkips' && error instanceof SkipError) {
if (testInfo.status === 'passed')
testInfo.status = 'skipped';
} else {
const serialized = serializeError(error);
// Do not overwrite any previous error and error status.
// Some (but not all) scenarios include:
// - expect() that fails after uncaught exception.
// - fail after the timeout, e.g. due to fixture teardown.
if (testInfo.status === 'passed')
testInfo.status = 'failed';
if (!('error' in testInfo))
testInfo.error = serialized;
return serialized;
}
}
}
private _reportDone() {
const donePayload: DonePayload = { fatalError: this._fatalError };
this.emit('done', donePayload);
@ -536,17 +383,16 @@ export class WorkerRunner extends EventEmitter {
}
}
function buildTestBeginPayload(testData: TestData, startWallTime: number): TestBeginPayload {
function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload {
return {
testId: testData.testId,
startWallTime,
testId: testInfo._test._id,
startWallTime: testInfo._startWallTime,
};
}
function buildTestEndPayload(testData: TestData): TestEndPayload {
const { testId, testInfo } = testData;
function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
return {
testId,
testId: testInfo._test._id,
duration: testInfo.duration,
status: testInfo.status!,
error: testInfo.error,
@ -561,33 +407,3 @@ function buildTestEndPayload(testData: TestData): TestEndPayload {
}))
};
}
function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
if (typeof modifierArgs[1] === 'function') {
throw new Error([
'It looks like you are calling test.skip() inside the test and pass a callback.',
'Pass a condition instead and optional description instead:',
`test('my test', async ({ page, isMobile }) => {`,
` test.skip(isMobile, 'This test is not applicable on mobile');`,
`});`,
].join('\n'));
}
if (modifierArgs.length >= 1 && !modifierArgs[0])
return;
const description = modifierArgs[1];
testInfo.annotations.push({ type, description });
if (type === 'slow') {
testInfo.setTimeout(testInfo.timeout * 3);
} else if (type === 'skip' || type === 'fixme') {
testInfo.expectedStatus = 'skipped';
throw new SkipError('Test is skipped: ' + (description || ''));
} else if (type === 'fail') {
if (testInfo.expectedStatus !== 'skipped')
testInfo.expectedStatus = 'failed';
}
}
class SkipError extends Error {
}