microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-01-05 19:04:43 +03:00
Code Issues Projects Releases Wiki Activity

chore: extract task runner for global setup (#20345)

Browse Source
This commit is contained in:
Pavel Feldman 2023-01-25 15:38:23 -08:00 committed by GitHub
parent cab52cded9
commit fe1dd7818d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 404 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/src/test-reporter-api/class-reporter.md
View File

@ -79,13 +79,11 @@ export default defineConfig({
```
Here is a typical order of reporter calls:
* [`method: Reporter.onConfigure`] is called once config has been resolved.
* [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
* [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes.
* [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet.
* [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more.
* [`method: Reporter.onEnd`] is called once after all tests that should run had finished.
* [`method: Reporter.onExit`] is called before test runner exits.
Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution,
and [`method: Reporter.onError`] is called when something went wrong outside of the test execution.
@ -109,17 +107,6 @@ Resolved configuration.
The root suite that contains all projects, files and test cases.
## optional method: Reporter.onConfigure
* since: v1.30
Called once config is resolved.
### param: Reporter.onConfigure.config
* since: v1.30
- `config` <[TestConfig]>
Resolved configuration.
## optional async method: Reporter.onEnd
* since: v1.10
@ -136,11 +123,6 @@ Result of the full test run.
* `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached.
* `'interrupted'` - Interrupted by the user.
## optional method: Reporter.onExit
* since: v1.30
Called before test runner exits.
## optional method: Reporter.onError
* since: v1.10

6
packages/playwright-test/src/cli.ts
View File

@ -170,7 +170,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
const grepInvertMatcher = opts.grepInvert ? createTitleMatcher(forceRegExp(opts.grepInvert)) : () => false;
const testTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title);
const result = await runner.runAllTests({
const status = await runner.runAllTests({
listOnly: !!opts.list,
testFileFilters,
testTitleMatcher,
@ -179,9 +179,9 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
});
await stopProfiling(undefined);
if (result.status === 'interrupted')
if (status === 'interrupted')
process.exit(130);
process.exit(result.status === 'passed' ? 0 : 1);
process.exit(status === 'passed' ? 0 : 1);
}

10
packages/playwright-test/src/globals.ts
View File

@ -35,14 +35,14 @@ export function currentlyLoadingFileSuite() {
return currentFileSuite;
}
let _fatalErrors: TestError[] | undefined;
export function setFatalErrorSink(fatalErrors: TestError[]) {
_fatalErrors = fatalErrors;
let _fatalErrorSink: ((fatalError: TestError) => void) | undefined;
export function setFatalErrorSink(fatalErrorSink: (fatalError: TestError) => void) {
_fatalErrorSink = fatalErrorSink;
}
export function addFatalError(message: string, location: Location) {
if (_fatalErrors)
_fatalErrors.push({ message: `Error: ${message}`, location });
if (_fatalErrorSink)
_fatalErrorSink({ message: `Error: ${message}`, location });
else
throw new Error(`${formatLocation(location)}: ${message}`);
}

6
packages/playwright-test/src/loaderHost.ts
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { TestError } from '../reporter';
import type { Reporter, TestError } from '../reporter';
import type { SerializedConfig } from './ipc';
import { ProcessHost } from './processHost';
import { Suite } from './test';
@ -28,9 +28,9 @@ export class LoaderHost extends ProcessHost {
await this.startRunner(config, true, {});
}
async loadTestFiles(files: string[], loadErrors: TestError[]): Promise<Suite> {
async loadTestFiles(files: string[], reporter: Reporter): Promise<Suite> {
const result = await this.sendMessage({ method: 'loadTestFiles', params: { files } }) as any;
loadErrors.push(...result.loadErrors);
result.loadErrors.forEach((e: TestError) => reporter.onError?.(e));
return Suite._deepParse(result.rootSuite);
}
}

2
packages/playwright-test/src/loaderMain.ts
View File

@ -38,7 +38,7 @@ export class LoaderMain extends ProcessRunner {
async loadTestFiles(params: { files: string[] }) {
const loadErrors: TestError[] = [];
setFatalErrorSink(loadErrors);
setFatalErrorSink(error => loadErrors.push(error));
const configLoader = await this._configLoader();
const rootSuite = await loadTestFilesInProcess(configLoader.fullConfig(), params.files, loadErrors);
return { rootSuite: rootSuite._deepSerialize(), loadErrors };

3
packages/playwright-test/src/reporters/DEPS.list
View File

@ -1,4 +1,5 @@
[*]
../util.ts
../babelBundle.ts
../test.ts
../util.ts
../utilsBundle.ts

5
packages/playwright-test/src/reporters/base.ts
View File

@ -63,12 +63,9 @@ export class BaseReporter implements Reporter {
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
}
onConfigure(config: FullConfig) {
onBegin(config: FullConfig, suite: Suite) {
this.monotonicStartTime = monotonicTime();
this.config = config as FullConfigInternal;
}
onBegin(config: FullConfig, suite: Suite) {
this.suite = suite;
this.totalTestCount = suite.allTests().length;
}

9
packages/playwright-test/src/reporters/html.ts
View File

@ -64,12 +64,9 @@ class HtmlReporter implements Reporter {
return false;
}
onConfigure(config: FullConfig) {
onBegin(config: FullConfig, suite: Suite) {
this._montonicStartTime = monotonicTime();
this.config = config as FullConfigInternal;
}
onBegin(config: FullConfig, suite: Suite) {
const { outputFolder, open } = this._resolveOptions();
this._outputFolder = outputFolder;
this._open = open;
@ -116,10 +113,10 @@ class HtmlReporter implements Reporter {
}
async onExit() {
if (process.env.CI)
if (process.env.CI || !this._buildResult)
return;
const { ok, singleTestId } = this._buildResult!;
const { ok, singleTestId } = this._buildResult;
const shouldOpen = this._open === 'always' || (!ok && this._open === 'on-failure');
if (shouldOpen) {
await showHTMLReport(this._outputFolder, this._options.host, this._options.port, singleTestId);

5
packages/playwright-test/src/reporters/json.ts
View File

@ -39,11 +39,8 @@ class JSONReporter implements Reporter {
return !this._outputFile;
}
onConfigure(config: FullConfig) {
this.config = config;
}
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
}

5
packages/playwright-test/src/reporters/junit.ts
View File

@ -48,11 +48,8 @@ class JUnitReporter implements Reporter {
return !this.outputFile;
}
onConfigure(config: FullConfig) {
this.config = config;
}
onBegin(config: FullConfig, suite: Suite) {
this.config = config;
this.suite = suite;
this.timestamp = Date.now();
this.startTime = monotonicTime();

59
packages/playwright-test/src/reporters/multiplexer.ts
View File

@ -14,10 +14,22 @@
* limitations under the License.
*/
import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter';
import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter';
import { Suite } from '../test';
type StdIOChunk = {
type: 'stdout' | 'stderr';
chunk: string | Buffer;
test?: TestCase;
result?: TestResult;
};
export class Multiplexer implements Reporter {
private _reporters: Reporter[];
private _deferredErrors: TestError[] | null = [];
private _deferredStdIO: StdIOChunk[] | null = [];
hasErrors = false;
private _config!: FullConfig;
constructor(reporters: Reporter[]) {
this._reporters = reporters;
@ -28,13 +40,26 @@ export class Multiplexer implements Reporter {
}
onConfigure(config: FullConfig) {
for (const reporter of this._reporters)
reporter.onConfigure?.(config);
this._config = config;
}
onBegin(config: FullConfig, suite: Suite) {
for (const reporter of this._reporters)
reporter.onBegin?.(config, suite);
const errors = this._deferredErrors!;
this._deferredErrors = null;
for (const error of errors)
this.onError(error);
const stdios = this._deferredStdIO!;
this._deferredStdIO = null;
for (const stdio of stdios) {
if (stdio.type === 'stdout')
this.onStdOut(stdio.chunk, stdio.test, stdio.result);
else
this.onStdErr(stdio.chunk, stdio.test, stdio.result);
}
}
onTestBegin(test: TestCase, result: TestResult) {
@ -43,11 +68,20 @@ export class Multiplexer implements Reporter {
}
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
if (this._deferredStdIO) {
this._deferredStdIO.push({ chunk, test, result, type: 'stdout' });
return;
}
for (const reporter of this._reporters)
wrap(() => reporter.onStdOut?.(chunk, test, result));
}
onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
if (this._deferredStdIO) {
this._deferredStdIO.push({ chunk, test, result, type: 'stderr' });
return;
}
for (const reporter of this._reporters)
wrap(() => reporter.onStdErr?.(chunk, test, result));
}
@ -57,17 +91,28 @@ export class Multiplexer implements Reporter {
wrap(() => reporter.onTestEnd?.(test, result));
}
async onEnd(result: FullResult) {
async onEnd() { }
async onExit(result: FullResult) {
if (this._deferredErrors) {
// onBegin was not reported, emit it.
this.onBegin(this._config, new Suite('', 'root'));
}
for (const reporter of this._reporters)
await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e));
}
async onExit() {
for (const reporter of this._reporters)
await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e));
await Promise.resolve().then(() => (reporter as any).onExit?.()).catch(e => console.error('Error in reporter', e));
}
onError(error: TestError) {
this.hasErrors = true;
if (this._deferredErrors) {
this._deferredErrors.push(error);
return;
}
for (const reporter of this._reporters)
wrap(() => reporter.onError?.(error));
}

363
packages/playwright-test/src/runner.ts
View File

@ -17,7 +17,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { monotonicTime, raceAgainstTimeout } from 'playwright-core/lib/utils';
import { monotonicTime } from 'playwright-core/lib/utils';
import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle';
import { promisify } from 'util';
import type { FullResult, Reporter, TestError } from '../types/testReporter';
@ -38,7 +38,6 @@ import JUnitReporter from './reporters/junit';
import LineReporter from './reporters/line';
import ListReporter from './reporters/list';
import { Multiplexer } from './reporters/multiplexer';
import { SigIntWatcher } from './sigIntWatcher';
import type { TestCase } from './test';
import { Suite } from './test';
import type { Config, FullConfigInternal, FullProjectInternal } from './types';
@ -48,6 +47,7 @@ import { setFatalErrorSink } from './globals';
import { buildFileSuiteForProject, filterOnly, filterSuite, filterSuiteWithOnlySemantics, filterTestsRemoveEmptySuites } from './suiteUtils';
import { LoaderHost } from './loaderHost';
import { loadTestFilesInProcess } from './testLoader';
import { TaskRunner } from './taskRunner';
const removeFolderAsync = promisify(rimraf);
const readDirAsync = promisify(fs.readdir);
@ -55,7 +55,7 @@ const readFileAsync = promisify(fs.readFile);
export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
type RunOptions = {
listOnly?: boolean;
listOnly: boolean;
testFileFilters: TestFileFilter[];
testTitleMatcher: Matcher;
projectFilter?: string[];
@ -83,14 +83,12 @@ export type ConfigCLIOverrides = {
export class Runner {
private _configLoader: ConfigLoader;
private _reporter!: Reporter;
private _reporter!: Multiplexer;
private _plugins: TestRunnerPlugin[] = [];
private _fatalErrors: TestError[] = [];
constructor(configCLIOverrides?: ConfigCLIOverrides) {
this._configLoader = new ConfigLoader(configCLIOverrides);
setRunnerToAddPluginsTo(this);
setFatalErrorSink(this._fatalErrors);
}
addPlugin(plugin: TestRunnerPlugin) {
@ -223,7 +221,6 @@ export class Runner {
const filesByProject = new Map<FullProjectInternal, string[]>();
const fileToProjectName = new Map<string, string>();
const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true;
for (const project of projects) {
const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
const testMatch = createFileMatcher(project.testMatch);
@ -243,18 +240,10 @@ export class Runner {
return filesByProject;
}
private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, testGroups: TestGroup[] }> {
private async _loadAllTests(options: RunOptions): Promise<{ rootSuite: Suite, testGroups: TestGroup[] }> {
const config = this._configLoader.fullConfig();
const projects = this._collectProjects(options.projectFilter);
const filesByProject = await this._collectFiles(projects, options.testFileFilters);
const rootSuite = await this._createFilteredRootSuite(options, filesByProject);
const testGroups = createTestGroups(rootSuite.suites, config.workers);
return { rootSuite, testGroups };
}
private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>): Promise<Suite> {
const config = this._configLoader.fullConfig();
const allTestFiles = new Set<string>();
for (const files of filesByProject.values())
files.forEach(file => allTestFiles.add(file));
@ -263,7 +252,7 @@ export class Runner {
const preprocessRoot = await this._loadTests(allTestFiles);
// Complain about duplicate titles.
this._fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot));
createDuplicateTitlesErrors(config, preprocessRoot).forEach(e => this._reporter.onError(e));
// Filter tests to respect line/column filter.
filterByFocusedLine(preprocessRoot, options.testFileFilters);
@ -272,13 +261,24 @@ export class Runner {
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
this._fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites));
createForbidOnlyErrors(config, onlyTestsAndSuites).forEach(e => this._reporter.onError(e));
}
// Filter only.
if (!options.listOnly)
filterOnly(preprocessRoot);
const rootSuite = await this._createRootSuite(preprocessRoot, options, filesByProject);
// Do not create test groups when listing.
if (options.listOnly)
return { rootSuite, testGroups: [] };
const testGroups = createTestGroups(rootSuite.suites, config.workers);
return { rootSuite, testGroups };
}
private async _createRootSuite(preprocessRoot: Suite, options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>): Promise<Suite> {
// Generate projects.
const fileSuites = new Map<string, Suite>();
for (const fileSuite of preprocessRoot.suites)
@ -321,12 +321,17 @@ export class Runner {
const loaderHost = new LoaderHost();
await loaderHost.start(this._configLoader.serializedConfig());
try {
return await loaderHost.loadTestFiles([...testFiles], this._fatalErrors);
return await loaderHost.loadTestFiles([...testFiles], this._reporter);
} finally {
await loaderHost.stop();
}
}
return loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], this._fatalErrors);
const loadErrors: TestError[] = [];
try {
return await loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], loadErrors);
} finally {
loadErrors.forEach(e => this._reporter.onError(e));
}
}
private _filterForCurrentShard(rootSuite: Suite, testGroups: TestGroup[]) {
@ -377,147 +382,131 @@ export class Runner {
}
}
async runAllTests(options: RunOptions): Promise<FullResult> {
this._reporter = await this._createReporter(!!options.listOnly);
async runAllTests(options: RunOptions): Promise<FullResult['status']> {
const config = this._configLoader.fullConfig();
const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 1 << 30;
// Legacy webServer support.
this._plugins.push(...webServerPluginsForConfig(config));
// Docker support.
this._plugins.push(dockerPlugin);
// Run configure.
this._reporter.onConfigure?.(config);
this._reporter = await this._createReporter(options.listOnly);
setFatalErrorSink(error => this._reporter.onError(error));
const taskRunner = new TaskRunner(this._reporter, config.globalTimeout);
// Run global setup.
let globalTearDown: (() => Promise<void>) | undefined;
{
const remainingTime = deadline - monotonicTime();
const raceResult = await raceAgainstTimeout(async () => {
const result: FullResult = { status: 'passed' };
globalTearDown = await this._performGlobalSetup(config, result);
return result;
}, remainingTime);
let result: FullResult;
if (raceResult.timedOut) {
this._reporter.onError?.(createStacklessError(
`Timed out waiting ${config.globalTimeout / 1000}s for the global setup to run`));
result = { status: 'timedout' } as FullResult;
} else {
result = raceResult.result;
}
if (result.status !== 'passed')
return result;
// Setup the plugins.
for (const plugin of this._plugins) {
taskRunner.addTask('plugin setup', async () => {
await plugin.setup?.(config, config._configDir, this._reporter);
return () => plugin.teardown?.();
});
}
// Run the tests.
let fullResult: FullResult;
{
const remainingTime = deadline - monotonicTime();
const raceResult = await raceAgainstTimeout(async () => {
try {
return await this._innerRun(options);
} catch (e) {
this._reporter.onError?.(serializeError(e));
return { status: 'failed' } as FullResult;
} finally {
await globalTearDown?.();
// Run global setup & teardown.
if (config.globalSetup || config.globalTeardown) {
taskRunner.addTask('global setup', async () => {
const setupHook = config.globalSetup ? await this._configLoader.loadGlobalHook(config.globalSetup) : undefined;
const teardownHook = config.globalTeardown ? await this._configLoader.loadGlobalHook(config.globalTeardown) : undefined;
const globalSetupResult = setupHook ? await setupHook(this._configLoader.fullConfig()) : undefined;
return async () => {
if (typeof globalSetupResult === 'function')
await globalSetupResult();
await teardownHook?.(config);
};
});
}
let status: FullResult['status'] = 'passed';
// Load tests.
let loadedTests!: { rootSuite: Suite, testGroups: TestGroup[] };
taskRunner.addTask('load tests', async () => {
loadedTests = await this._loadAllTests(options);
if (this._reporter.hasErrors) {
status = 'failed';
taskRunner.stop();
return;
}
// Fail when no tests.
if (!loadedTests.rootSuite.allTests().length && !options.passWithNoTests) {
this._reporter.onError(createNoTestsError());
status = 'failed';
taskRunner.stop();
return;
}
if (!options.listOnly) {
this._filterForCurrentShard(loadedTests.rootSuite, loadedTests.testGroups);
config._maxConcurrentTestGroups = loadedTests.testGroups.length;
}
});
if (!options.listOnly) {
taskRunner.addTask('prepare to run', async () => {
// Remove output directores.
if (!await this._removeOutputDirs(options)) {
status = 'failed';
taskRunner.stop();
return;
}
}, remainingTime);
});
// If timed out, bail.
let result: FullResult;
if (raceResult.timedOut) {
this._reporter.onError?.(createStacklessError(
`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
result = { status: 'timedout' };
} else {
result = raceResult.result;
}
// Report end.
await this._reporter.onEnd?.(result);
fullResult = result;
taskRunner.addTask('plugin begin', async () => {
for (const plugin of this._plugins)
await plugin.begin?.(loadedTests.rootSuite);
});
}
taskRunner.addTask('report begin', async () => {
this._reporter.onBegin?.(config, loadedTests.rootSuite);
return async () => {
await this._reporter.onEnd();
};
});
if (!options.listOnly) {
let dispatcher: Dispatcher;
taskRunner.addTask('setup workers', async () => {
const { rootSuite, testGroups } = loadedTests;
if (config._ignoreSnapshots) {
this._reporter.onStdOut(colors.dim([
'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:',
'- expect().toMatchSnapshot()',
'- expect().toHaveScreenshot()',
'',
].join('\n')));
}
dispatcher = new Dispatcher(this._configLoader, testGroups, this._reporter);
return async () => {
// Stop will stop workers and mark some tests as interrupted.
await dispatcher.stop();
if (dispatcher.hasWorkerErrors() || rootSuite.allTests().some(test => !test.ok()))
status = 'failed';
};
});
taskRunner.addTask('test suite', async () => {
await dispatcher.run();
});
}
const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 0;
this._reporter.onConfigure(config);
const taskStatus = await taskRunner.run(deadline);
if (status === 'passed' && taskStatus !== 'passed')
status = taskStatus;
await this._reporter.onExit({ status });
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
// See https://github.com/nodejs/node/issues/12921
await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
await this._reporter.onExit?.();
return fullResult;
}
private async _innerRun(options: RunOptions): Promise<FullResult> {
const config = this._configLoader.fullConfig();
// Each entry is an array of test groups that can be run concurrently. All
// test groups from the previos entries must finish before entry starts.
const { rootSuite, testGroups } = await this._collectTestGroups(options);
// Fail when no tests.
if (!rootSuite.allTests().length && !options.passWithNoTests)
this._fatalErrors.push(createNoTestsError());
this._filterForCurrentShard(rootSuite, testGroups);
config._maxConcurrentTestGroups = testGroups.length;
const result: FullResult = { status: 'passed' };
for (const plugin of this._plugins)
await plugin.begin?.(rootSuite);
// Report begin
this._reporter.onBegin?.(config, rootSuite);
// Bail out on errors prior to running global setup.
if (this._fatalErrors.length) {
for (const error of this._fatalErrors)
this._reporter.onError?.(error);
return { status: 'failed' };
}
// Bail out if list mode only, don't do any work.
if (options.listOnly)
return { status: 'passed' };
// Remove output directores.
if (!await this._removeOutputDirs(options))
return { status: 'failed' };
if (config._ignoreSnapshots) {
this._reporter.onStdOut?.(colors.dim([
'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:',
'- expect().toMatchSnapshot()',
'- expect().toHaveScreenshot()',
'',
].join('\n')));
}
// Run tests.
const dispatchResult = await this._dispatchToWorkers(testGroups);
if (dispatchResult === 'signal') {
result.status = 'interrupted';
} else {
const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok());
result.status = failed ? 'failed' : 'passed';
}
return result;
}
private async _dispatchToWorkers(stageGroups: TestGroup[]): Promise<'success'|'signal'|'workererror'> {
const dispatcher = new Dispatcher(this._configLoader, [...stageGroups], this._reporter);
const sigintWatcher = new SigIntWatcher();
await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
if (!sigintWatcher.hadSignal()) {
// We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
// as soon as we can.
sigintWatcher.disarm();
}
await dispatcher.stop();
if (sigintWatcher.hadSignal())
return 'signal';
if (dispatcher.hasWorkerErrors())
return 'workererror';
return 'success';
return status;
}
private async _removeOutputDirs(options: RunOptions): Promise<boolean> {
@ -541,89 +530,11 @@ export class Runner {
}
})));
} catch (e) {
this._reporter.onError?.(serializeError(e));
this._reporter.onError(serializeError(e));
return false;
}
return true;
}
private async _performGlobalSetup(config: FullConfigInternal, result: FullResult): Promise<(() => Promise<void>) | undefined> {
let globalSetupResult: any = undefined;
const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
const sigintWatcher = new SigIntWatcher();
const tearDown = async () => {
await this._runAndReportError(async () => {
if (globalSetupResult && typeof globalSetupResult === 'function')
await globalSetupResult(this._configLoader.fullConfig());
}, result);
await this._runAndReportError(async () => {
if (globalSetupResult && config.globalTeardown)
await (await this._configLoader.loadGlobalHook(config.globalTeardown))(this._configLoader.fullConfig());
}, result);
for (const plugin of pluginsThatWereSetUp.reverse()) {
await this._runAndReportError(async () => {
await plugin.teardown?.();
}, result);
}
};
// Legacy webServer support.
this._plugins.push(...webServerPluginsForConfig(config));
// Docker support.
this._plugins.push(dockerPlugin);
await this._runAndReportError(async () => {
// First run the plugins, if plugin is a web server we want it to run before the
// config's global setup.
for (const plugin of this._plugins) {
await Promise.race([
plugin.setup?.(config, config._configDir, this._reporter),
sigintWatcher.promise(),
]);
if (sigintWatcher.hadSignal())
break;
pluginsThatWereSetUp.push(plugin);
}
// Then do global setup.
if (!sigintWatcher.hadSignal()) {
if (config.globalSetup) {
const hook = await this._configLoader.loadGlobalHook(config.globalSetup);
await Promise.race([
Promise.resolve().then(() => hook(this._configLoader.fullConfig())).then((r: any) => globalSetupResult = r || '<noop>'),
sigintWatcher.promise(),
]);
} else {
// Make sure we run the teardown.
globalSetupResult = '<noop>';
}
}
}, result);
sigintWatcher.disarm();
if (result.status !== 'passed' || sigintWatcher.hadSignal()) {
await tearDown();
result.status = sigintWatcher.hadSignal() ? 'interrupted' : 'failed';
return;
}
return tearDown;
}
private async _runAndReportError(callback: () => Promise<void>, result: FullResult) {
try {
await callback();
} catch (e) {
result.status = 'failed';
this._reporter.onError?.(serializeError(e));
}
}
}
function createFileMatcherFromFilter(filter: TestFileFilter) {

132
packages/playwright-test/src/taskRunner.ts Normal file
View File

@ -0,0 +1,132 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { debug } from 'playwright-core/lib/utilsBundle';
import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils';
import type { FullResult, Reporter } from '../reporter';
import { SigIntWatcher } from './sigIntWatcher';
import { serializeError } from './util';
type TaskTeardown = () => Promise<any> | undefined;
type Task = () => Promise<TaskTeardown | void> | undefined;
export class TaskRunner {
private _tasks: { name: string, task: Task }[] = [];
private _reporter: Reporter;
private _hasErrors = false;
private _interrupted = false;
private _isTearDown = false;
private _globalTimeoutForError: number;
constructor(reporter: Reporter, globalTimeoutForError: number) {
this._reporter = reporter;
this._globalTimeoutForError = globalTimeoutForError;
}
addTask(name: string, task: Task) {
this._tasks.push({ name, task });
}
stop() {
this._interrupted = true;
}
async run(deadline: number): Promise<FullResult['status']> {
const sigintWatcher = new SigIntWatcher();
const timeoutWatcher = new TimeoutWatcher(deadline);
const teardownRunner = new TaskRunner(this._reporter, this._globalTimeoutForError);
teardownRunner._isTearDown = true;
try {
let currentTaskName: string | undefined;
const taskLoop = async () => {
for (const { name, task } of this._tasks) {
currentTaskName = name;
if (this._interrupted)
break;
debug('pw:test:task')(`"${name}" started`);
try {
const teardown = await task();
if (teardown)
teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: teardown });
} catch (e) {
debug('pw:test:task')(`error in "${name}": `, e);
this._reporter.onError?.(serializeError(e));
if (!this._isTearDown)
this._interrupted = true;
this._hasErrors = true;
}
debug('pw:test:task')(`"${name}" finished`);
}
};
await Promise.race([
taskLoop(),
sigintWatcher.promise(),
timeoutWatcher.promise,
]);
// Prevent subsequent tasks from running.
this._interrupted = true;
let status: FullResult['status'] = 'passed';
if (sigintWatcher.hadSignal()) {
status = 'interrupted';
} else if (timeoutWatcher.timedOut()) {
this._reporter.onError?.({ message: `Timed out waiting ${this._globalTimeoutForError / 1000}s for the ${currentTaskName} to run` });
status = 'timedout';
} else if (this._hasErrors) {
status = 'failed';
}
return status;
} finally {
sigintWatcher.disarm();
timeoutWatcher.disarm();
if (!this._isTearDown)
await teardownRunner.run(deadline);
}
}
}
export class TimeoutWatcher {
private _timedOut = false;
readonly promise = new ManualPromise();
private _timer: NodeJS.Timeout | undefined;
constructor(deadline: number) {
if (!deadline)
return;
if (deadline - monotonicTime() <= 0) {
this._timedOut = true;
this.promise.resolve();
return;
}
this._timer = setTimeout(() => {
this._timedOut = true;
this.promise.resolve();
}, deadline - monotonicTime());
}
timedOut(): boolean {
return this._timedOut;
}
disarm() {
clearTimeout(this._timer);
}
}

14
packages/playwright-test/types/testReporter.d.ts vendored
View File

@ -349,8 +349,6 @@ export interface FullResult {
* ```
*
* Here is a typical order of reporter calls:
* - [reporter.onConfigure(config)](https://playwright.dev/docs/api/class-reporter#reporter-on-configure) is called
* once config has been resolved.
* - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called
* once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
* - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is
@ -367,8 +365,6 @@ export interface FullResult {
* [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more.
* - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after
* all tests that should run had finished.
* - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called before test
* runner exits.
*
* Additionally,
* [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and
@ -383,11 +379,6 @@ export interface FullResult {
* to enhance user experience.
*/
export interface Reporter {
/**
* Called once config is resolved.
* @param config Resolved configuration.
*/
onConfigure?(config: FullConfig): void;
/**
* Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
* @param config Resolved configuration.
@ -406,11 +397,6 @@ export interface Reporter {
* - `'interrupted'` - Interrupted by the user.
*/
onEnd?(result: FullResult): void | Promise<void>;
/**
* Called before test runner exits.
*/
onExit?(): void;
/**
* Called on some global error, for example unhandled exception in the worker process.
* @param error The error.

2
tests/playwright-test/exit-code.spec.ts
View File

@ -130,7 +130,7 @@ test('should respect global timeout', async ({ runInlineTest }) => {
`
}, { 'timeout': 100000, 'global-timeout': 3000 });
expect(exitCode).toBe(1);
expect(output).toContain('Timed out waiting 3s for the entire test run');
expect(output).toContain('Timed out waiting 3s for the test suite to run');
expect(monotonicTime() - now).toBeGreaterThan(2900);
});

62
tests/playwright-test/global-setup.spec.ts
View File

@ -321,3 +321,65 @@ test('globalSetup auth should compile', async ({ runTSC }) => {
const result = await runTSC(authFiles);
expect(result.exitCode).toBe(0);
});
test('teardown order', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
for (let i = 1; i < 4; ++i) {
pwt._addRunnerPlugin(() => ({
setup: () => console.log('\\n%%setup ' + i),
teardown: () => console.log('\\n%%teardown ' + i),
}));
}
export default {};
`,
'a.test.js': `
pwt.test('test', () => {});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%setup 1',
'%%setup 2',
'%%setup 3',
'%%teardown 3',
'%%teardown 2',
'%%teardown 1',
]);
});
test('teardown after error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
for (let i = 1; i < 4; ++i) {
pwt._addRunnerPlugin(() => ({
setup: () => console.log('\\n%%setup ' + i),
teardown: () => {
console.log('\\n%%teardown ' + i);
throw new Error('failed teardown ' + i)
},
}));
}
export default {};
`,
'a.test.js': `
pwt.test('test', () => {});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
const output = stripAnsi(result.output);
expect(output).toContain('Error: failed teardown 1');
expect(output).toContain('Error: failed teardown 2');
expect(output).toContain('Error: failed teardown 3');
expect(output).toContain('throw new Error(\'failed teardown');
expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%setup 1',
'%%setup 2',
'%%setup 3',
'%%teardown 3',
'%%teardown 2',
'%%teardown 1',
]);
});

1
utils/generate_types/overrides-testReporter.d.ts vendored
View File

@ -44,7 +44,6 @@ export interface FullResult {
}
export interface Reporter {
onConfigure?(config: FullConfig): void;
onBegin?(config: FullConfig, suite: Suite): void;
onEnd?(result: FullResult): void | Promise<void>;
}