mirror of
https://github.com/microsoft/playwright.git
synced 2025-01-05 19:04:43 +03:00
05b019f1ba
This re-lands PR https://github.com/microsoft/playwright/pull/2769 It was reverted before in https://github.com/microsoft/playwright/pull/2790 because it was breaking the new CHANNEL bot.
582 lines
20 KiB
JavaScript
582 lines
20 KiB
JavaScript
/**
|
|
* Copyright 2017 Google Inc. All rights reserved.
|
|
* Modifications copyright (c) Microsoft Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
const { SourceMapSupport } = require('./SourceMapSupport');
|
|
const debug = require('debug');
|
|
const { TestExpectation } = require('./Test');
|
|
|
|
const TimeoutError = new Error('Timeout');
|
|
const TerminatedError = new Error('Terminated');
|
|
|
|
function runUserCallback(callback, timeout, args) {
|
|
let terminateCallback;
|
|
let timeoutId;
|
|
const promise = Promise.race([
|
|
Promise.resolve().then(callback.bind(null, ...args)).then(() => null).catch(e => e),
|
|
new Promise(resolve => {
|
|
timeoutId = setTimeout(resolve.bind(null, TimeoutError), timeout);
|
|
}),
|
|
new Promise(resolve => terminateCallback = resolve),
|
|
]).catch(e => e).finally(() => clearTimeout(timeoutId));
|
|
const terminate = () => terminateCallback(TerminatedError);
|
|
return { promise, terminate };
|
|
}
|
|
|
|
const TestResult = {
|
|
Ok: 'ok',
|
|
MarkedAsFailing: 'markedAsFailing', // User marked as failed
|
|
Skipped: 'skipped', // User marked as skipped
|
|
Failed: 'failed', // Exception happened during running
|
|
TimedOut: 'timedout', // Timeout Exceeded while running
|
|
Terminated: 'terminated', // Execution terminated
|
|
Crashed: 'crashed', // If testrunner crashed due to this test
|
|
};
|
|
|
|
function isEmptyEnvironment(env) {
|
|
return !env.afterEach && !env.afterAll && !env.beforeEach && !env.beforeAll &&
|
|
!env.globalSetup && !env.globalTeardown;
|
|
}
|
|
|
|
class TestRun {
|
|
constructor(test) {
|
|
this._test = test;
|
|
this._result = null;
|
|
this._error = null;
|
|
this._startTimestamp = 0;
|
|
this._endTimestamp = 0;
|
|
this._workerId = null;
|
|
this._output = [];
|
|
|
|
this._environments = test._environments.filter(env => !isEmptyEnvironment(env)).reverse();
|
|
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
|
this._environments.push(...suite._environments.filter(env => !isEmptyEnvironment(env)).reverse());
|
|
this._environments.reverse();
|
|
}
|
|
|
|
finished() {
|
|
return this._result !== null && this._result !== 'running';
|
|
}
|
|
|
|
isFailure() {
|
|
return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed;
|
|
}
|
|
|
|
ok() {
|
|
return this._result === TestResult.Ok;
|
|
}
|
|
|
|
result() {
|
|
return this._result;
|
|
}
|
|
|
|
error() {
|
|
return this._error;
|
|
}
|
|
|
|
duration() {
|
|
return this._endTimestamp - this._startTimestamp;
|
|
}
|
|
|
|
test() {
|
|
return this._test;
|
|
}
|
|
|
|
workerId() {
|
|
return this._workerId;
|
|
}
|
|
|
|
log(log) {
|
|
this._output.push(log);
|
|
}
|
|
|
|
output() {
|
|
return this._output;
|
|
}
|
|
}
|
|
|
|
class Result {
|
|
constructor() {
|
|
this.result = TestResult.Ok;
|
|
this.exitCode = 0;
|
|
this.message = '';
|
|
this.errors = [];
|
|
this.runs = [];
|
|
}
|
|
|
|
setResult(result, message) {
|
|
if (!this.ok())
|
|
return;
|
|
this.result = result;
|
|
this.message = message || '';
|
|
if (result === TestResult.Ok)
|
|
this.exitCode = 0;
|
|
else if (result === TestResult.Terminated)
|
|
this.exitCode = 130;
|
|
else if (result === TestResult.Crashed)
|
|
this.exitCode = 2;
|
|
else
|
|
this.exitCode = 1;
|
|
}
|
|
|
|
addError(message, error, worker) {
|
|
const data = { message, error, runs: [] };
|
|
if (worker)
|
|
data.runs = worker._runs.slice();
|
|
this.errors.push(data);
|
|
}
|
|
|
|
ok() {
|
|
return this.result === TestResult.Ok;
|
|
}
|
|
}
|
|
|
|
class TestWorker {
|
|
constructor(testRunner, hookRunner, workerId, parallelIndex) {
|
|
this._testRunner = testRunner;
|
|
this._hookRunner = hookRunner;
|
|
this._state = { parallelIndex };
|
|
this._environmentStack = [];
|
|
this._terminating = false;
|
|
this._workerId = workerId;
|
|
this._runningTestTerminate = null;
|
|
this._runs = [];
|
|
}
|
|
|
|
terminate(terminateHooks) {
|
|
this._terminating = true;
|
|
if (this._runningTestTerminate)
|
|
this._runningTestTerminate();
|
|
this._hookRunner.terminateWorker(this);
|
|
}
|
|
|
|
_markTerminated(testRun) {
|
|
if (!this._terminating)
|
|
return false;
|
|
testRun._result = TestResult.Terminated;
|
|
return true;
|
|
}
|
|
|
|
async run(testRun) {
|
|
this._runs.push(testRun);
|
|
|
|
const test = testRun.test();
|
|
let skipped = test.skipped();
|
|
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
|
skipped = skipped || suite.skipped();
|
|
if (skipped) {
|
|
await this._willStartTestRun(testRun);
|
|
testRun._result = TestResult.Skipped;
|
|
await this._didFinishTestRun(testRun);
|
|
return;
|
|
}
|
|
|
|
let expectedToFail = test.expectation() === TestExpectation.Fail;
|
|
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
|
expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail);
|
|
if (expectedToFail) {
|
|
await this._willStartTestRun(testRun);
|
|
testRun._result = TestResult.MarkedAsFailing;
|
|
await this._didFinishTestRun(testRun);
|
|
return;
|
|
}
|
|
|
|
const environmentStack = testRun._environments;
|
|
let common = 0;
|
|
while (common < environmentStack.length && this._environmentStack[common] === environmentStack[common])
|
|
common++;
|
|
|
|
while (this._environmentStack.length > common) {
|
|
if (this._markTerminated(testRun))
|
|
return;
|
|
const environment = this._environmentStack.pop();
|
|
if (!await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, testRun))
|
|
return;
|
|
if (!await this._hookRunner.maybeRunGlobalTeardown(environment))
|
|
return;
|
|
}
|
|
while (this._environmentStack.length < environmentStack.length) {
|
|
if (this._markTerminated(testRun))
|
|
return;
|
|
const environment = environmentStack[this._environmentStack.length];
|
|
this._environmentStack.push(environment);
|
|
if (!await this._hookRunner.maybeRunGlobalSetup(environment))
|
|
return;
|
|
if (!await this._hookRunner.runHook(environment, 'beforeAll', [this._state], this, testRun))
|
|
return;
|
|
}
|
|
|
|
if (this._markTerminated(testRun))
|
|
return;
|
|
|
|
// From this point till the end, we have to run all hooks
|
|
// no matter what happens.
|
|
|
|
await this._willStartTestRun(testRun);
|
|
for (const environment of this._environmentStack) {
|
|
await this._hookRunner.runHook(environment, 'beforeEach', [this._state, testRun], this, testRun);
|
|
}
|
|
|
|
if (!testRun._error && !this._markTerminated(testRun)) {
|
|
await this._willStartTestBody(testRun);
|
|
const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, testRun]);
|
|
this._runningTestTerminate = terminate;
|
|
testRun._error = await promise;
|
|
this._runningTestTerminate = null;
|
|
if (testRun._error && testRun._error.stack)
|
|
await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error);
|
|
if (!testRun._error)
|
|
testRun._result = TestResult.Ok;
|
|
else if (testRun._error === TimeoutError)
|
|
testRun._result = TestResult.TimedOut;
|
|
else if (testRun._error === TerminatedError)
|
|
testRun._result = TestResult.Terminated;
|
|
else
|
|
testRun._result = TestResult.Failed;
|
|
await this._didFinishTestBody(testRun);
|
|
}
|
|
|
|
for (const environment of this._environmentStack.slice().reverse())
|
|
await this._hookRunner.runHook(environment, 'afterEach', [this._state, testRun], this, testRun);
|
|
await this._didFinishTestRun(testRun);
|
|
}
|
|
|
|
async _willStartTestRun(testRun) {
|
|
testRun._startTimestamp = Date.now();
|
|
testRun._workerId = this._workerId;
|
|
await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]);
|
|
}
|
|
|
|
async _didFinishTestRun(testRun) {
|
|
testRun._endTimestamp = Date.now();
|
|
testRun._workerId = this._workerId;
|
|
|
|
this._hookRunner.markFinishedTestRun(testRun);
|
|
await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]);
|
|
}
|
|
|
|
async _willStartTestBody(testRun) {
|
|
debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location()})`);
|
|
}
|
|
|
|
async _didFinishTestBody(testRun) {
|
|
debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`);
|
|
}
|
|
|
|
async shutdown() {
|
|
while (this._environmentStack.length > 0) {
|
|
const environment = this._environmentStack.pop();
|
|
await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, null);
|
|
await this._hookRunner.maybeRunGlobalTeardown(environment);
|
|
}
|
|
}
|
|
}
|
|
|
|
class HookRunner {
|
|
constructor(testRunner, testRuns) {
|
|
this._testRunner = testRunner;
|
|
this._runningHookTerminations = new Map();
|
|
|
|
this._environmentToGlobalState = new Map();
|
|
for (const testRun of testRuns) {
|
|
for (const env of testRun._environments) {
|
|
let globalState = this._environmentToGlobalState.get(env);
|
|
if (!globalState) {
|
|
globalState = {
|
|
pendingTestRuns: new Set(),
|
|
globalSetupPromise: null,
|
|
globalTeardownPromise: null,
|
|
};
|
|
this._environmentToGlobalState.set(env, globalState);
|
|
}
|
|
globalState.pendingTestRuns.add(testRun);
|
|
}
|
|
}
|
|
}
|
|
|
|
terminateWorker(worker) {
|
|
let termination = this._runningHookTerminations.get(worker);
|
|
this._runningHookTerminations.delete(worker);
|
|
if (termination)
|
|
termination();
|
|
}
|
|
|
|
terminateAll() {
|
|
for (const termination of this._runningHookTerminations.values())
|
|
termination();
|
|
this._runningHookTerminations.clear();
|
|
}
|
|
|
|
markFinishedTestRun(testRun) {
|
|
for (const environment of testRun._environments) {
|
|
const globalState = this._environmentToGlobalState.get(environment);
|
|
globalState.pendingTestRuns.delete(testRun);
|
|
}
|
|
}
|
|
|
|
async _runHookInternal(worker, testRun, hook, fullName, hookArgs = []) {
|
|
await this._willStartHook(worker, testRun, hook, fullName);
|
|
const timeout = this._testRunner._hookTimeout;
|
|
const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs);
|
|
this._runningHookTerminations.set(worker, terminate);
|
|
let error = await promise;
|
|
this._runningHookTerminations.delete(worker);
|
|
|
|
if (error) {
|
|
if (testRun && testRun._result !== TestResult.Terminated) {
|
|
// Prefer terminated result over any hook failures.
|
|
testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed;
|
|
}
|
|
let message;
|
|
if (error === TimeoutError) {
|
|
message = `Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`;
|
|
error = null;
|
|
} else if (error === TerminatedError) {
|
|
// Do not report termination details - it's just noise.
|
|
message = '';
|
|
error = null;
|
|
} else {
|
|
if (error.stack)
|
|
await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
|
|
message = `FAILED while running "${hook.name}" in suite "${fullName}": `;
|
|
}
|
|
await this._didFailHook(worker, testRun, hook, fullName, message, error);
|
|
if (testRun)
|
|
testRun._error = error;
|
|
return false;
|
|
}
|
|
|
|
await this._didCompleteHook(worker, testRun, hook, fullName);
|
|
return true;
|
|
}
|
|
|
|
async runHook(environment, hookName, hookArgs, worker = null, testRun = null) {
|
|
const hookBody = environment[hookName];
|
|
if (!hookBody)
|
|
return true;
|
|
const envName = environment.name ? environment.name() : environment.constructor.name;
|
|
return await this._runHookInternal(worker, testRun, {name: hookName, body: hookBody.bind(environment)}, envName, hookArgs);
|
|
}
|
|
|
|
async maybeRunGlobalSetup(environment) {
|
|
const globalState = this._environmentToGlobalState.get(environment);
|
|
if (!globalState.globalSetupPromise)
|
|
globalState.globalSetupPromise = this.runHook(environment, 'globalSetup', []);
|
|
if (!await globalState.globalSetupPromise) {
|
|
await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
async maybeRunGlobalTeardown(environment) {
|
|
const globalState = this._environmentToGlobalState.get(environment);
|
|
if (!globalState.globalTeardownPromise) {
|
|
if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise))
|
|
globalState.globalTeardownPromise = this.runHook(environment, 'globalTeardown', []);
|
|
}
|
|
if (!globalState.globalTeardownPromise)
|
|
return true;
|
|
if (!await globalState.globalTeardownPromise) {
|
|
await this._testRunner._terminate(TestResult.Crashed, 'Global teardown failed!', false, null);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
async _willStartHook(worker, testRun, hook, fullName) {
|
|
debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}"`);
|
|
}
|
|
|
|
async _didFailHook(worker, testRun, hook, fullName, message, error) {
|
|
debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}"`);
|
|
if (message)
|
|
this._testRunner._result.addError(message, error, worker);
|
|
this._testRunner._result.setResult(TestResult.Crashed, message);
|
|
}
|
|
|
|
async _didCompleteHook(worker, testRun, hook, fullName) {
|
|
debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}"`);
|
|
}
|
|
}
|
|
|
|
function workerName(worker) {
|
|
return worker ? `<worker ${worker._workerId}>` : `<_global_>`;
|
|
}
|
|
|
|
class TestRunner {
|
|
constructor() {
|
|
this._sourceMapSupport = new SourceMapSupport();
|
|
this._nextWorkerId = 1;
|
|
this._workers = [];
|
|
this._terminating = false;
|
|
this._result = null;
|
|
this._hookRunner = null;
|
|
}
|
|
|
|
async _runDelegateCallback(callback, args) {
|
|
let { promise, terminate } = runUserCallback(callback, this._hookTimeout, args);
|
|
// Note: we do not terminate the delegate to keep reporting even when terminating.
|
|
const e = await promise;
|
|
if (e) {
|
|
debug('testrunner')(`Error while running delegate method: ${e}`);
|
|
const { message, error } = this._toError('INTERNAL ERROR', e);
|
|
this._terminate(TestResult.Crashed, message, false, error);
|
|
}
|
|
}
|
|
|
|
_toError(message, error) {
|
|
if (!(error instanceof Error)) {
|
|
message += ': ' + error;
|
|
error = new Error();
|
|
error.stack = '';
|
|
}
|
|
return { message, error };
|
|
}
|
|
|
|
async run(testRuns, options = {}) {
|
|
const {
|
|
parallel = 1,
|
|
breakOnFailure = false,
|
|
hookTimeout = 10 * 1000,
|
|
totalTimeout = 0,
|
|
onStarted = async (testRuns) => {},
|
|
onFinished = async (result) => {},
|
|
onTestRunStarted = async (testRun) => {},
|
|
onTestRunFinished = async (testRun) => {},
|
|
} = options;
|
|
this._breakOnFailure = breakOnFailure;
|
|
this._hookTimeout = hookTimeout === 0 ? 100000000 : hookTimeout;
|
|
this._delegate = {
|
|
onStarted,
|
|
onFinished,
|
|
onTestRunStarted,
|
|
onTestRunFinished
|
|
};
|
|
|
|
this._result = new Result();
|
|
this._result.runs = testRuns;
|
|
|
|
const terminationPromises = [];
|
|
const handleSIGINT = () => this._terminate(TestResult.Terminated, 'SIGINT received', false, null);
|
|
const handleSIGHUP = () => this._terminate(TestResult.Terminated, 'SIGHUP received', false, null);
|
|
const handleSIGTERM = () => this._terminate(TestResult.Terminated, 'SIGTERM received', true, null);
|
|
const handleRejection = e => {
|
|
const { message, error } = this._toError('UNHANDLED PROMISE REJECTION', e);
|
|
terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error));
|
|
};
|
|
const handleException = e => {
|
|
const { message, error } = this._toError('UNHANDLED ERROR', e);
|
|
terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error));
|
|
};
|
|
process.on('SIGINT', handleSIGINT);
|
|
process.on('SIGHUP', handleSIGHUP);
|
|
process.on('SIGTERM', handleSIGTERM);
|
|
process.on('unhandledRejection', handleRejection);
|
|
process.on('uncaughtException', handleException);
|
|
|
|
let timeoutId;
|
|
if (totalTimeout) {
|
|
timeoutId = setTimeout(() => {
|
|
terminationPromises.push(this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */));
|
|
}, totalTimeout);
|
|
}
|
|
await this._runDelegateCallback(this._delegate.onStarted, [testRuns]);
|
|
|
|
this._hookRunner = new HookRunner(this, testRuns);
|
|
|
|
const workerCount = Math.min(parallel, testRuns.length);
|
|
const workerPromises = [];
|
|
for (let i = 0; i < workerCount; ++i) {
|
|
const initialTestRunIndex = i * Math.floor(testRuns.length / workerCount);
|
|
workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i));
|
|
}
|
|
await Promise.all(workerPromises);
|
|
await Promise.all(terminationPromises);
|
|
|
|
if (testRuns.some(run => run.isFailure()))
|
|
this._result.setResult(TestResult.Failed, '');
|
|
|
|
await this._runDelegateCallback(this._delegate.onFinished, [this._result]);
|
|
clearTimeout(timeoutId);
|
|
|
|
process.removeListener('SIGINT', handleSIGINT);
|
|
process.removeListener('SIGHUP', handleSIGHUP);
|
|
process.removeListener('SIGTERM', handleSIGTERM);
|
|
process.removeListener('unhandledRejection', handleRejection);
|
|
process.removeListener('uncaughtException', handleException);
|
|
return this._result;
|
|
}
|
|
|
|
async _runWorker(testRunIndex, testRuns, parallelIndex) {
|
|
let worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex);
|
|
this._workers[parallelIndex] = worker;
|
|
while (!this._terminating) {
|
|
let skipped = 0;
|
|
while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) {
|
|
testRunIndex = (testRunIndex + 1) % testRuns.length;
|
|
skipped++;
|
|
}
|
|
const testRun = testRuns[testRunIndex];
|
|
if (testRun._result !== null) {
|
|
// All tests have been run.
|
|
break;
|
|
}
|
|
|
|
// Mark as running so that other workers do not run it again.
|
|
testRun._result = 'running';
|
|
await worker.run(testRun);
|
|
if (testRun.isFailure()) {
|
|
// Something went wrong during test run, let's use a fresh worker.
|
|
await worker.shutdown();
|
|
if (this._breakOnFailure) {
|
|
const message = `Terminating because a test has failed and |testRunner.breakOnFailure| is enabled`;
|
|
await this._terminate(TestResult.Terminated, message, false /* force */, null /* error */);
|
|
return;
|
|
}
|
|
worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex);
|
|
this._workers[parallelIndex] = worker;
|
|
}
|
|
}
|
|
await worker.shutdown();
|
|
}
|
|
|
|
async _terminate(result, message, force, error) {
|
|
debug('testrunner')(`TERMINATED result = ${result}, message = ${message}`);
|
|
this._terminating = true;
|
|
for (const worker of this._workers)
|
|
worker.terminate(force /* terminateHooks */);
|
|
if (this._hookRunner)
|
|
this._hookRunner.terminateAll();
|
|
this._result.setResult(result, message);
|
|
if (this._result.message === 'SIGINT received' && message === 'SIGTERM received')
|
|
this._result.message = message;
|
|
if (error) {
|
|
if (error.stack)
|
|
await this._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
|
|
this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null);
|
|
}
|
|
}
|
|
|
|
async terminate() {
|
|
if (!this._result)
|
|
return;
|
|
await this._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', true /* force */, null /* error */);
|
|
}
|
|
}
|
|
|
|
module.exports = { TestRunner, TestRun, TestResult, Result };
|