mirror of
https://github.com/microsoft/playwright.git
synced 2024-10-28 06:07:53 +03:00
feat(testrunner): catch delegate errors (#1704)
This ensures we get a proper error when something goes wrong. Should also help with producing the right error code in the case of internal error. Drive-by: fix location issue which manifests on the bots. Drive-by: remove the use of Array.prototype.flat to make it work on bots.
This commit is contained in:
parent
0ff2e6a03e
commit
20ff327827
@ -15,7 +15,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const path = require('path');
|
||||
|
||||
// Hack for our own tests.
|
||||
const testRunnerTestFile = path.join(__dirname, 'test', 'testrunner.spec.js');
|
||||
|
||||
class Location {
|
||||
constructor() {
|
||||
@ -69,7 +72,7 @@ class Location {
|
||||
if (!match)
|
||||
return null;
|
||||
const filePath = match[1];
|
||||
if (filePath === __filename || filePath.startsWith(ignorePrefix))
|
||||
if (filePath === __filename || (filePath.startsWith(ignorePrefix) && filePath !== testRunnerTestFile))
|
||||
continue;
|
||||
|
||||
location._filePath = filePath;
|
||||
|
@ -102,8 +102,6 @@ function stringFormatter(received, expected) {
|
||||
}
|
||||
|
||||
function objectFormatter(received, expected) {
|
||||
const receivedLines = received.split('\n');
|
||||
const expectedLines = expected.split('\n');
|
||||
const encodingMap = new Map();
|
||||
const decodingMap = new Map();
|
||||
|
||||
@ -142,8 +140,12 @@ function objectFormatter(received, expected) {
|
||||
if (type === 1)
|
||||
return lines.map(line => '+ ' + colors.bgGreen.black(line));
|
||||
return lines.map(line => ' ' + line);
|
||||
}).flat().join('\n');
|
||||
return `Received:\n${highlighted}`;
|
||||
});
|
||||
|
||||
const flattened = [];
|
||||
for (const list of highlighted)
|
||||
flattened.push(...list);
|
||||
return `Received:\n${flattened.join('\n')}`;
|
||||
}
|
||||
|
||||
function toBeFormatter(received, expected) {
|
||||
|
@ -302,13 +302,13 @@ class TestWorker {
|
||||
async _willStartTestRun(testRun) {
|
||||
testRun._startTimestamp = Date.now();
|
||||
testRun._workerId = this._workerId;
|
||||
await this._testRunner._delegate.onTestRunStarted(testRun);
|
||||
await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]);
|
||||
}
|
||||
|
||||
async _didFinishTestRun(testRun) {
|
||||
testRun._endTimestamp = Date.now();
|
||||
testRun._workerId = this._workerId;
|
||||
await this._testRunner._delegate.onTestRunFinished(testRun);
|
||||
await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]);
|
||||
}
|
||||
|
||||
async _willStartTestBody(testRun) {
|
||||
@ -352,6 +352,26 @@ class TestRunner {
|
||||
this._result = null;
|
||||
}
|
||||
|
||||
async _runDelegateCallback(callback, args) {
|
||||
let { promise, terminate } = runUserCallback(callback, this._hookTimeout, args);
|
||||
// Note: we do not terminate the delegate to keep reporting even when terminating.
|
||||
const e = await promise;
|
||||
if (e) {
|
||||
debug('testrunner')(`Error while running delegate method: ${e}`);
|
||||
const { message, error } = this._toError('INTERNAL ERROR', e);
|
||||
this._terminate(TestResult.Crashed, message, false, error);
|
||||
}
|
||||
}
|
||||
|
||||
_toError(message, error) {
|
||||
if (!(error instanceof Error)) {
|
||||
message += ': ' + error;
|
||||
error = new Error();
|
||||
error.stack = '';
|
||||
}
|
||||
return { message, error };
|
||||
}
|
||||
|
||||
async run(testRuns, options = {}) {
|
||||
const {
|
||||
parallel = 1,
|
||||
@ -360,7 +380,7 @@ class TestRunner {
|
||||
totalTimeout = 0,
|
||||
onStarted = async (testRuns) => {},
|
||||
onFinished = async (result) => {},
|
||||
onTestRunStarted = async(testRun) => {},
|
||||
onTestRunStarted = async (testRun) => {},
|
||||
onTestRunFinished = async (testRun) => {},
|
||||
} = options;
|
||||
this._breakOnFailure = breakOnFailure;
|
||||
@ -374,34 +394,16 @@ class TestRunner {
|
||||
|
||||
this._result = new Result();
|
||||
this._result.runs = testRuns;
|
||||
await this._delegate.onStarted(testRuns);
|
||||
|
||||
let timeoutId;
|
||||
if (totalTimeout) {
|
||||
timeoutId = setTimeout(() => {
|
||||
this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */);
|
||||
}, totalTimeout);
|
||||
}
|
||||
|
||||
const handleSIGINT = () => this._terminate(TestResult.Terminated, 'SIGINT received', false, null);
|
||||
const handleSIGHUP = () => this._terminate(TestResult.Terminated, 'SIGHUP received', false, null);
|
||||
const handleSIGTERM = () => this._terminate(TestResult.Terminated, 'SIGTERM received', true, null);
|
||||
const handleRejection = error => {
|
||||
let message = 'UNHANDLED PROMISE REJECTION';
|
||||
if (!(error instanceof Error)) {
|
||||
message += ': ' + error;
|
||||
error = new Error();
|
||||
error.stack = '';
|
||||
}
|
||||
const handleRejection = e => {
|
||||
const { message, error } = this._toError('UNHANDLED PROMISE REJECTION', e);
|
||||
this._terminate(TestResult.Crashed, message, false, error);
|
||||
};
|
||||
const handleException = error => {
|
||||
let message = 'UNHANDLED ERROR';
|
||||
if (!(error instanceof Error)) {
|
||||
message += ': ' + error;
|
||||
error = new Error();
|
||||
error.stack = '';
|
||||
}
|
||||
const handleException = e => {
|
||||
const { message, error } = this._toError('UNHANDLED ERROR', e);
|
||||
this._terminate(TestResult.Crashed, message, false, error);
|
||||
};
|
||||
process.on('SIGINT', handleSIGINT);
|
||||
@ -410,6 +412,14 @@ class TestRunner {
|
||||
process.on('unhandledRejection', handleRejection);
|
||||
process.on('uncaughtException', handleException);
|
||||
|
||||
let timeoutId;
|
||||
if (totalTimeout) {
|
||||
timeoutId = setTimeout(() => {
|
||||
this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */);
|
||||
}, totalTimeout);
|
||||
}
|
||||
await this._runDelegateCallback(this._delegate.onStarted, [testRuns]);
|
||||
|
||||
const workerCount = Math.min(parallel, testRuns.length);
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < workerCount; ++i) {
|
||||
@ -418,23 +428,18 @@ class TestRunner {
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
if (testRuns.some(run => run.isFailure()))
|
||||
this._result.setResult(TestResult.Failed, '');
|
||||
|
||||
await this._runDelegateCallback(this._delegate.onFinished, [this._result]);
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
process.removeListener('SIGINT', handleSIGINT);
|
||||
process.removeListener('SIGHUP', handleSIGHUP);
|
||||
process.removeListener('SIGTERM', handleSIGTERM);
|
||||
process.removeListener('unhandledRejection', handleRejection);
|
||||
process.removeListener('uncaughtException', handleException);
|
||||
|
||||
if (testRuns.some(run => run.isFailure()))
|
||||
this._result.setResult(TestResult.Failed, '');
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
await this._delegate.onFinished(this._result);
|
||||
|
||||
const result = this._result;
|
||||
this._result = null;
|
||||
this._workers = [];
|
||||
this._terminating = false;
|
||||
return result;
|
||||
return this._result;
|
||||
}
|
||||
|
||||
async _runWorker(testRunIndex, testRuns, parallelIndex) {
|
||||
|
@ -822,6 +822,23 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit
|
||||
]);
|
||||
expect(result.result).toBe('ok');
|
||||
});
|
||||
it('should crash when onStarted throws', async() => {
|
||||
const t = new Runner({
|
||||
onStarted: () => { throw 42; },
|
||||
});
|
||||
const result = await t.run();
|
||||
expect(result.ok()).toBe(false);
|
||||
expect(result.message).toBe('INTERNAL ERROR: 42');
|
||||
});
|
||||
it('should crash when onFinished throws', async() => {
|
||||
const t = new Runner({
|
||||
onFinished: () => { throw new Error('42'); },
|
||||
});
|
||||
const result = await t.run();
|
||||
expect(result.ok()).toBe(false);
|
||||
expect(result.message).toBe('INTERNAL ERROR');
|
||||
expect(result.result).toBe('crashed');
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user