diff --git a/utils/testrunner/Location.js b/utils/testrunner/Location.js index a18e3618a2..2a6eeacd3d 100644 --- a/utils/testrunner/Location.js +++ b/utils/testrunner/Location.js @@ -15,7 +15,10 @@ * limitations under the License. */ - const path = require('path'); +const path = require('path'); + +// Hack for our own tests. +const testRunnerTestFile = path.join(__dirname, 'test', 'testrunner.spec.js'); class Location { constructor() { @@ -69,7 +72,7 @@ class Location { if (!match) return null; const filePath = match[1]; - if (filePath === __filename || filePath.startsWith(ignorePrefix)) + if (filePath === __filename || (filePath.startsWith(ignorePrefix) && filePath !== testRunnerTestFile)) continue; location._filePath = filePath; diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js index b8c5c5c5c6..22f8c75aac 100644 --- a/utils/testrunner/Matchers.js +++ b/utils/testrunner/Matchers.js @@ -102,8 +102,6 @@ function stringFormatter(received, expected) { } function objectFormatter(received, expected) { - const receivedLines = received.split('\n'); - const expectedLines = expected.split('\n'); const encodingMap = new Map(); const decodingMap = new Map(); @@ -142,8 +140,12 @@ function objectFormatter(received, expected) { if (type === 1) return lines.map(line => '+ ' + colors.bgGreen.black(line)); return lines.map(line => ' ' + line); - }).flat().join('\n'); - return `Received:\n${highlighted}`; + }); + + const flattened = []; + for (const list of highlighted) + flattened.push(...list); + return `Received:\n${flattened.join('\n')}`; } function toBeFormatter(received, expected) { diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index bd97dcc532..28621e9460 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -302,13 +302,13 @@ class TestWorker { async _willStartTestRun(testRun) { testRun._startTimestamp = Date.now(); testRun._workerId = this._workerId; - await this._testRunner._delegate.onTestRunStarted(testRun); + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]); } async _didFinishTestRun(testRun) { testRun._endTimestamp = Date.now(); testRun._workerId = this._workerId; - await this._testRunner._delegate.onTestRunFinished(testRun); + await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]); } async _willStartTestBody(testRun) { @@ -352,6 +352,26 @@ class TestRunner { this._result = null; } + async _runDelegateCallback(callback, args) { + let { promise, terminate } = runUserCallback(callback, this._hookTimeout, args); + // Note: we do not terminate the delegate to keep reporting even when terminating. + const e = await promise; + if (e) { + debug('testrunner')(`Error while running delegate method: ${e}`); + const { message, error } = this._toError('INTERNAL ERROR', e); + this._terminate(TestResult.Crashed, message, false, error); + } + } + + _toError(message, error) { + if (!(error instanceof Error)) { + message += ': ' + error; + error = new Error(); + error.stack = ''; + } + return { message, error }; + } + async run(testRuns, options = {}) { const { parallel = 1, @@ -360,7 +380,7 @@ class TestRunner { totalTimeout = 0, onStarted = async (testRuns) => {}, onFinished = async (result) => {}, - onTestRunStarted = async(testRun) => {}, + onTestRunStarted = async (testRun) => {}, onTestRunFinished = async (testRun) => {}, } = options; this._breakOnFailure = breakOnFailure; @@ -374,34 +394,16 @@ class TestRunner { this._result = new Result(); this._result.runs = testRuns; - await this._delegate.onStarted(testRuns); - - let timeoutId; - if (totalTimeout) { - timeoutId = setTimeout(() => { - this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); - }, totalTimeout); - } const handleSIGINT = () => this._terminate(TestResult.Terminated, 'SIGINT received', false, null); const handleSIGHUP = () => this._terminate(TestResult.Terminated, 'SIGHUP received', false, null); const handleSIGTERM = () => this._terminate(TestResult.Terminated, 'SIGTERM received', true, null); - const handleRejection = error => { - let message = 'UNHANDLED PROMISE REJECTION'; - if (!(error instanceof Error)) { - message += ': ' + error; - error = new Error(); - error.stack = ''; - } + const handleRejection = e => { + const { message, error } = this._toError('UNHANDLED PROMISE REJECTION', e); this._terminate(TestResult.Crashed, message, false, error); }; - const handleException = error => { - let message = 'UNHANDLED ERROR'; - if (!(error instanceof Error)) { - message += ': ' + error; - error = new Error(); - error.stack = ''; - } + const handleException = e => { + const { message, error } = this._toError('UNHANDLED ERROR', e); this._terminate(TestResult.Crashed, message, false, error); }; process.on('SIGINT', handleSIGINT); @@ -410,6 +412,14 @@ class TestRunner { process.on('unhandledRejection', handleRejection); process.on('uncaughtException', handleException); + let timeoutId; + if (totalTimeout) { + timeoutId = setTimeout(() => { + this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); + }, totalTimeout); + } + await this._runDelegateCallback(this._delegate.onStarted, [testRuns]); + const workerCount = Math.min(parallel, testRuns.length); const workerPromises = []; for (let i = 0; i < workerCount; ++i) { @@ -418,23 +428,18 @@ class TestRunner { } await Promise.all(workerPromises); + if (testRuns.some(run => run.isFailure())) + this._result.setResult(TestResult.Failed, ''); + + await this._runDelegateCallback(this._delegate.onFinished, [this._result]); + clearTimeout(timeoutId); + process.removeListener('SIGINT', handleSIGINT); process.removeListener('SIGHUP', handleSIGHUP); process.removeListener('SIGTERM', handleSIGTERM); process.removeListener('unhandledRejection', handleRejection); process.removeListener('uncaughtException', handleException); - - if (testRuns.some(run => run.isFailure())) - this._result.setResult(TestResult.Failed, ''); - - clearTimeout(timeoutId); - await this._delegate.onFinished(this._result); - - const result = this._result; - this._result = null; - this._workers = []; - this._terminating = false; - return result; + return this._result; } async _runWorker(testRunIndex, testRuns, parallelIndex) { diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index 55e985c97c..fda5fe89c8 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -822,6 +822,23 @@ module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit ]); expect(result.result).toBe('ok'); }); + it('should crash when onStarted throws', async() => { + const t = new Runner({ + onStarted: () => { throw 42; }, + }); + const result = await t.run(); + expect(result.ok()).toBe(false); + expect(result.message).toBe('INTERNAL ERROR: 42'); + }); + it('should crash when onFinished throws', async() => { + const t = new Runner({ + onFinished: () => { throw new Error('42'); }, + }); + const result = await t.run(); + expect(result.ok()).toBe(false); + expect(result.message).toBe('INTERNAL ERROR'); + expect(result.result).toBe('crashed'); + }); }); };