diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9c45d21186..bf0c559bd1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,8 @@ on: env: CI: true + # Force terminal colors. @see https://www.npmjs.com/package/colors + FORCE_COLOR: 1 jobs: chromium_linux: diff --git a/package.json b/package.json index 3e9ea44e7d..055e825c60 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@types/ws": "^6.0.1", "@typescript-eslint/eslint-plugin": "^2.6.1", "@typescript-eslint/parser": "^2.6.1", + "colors": "^1.4.0", "commonmark": "^0.28.1", "cross-env": "^5.0.5", "eslint": "^6.6.0", diff --git a/test/golden-utils.js b/test/golden-utils.js index ebb152f0e9..7ae32399d8 100644 --- a/test/golden-utils.js +++ b/test/golden-utils.js @@ -20,6 +20,7 @@ const Diff = require('text-diff'); const PNG = require('pngjs').PNG; const jpeg = require('jpeg-js'); const pixelmatch = require('pixelmatch'); +const c = require('colors/safe'); module.exports = {compare}; @@ -51,7 +52,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) { const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer); if (expected.width !== actual.width || expected.height !== actual.height) { return { - errorMessage: `Sizes differ: expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` + errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` }; } const diff = new PNG({width: expected.width, height: expected.height}); @@ -110,23 +111,34 @@ function compare(goldenPath, outputPath, actual, goldenName) { if (!comparator) { return { pass: false, - message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName + message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName, }; } const result = comparator(actual, expected, mimeType); if (!result) return { pass: true }; ensureOutputDir(); + const output = [ + c.red(`GOLDEN FAILED: `) + c.yellow('"' + goldenName + '"'), + ]; + if (result.errorMessage) + output.push(' ' + result.errorMessage); + output.push(''); + output.push(`Expected: ${c.yellow(expectedPath)}`); if (goldenPath === outputPath) { - fs.writeFileSync(addSuffix(actualPath, '-actual'), actual); + const filepath = addSuffix(actualPath, '-actual'); + fs.writeFileSync(filepath, actual); + output.push(`Received: ${c.yellow(filepath)}`); } else { fs.writeFileSync(actualPath, actual); // Copy expected to the output/ folder for convenience. fs.writeFileSync(addSuffix(actualPath, '-expected'), expected); + output.push(`Received: ${c.yellow(actualPath)}`); } if (result.diff) { const diffPath = addSuffix(actualPath, '-diff', result.diffExtension); fs.writeFileSync(diffPath, result.diff); + output.push(` Diff: ${c.yellow(diffPath)}`); } let message = goldenName + ' mismatch!'; @@ -134,7 +146,8 @@ function compare(goldenPath, outputPath, actual, goldenName) { message += ' ' + result.errorMessage; return { pass: false, - message: message + ' ' + messageSuffix + message: message + ' ' + messageSuffix, + formatter: () => output.join('\n'), }; function ensureOutputDir() { diff --git a/test/test.js b/test/test.js index 688735ed08..7d5f63c431 100644 --- a/test/test.js +++ b/test/test.js @@ -124,7 +124,6 @@ if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) { new Reporter(testRunner, { verbose: process.argv.includes('--verbose'), summary: !process.argv.includes('--verbose'), - projectFolder: utils.projectRoot(), showSlowTests: process.env.CI ? 5 : 0, showSkippedTests: 10, }); diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js index 000751d43c..b9273ca509 100644 --- a/utils/testrunner/Matchers.js +++ b/utils/testrunner/Matchers.js @@ -14,7 +14,11 @@ * limitations under the License. */ -module.exports = class Matchers { +const {getCallerLocation} = require('./utils.js'); +const colors = require('colors/safe'); +const Diff = require('text-diff'); + +class Matchers { constructor(customMatchers = {}) { this._matchers = {}; Object.assign(this._matchers, DefaultMatchers); @@ -26,99 +30,202 @@ module.exports = class Matchers { this._matchers[name] = matcher; } - expect(value) { - return new Expect(value, this._matchers); + expect(received) { + return new Expect(received, this._matchers); } }; +class MatchError extends Error { + constructor(message, formatter) { + super(message); + this.name = this.constructor.name; + this.formatter = formatter; + this.location = getCallerLocation(__filename); + Error.captureStackTrace(this, this.constructor); + } +} + +module.exports = {Matchers, MatchError}; + class Expect { - constructor(value, matchers) { + constructor(received, matchers) { this.not = {}; this.not.not = this; for (const matcherName of Object.keys(matchers)) { const matcher = matchers[matcherName]; - this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, value); - this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, value); + this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received); + this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received); } - function applyMatcher(matcherName, matcher, inverse, value, ...args) { - const result = matcher.call(null, value, ...args); + function applyMatcher(matcherName, matcher, inverse, received, ...args) { + const result = matcher.call(null, received, ...args); const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : ''); if (result.pass === inverse) - throw new Error(message); + throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received)); } } } +function defaultFormatter(received) { + return `Received: ${colors.red(JSON.stringify(received))}`; +} + +function stringFormatter(received, expected) { + const diff = new Diff(); + const result = diff.main(expected, received); + diff.cleanupSemantic(result); + const highlighted = result.map(([type, text]) => { + if (type === -1) + return colors.bgRed(text); + if (type === 1) + return colors.bgGreen.black(text); + return text; + }).join(''); + const output = [ + `Expected: ${expected}`, + `Received: ${highlighted}`, + ]; + for (let i = 0; i < Math.min(expected.length, received.length); ++i) { + if (expected[i] !== received[i]) { + const padding = ' '.repeat('Expected: '.length); + const firstDiffCharacter = '~'.repeat(i) + '^'; + output.push(colors.red(padding + firstDiffCharacter)); + break; + } + } + return output.join('\n'); +} + +function objectFormatter(received, expected) { + const receivedLines = received.split('\n'); + const expectedLines = expected.split('\n'); + const encodingMap = new Map(); + const decodingMap = new Map(); + + const doEncodeLines = (lines) => { + let encoded = ''; + for (const line of lines) { + let code = encodingMap.get(line); + if (!code) { + code = String.fromCodePoint(encodingMap.size); + encodingMap.set(line, code); + decodingMap.set(code, line); + } + encoded += code; + } + return encoded; + }; + + const doDecodeLines = (text) => { + let decoded = []; + for (const codepoint of [...text]) + decoded.push(decodingMap.get(codepoint)); + return decoded; + } + + let receivedEncoded = doEncodeLines(received.split('\n')); + let expectedEncoded = doEncodeLines(expected.split('\n')); + + const diff = new Diff(); + const result = diff.main(expectedEncoded, receivedEncoded); + diff.cleanupSemantic(result); + + const highlighted = result.map(([type, text]) => { + const lines = doDecodeLines(text); + if (type === -1) + return lines.map(line => '- ' + colors.bgRed(line)); + if (type === 1) + return lines.map(line => '+ ' + colors.bgGreen.black(line)); + return lines.map(line => ' ' + line); + }).flat().join('\n'); + return `Received:\n${highlighted}`; +} + +function toBeFormatter(received, expected) { + if (typeof expected === 'string' && typeof received === 'string') { + return stringFormatter(JSON.stringify(received), JSON.stringify(expected)); + } + return [ + `Expected: ${JSON.stringify(expected)}`, + `Received: ${colors.red(JSON.stringify(received))}`, + ].join('\n'); +} + const DefaultMatchers = { - toBe: function(value, other, message) { - message = message || `${value} == ${other}`; - return { pass: value === other, message }; + toBe: function(received, expected, message) { + message = message || `${received} == ${expected}`; + return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) }; }, - toBeFalsy: function(value, message) { - message = message || `${value}`; - return { pass: !value, message }; + toBeFalsy: function(received, message) { + message = message || `${received}`; + return { pass: !received, message }; }, - toBeTruthy: function(value, message) { - message = message || `${value}`; - return { pass: !!value, message }; + toBeTruthy: function(received, message) { + message = message || `${received}`; + return { pass: !!received, message }; }, - toBeGreaterThan: function(value, other, message) { - message = message || `${value} > ${other}`; - return { pass: value > other, message }; + toBeGreaterThan: function(received, other, message) { + message = message || `${received} > ${other}`; + return { pass: received > other, message }; }, - toBeGreaterThanOrEqual: function(value, other, message) { - message = message || `${value} >= ${other}`; - return { pass: value >= other, message }; + toBeGreaterThanOrEqual: function(received, other, message) { + message = message || `${received} >= ${other}`; + return { pass: received >= other, message }; }, - toBeLessThan: function(value, other, message) { - message = message || `${value} < ${other}`; - return { pass: value < other, message }; + toBeLessThan: function(received, other, message) { + message = message || `${received} < ${other}`; + return { pass: received < other, message }; }, - toBeLessThanOrEqual: function(value, other, message) { - message = message || `${value} <= ${other}`; - return { pass: value <= other, message }; + toBeLessThanOrEqual: function(received, other, message) { + message = message || `${received} <= ${other}`; + return { pass: received <= other, message }; }, - toBeNull: function(value, message) { - message = message || `${value} == null`; - return { pass: value === null, message }; + toBeNull: function(received, message) { + message = message || `${received} == null`; + return { pass: received === null, message }; }, - toContain: function(value, other, message) { - message = message || `${value} ⊇ ${other}`; - return { pass: value.includes(other), message }; + toContain: function(received, other, message) { + message = message || `${received} ⊇ ${other}`; + return { pass: received.includes(other), message }; }, - toEqual: function(value, other, message) { - const valueJson = stringify(value); - const otherJson = stringify(other); - message = message || `\n${valueJson} ≈ ${otherJson}`; - return { pass: valueJson === otherJson, message }; + toEqual: function(received, other, message) { + let receivedJson = stringify(received); + let otherJson = stringify(other); + let formatter = objectFormatter.bind(null, receivedJson, otherJson); + if (receivedJson.length < 40 && otherJson.length < 40) { + receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' '); + otherJson = otherJson.split('\n').map(line => line.trim()).join(' '); + formatter = stringFormatter.bind(null, receivedJson, otherJson); + } + message = message || `\n${receivedJson} ≈ ${otherJson}`; + return { pass: receivedJson === otherJson, message, formatter }; }, - toBeCloseTo: function(value, other, precision, message) { + toBeCloseTo: function(received, other, precision, message) { return { - pass: Math.abs(value - other) < Math.pow(10, -precision), + pass: Math.abs(received - other) < Math.pow(10, -precision), message }; }, - toBeInstanceOf: function(value, other, message) { - message = message || `${value.constructor.name} instanceof ${other.name}`; - return { pass: value instanceof other, message }; + toBeInstanceOf: function(received, other, message) { + message = message || `${received.constructor.name} instanceof ${other.name}`; + return { pass: received instanceof other, message }; }, }; function stringify(value) { function stabilize(key, object) { - if (typeof object !== 'object' || object === undefined || object === null) + if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object)) return object; const result = {}; for (const key of Object.keys(object).sort()) diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index 30d3425e56..f40b7b6ee8 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -14,24 +14,20 @@ * limitations under the License. */ -const RED_COLOR = '\x1b[31m'; -const GREEN_COLOR = '\x1b[32m'; -const GRAY_COLOR = '\x1b[90m'; -const YELLOW_COLOR = '\x1b[33m'; -const MAGENTA_COLOR = '\x1b[35m'; -const RESET_COLOR = '\x1b[0m'; +const fs = require('fs'); +const colors = require('colors/safe'); +const {MatchError} = require('./Matchers.js'); class Reporter { constructor(runner, options = {}) { const { - projectFolder = null, showSlowTests = 3, showSkippedTests = Infinity, verbose = false, summary = true, } = options; + this._filePathToLines = new Map(); this._runner = runner; - this._projectFolder = projectFolder; this._showSlowTests = showSlowTests; this._showSkippedTests = showSkippedTests; this._verbose = verbose; @@ -48,19 +44,20 @@ class Reporter { this._testCounter = 0; this._timestamp = Date.now(); const allTests = this._runner.tests(); - if (allTests.length === runnableTests.length) - console.log(`Running all ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} tests on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); - else - console.log(`Running ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} focused tests out of total ${YELLOW_COLOR}${allTests.length}${RESET_COLOR} on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); + if (allTests.length === runnableTests.length) { + console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`); + } else { + console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`); + } } _printTermination(result, message, error) { - console.log(`${RED_COLOR}## ${result.toUpperCase()} ##${RESET_COLOR}`); + console.log(colors.red(`## ${result.toUpperCase()} ##`)); console.log('Message:'); - console.log(` ${RED_COLOR}${message}${RESET_COLOR}`); + console.log(` ${colors.red(message)}`); if (error && error.stack) { console.log('Stack:'); - console.log(error.stack.split('\n').map(line => ' ' + line).join('\n')); + console.log(padLines(error.stack, 2)); } console.log('WORKERS STATE'); const workerIds = Array.from(this._workersState.keys()); @@ -69,23 +66,25 @@ class Reporter { const {isRunning, test} = this._workersState.get(workerId); let description = ''; if (isRunning) - description = `${YELLOW_COLOR}RUNNING${RESET_COLOR}`; + description = colors.yellow('RUNNING'); else if (test.result === 'ok') - description = `${GREEN_COLOR}OK${RESET_COLOR}`; + description = colors.green('OK'); else if (test.result === 'skipped') - description = `${YELLOW_COLOR}SKIPPED${RESET_COLOR}`; + description = colors.yellow('SKIPPED'); else if (test.result === 'failed') - description = `${RED_COLOR}FAILED${RESET_COLOR}`; + description = colors.red('FAILED'); else if (test.result === 'crashed') - description = `${RED_COLOR}CRASHED${RESET_COLOR}`; + description = colors.red('CRASHED'); else if (test.result === 'timedout') - description = `${RED_COLOR}TIMEDOUT${RESET_COLOR}`; + description = colors.red('TIMEDOUT'); else if (test.result === 'terminated') - description = `${MAGENTA_COLOR}TERMINATED${RESET_COLOR}`; + description = colors.magenta('TERMINATED'); else - description = `${RED_COLOR}${RESET_COLOR}`; - console.log(` ${workerId}: [${description}] ${test.fullName} (${formatTestLocation(test)})`); + description = colors.red(''); + console.log(` ${workerId}: [${description}] ${test.fullName} (${formatLocation(test.location)})`); } + console.log(''); + console.log(''); process.exitCode = 2; } @@ -105,24 +104,7 @@ class Reporter { console.log('\nFailures:'); for (let i = 0; i < failedTests.length; ++i) { const test = failedTests[i]; - console.log(`${i + 1}) ${test.fullName} (${formatTestLocation(test)})`); - if (test.result === 'timedout') { - console.log(' Message:'); - console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`); - } else if (test.result === 'crashed') { - console.log(' Message:'); - console.log(` ${RED_COLOR}CRASHED${RESET_COLOR}`); - } else { - console.log(' Message:'); - console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`); - console.log(' Stack:'); - if (test.error.stack) - console.log(this._beautifyStack(test.error.stack)); - } - if (test.output) { - console.log(' Output:'); - console.log(test.output.split('\n').map(line => ' ' + line).join('\n')); - } + this._printVerboseTestResult(i + 1, test); console.log(''); } } @@ -132,12 +114,12 @@ class Reporter { if (skippedTests.length > 0) { console.log('\nSkipped:'); skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => { - console.log(`${index + 1}) ${test.fullName} (${formatTestLocation(test)})`); + console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`); }); } if (this._showSkippedTests < skippedTests.length) { console.log(''); - console.log(`... and ${YELLOW_COLOR}${skippedTests.length - this._showSkippedTests}${RESET_COLOR} more skipped tests ...`); + console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`); } } @@ -151,7 +133,7 @@ class Reporter { for (let i = 0; i < slowTests.length; ++i) { const test = slowTests[i]; const duration = test.endTimestamp - test.startTimestamp; - console.log(` (${i + 1}) ${YELLOW_COLOR}${duration / 1000}s${RESET_COLOR} - ${test.fullName} (${formatTestLocation(test)})`); + console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName} (${formatLocation(test.location)})`); } } @@ -160,39 +142,18 @@ class Reporter { const okTestsLength = executedTests.length - failedTests.length - skippedTests.length; let summaryText = ''; if (failedTests.length || skippedTests.length) { - const summary = [`ok - ${GREEN_COLOR}${okTestsLength}${RESET_COLOR}`]; + const summary = [`ok - ${colors.green(okTestsLength)}`]; if (failedTests.length) - summary.push(`failed - ${RED_COLOR}${failedTests.length}${RESET_COLOR}`); + summary.push(`failed - ${colors.red(failedTests.length)}`); if (skippedTests.length) - summary.push(`skipped - ${YELLOW_COLOR}${skippedTests.length}${RESET_COLOR}`); - summaryText = `(${summary.join(', ')})`; + summary.push(`skipped - ${colors.yellow(skippedTests.length)}`); + summaryText = ` (${summary.join(', ')})`; } - console.log(`\nRan ${executedTests.length} ${summaryText} of ${tests.length} test(s)`); + console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`); const milliseconds = Date.now() - this._timestamp; const seconds = milliseconds / 1000; - console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`); - } - - _beautifyStack(stack) { - if (!this._projectFolder) - return stack; - const lines = stack.split('\n').map(line => ' ' + line); - // Find last stack line that include testrunner code. - let index = 0; - while (index < lines.length && !lines[index].includes(__dirname)) - ++index; - while (index < lines.length && lines[index].includes(__dirname)) - ++index; - if (index >= lines.length) - return stack; - const line = lines[index]; - const fromIndex = line.lastIndexOf(this._projectFolder) + this._projectFolder.length; - let toIndex = line.lastIndexOf(')'); - if (toIndex === -1) - toIndex = line.length; - lines[index] = line.substring(0, fromIndex) + YELLOW_COLOR + line.substring(fromIndex, toIndex) + RESET_COLOR + line.substring(toIndex); - return lines.join('\n'); + console.log(`Finished in ${colors.yellow(seconds)} seconds`); } _onTestStarted(test, workerId) { @@ -203,55 +164,92 @@ class Reporter { this._workersState.set(workerId, {test, isRunning: false}); if (this._verbose) { ++this._testCounter; - let prefix = `${this._testCounter})`; - if (this._runner.parallel() > 1) - prefix += ` ${GRAY_COLOR}[worker = ${workerId}]${RESET_COLOR}`; - if (test.result === 'ok') { - console.log(`${prefix} ${GREEN_COLOR}[ OK ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'terminated') { - console.log(`${prefix} ${MAGENTA_COLOR}[ TERMINATED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'crashed') { - console.log(`${prefix} ${RED_COLOR}[ CRASHED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'skipped') { - console.log(`${prefix} ${YELLOW_COLOR}[SKIP]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'failed') { - console.log(`${prefix} ${RED_COLOR}[FAIL]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - console.log(' Message:'); - console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`); - console.log(' Stack:'); - if (test.error.stack) - console.log(this._beautifyStack(test.error.stack)); - if (test.output) { - console.log(' Output:'); - console.log(test.output.split('\n').map(line => ' ' + line).join('\n')); - } - } else if (test.result === 'timedout') { - console.log(`${prefix} ${RED_COLOR}[TIME]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - console.log(' Message:'); - console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`); - } + this._printVerboseTestResult(this._testCounter, test, workerId); } else { if (test.result === 'ok') - process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`); + process.stdout.write(colors.green('.')); else if (test.result === 'skipped') - process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`); + process.stdout.write(colors.yellow('*')); else if (test.result === 'failed') - process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`); + process.stdout.write(colors.red('F')); else if (test.result === 'crashed') - process.stdout.write(`${RED_COLOR}C${RESET_COLOR}`); + process.stdout.write(colors.red('C')); else if (test.result === 'terminated') - process.stdout.write(`${MAGENTA_COLOR}.${RESET_COLOR}`); + process.stdout.write(colors.magenta('.')); else if (test.result === 'timedout') - process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`); + process.stdout.write(colors.red('T')); + } + } + + _printVerboseTestResult(resultIndex, test, workerId = undefined) { + let prefix = `${resultIndex})`; + if (this._runner.parallel() > 1 && workerId !== undefined) + prefix += ' ' + colors.gray(`[worker = ${workerId}]`); + if (test.result === 'ok') { + console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'terminated') { + console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'crashed') { + console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'skipped') { + console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'timedout') { + console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'failed') { + console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName} (${formatLocation(test.location)})`); + if (test.error instanceof MatchError) { + let lines = this._filePathToLines.get(test.error.location.filePath); + if (!lines) { + try { + lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n'); + } catch (e) { + lines = []; + } + this._filePathToLines.set(test.error.location.filePath, lines); + } + const lineNumber = test.error.location.lineNumber; + if (lineNumber < lines.length) { + const lineNumberLength = (lineNumber + 1 + '').length; + const FROM = Math.max(test.location.lineNumber - 1, lineNumber - 5); + const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n'); + const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^'; + console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n'); + } + console.log(padLines(test.error.formatter(), 4)); + console.log(''); + } else { + console.log(' Message:'); + console.log(` ${colors.red(test.error.message || test.error)}`); + if (test.error.stack) { + console.log(' Stack:'); + let stack = test.error.stack; + // Highlight first test location, if any. + const match = stack.match(new RegExp(test.location.filePath + ':(\\d+):(\\d+)')); + if (match) { + const [, line, column] = match; + const fileName = `${test.location.fileName}:${line}:${column}`; + stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName)); + } + console.log(padLines(stack, 4)); + } + } + if (test.output) { + console.log(' Output:'); + console.log(padLines(test.output, 4)); + } } } } -function formatTestLocation(test) { - const location = test.location; +function formatLocation(location) { if (!location) return ''; - return `${YELLOW_COLOR}${location.fileName}:${location.lineNumber}:${location.columnNumber}${RESET_COLOR}`; + return colors.yellow(`${location.fileName}:${location.lineNumber}:${location.columnNumber}`); +} + +function padLines(text, spaces = 0) { + const indent = ' '.repeat(spaces); + return text.split('\n').map(line => indent + line).join('\n'); } module.exports = Reporter; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index 7598eb8c76..74793fc994 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -23,6 +23,7 @@ const Multimap = require('./Multimap'); const fs = require('fs'); const {SourceMapSupport} = require('./SourceMapSupport'); const debug = require('debug'); +const {getCallerLocation} = require('./utils'); const INFINITE_TIMEOUT = 2147483647; @@ -41,7 +42,7 @@ class UserCallback { }); this.timeout = timeout; - this.location = this._getLocation(); + this.location = getCallerLocation(__filename); } async run(...args) { @@ -62,35 +63,6 @@ class UserCallback { } } - _getLocation() { - const error = new Error(); - const stackFrames = error.stack.split('\n').slice(1); - // Find first stackframe that doesn't point to this file. - for (let frame of stackFrames) { - frame = frame.trim(); - if (!frame.startsWith('at ')) - return null; - if (frame.endsWith(')')) { - const from = frame.indexOf('('); - frame = frame.substring(from + 1, frame.length - 1); - } else { - frame = frame.substring('at '.length); - } - - const match = frame.match(/^(.*):(\d+):(\d+)$/); - if (!match) - return null; - const filePath = match[1]; - const lineNumber = parseInt(match[2], 10); - const columnNumber = parseInt(match[3], 10); - if (filePath === __filename) - continue; - const fileName = filePath.split(path.sep).pop(); - return { fileName, filePath, lineNumber, columnNumber }; - } - return null; - } - terminate() { this._terminateCallback(TerminatedError); } @@ -119,6 +91,7 @@ class Test { this.declaredMode = declaredMode; this._userCallback = new UserCallback(callback, timeout); this.location = this._userCallback.location; + this.timeout = timeout; // Test results this.result = null; @@ -422,7 +395,10 @@ class TestRunner extends EventEmitter { const runnableTests = this._runnableTests(); this.emit(TestRunner.Events.Started, runnableTests); this._runningPass = new TestPass(this, this._rootSuite, runnableTests, this._parallel, this._breakOnFailure); - const termination = await this._runningPass.run(); + const termination = await this._runningPass.run().catch(e => { + console.error(e); + throw e; + }); this._runningPass = null; const result = {}; if (termination) { diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js index 998ea3d150..84f9410671 100644 --- a/utils/testrunner/index.js +++ b/utils/testrunner/index.js @@ -16,6 +16,6 @@ const TestRunner = require('./TestRunner'); const Reporter = require('./Reporter'); -const Matchers = require('./Matchers'); +const {Matchers} = require('./Matchers'); module.exports = { TestRunner, Reporter, Matchers }; diff --git a/utils/testrunner/test/test.js b/utils/testrunner/test/test.js index d4f6b461a3..f18570038c 100644 --- a/utils/testrunner/test/test.js +++ b/utils/testrunner/test/test.js @@ -8,7 +8,6 @@ require('./testrunner.spec.js').addTests({testRunner, expect}); new Reporter(testRunner, { verbose: process.argv.includes('--verbose'), summary: true, - projectFolder: require('path').join(__dirname, '..'), showSlowTests: 0, }); testRunner.run(); diff --git a/utils/testrunner/utils.js b/utils/testrunner/utils.js new file mode 100644 index 0000000000..9934a7d7be --- /dev/null +++ b/utils/testrunner/utils.js @@ -0,0 +1,32 @@ +const path = require('path'); + +module.exports = { + getCallerLocation: function(filename) { + const error = new Error(); + const stackFrames = error.stack.split('\n').slice(1); + // Find first stackframe that doesn't point to this file. + for (let frame of stackFrames) { + frame = frame.trim(); + if (!frame.startsWith('at ')) + return null; + if (frame.endsWith(')')) { + const from = frame.indexOf('('); + frame = frame.substring(from + 1, frame.length - 1); + } else { + frame = frame.substring('at '.length); + } + + const match = frame.match(/^(.*):(\d+):(\d+)$/); + if (!match) + return null; + const filePath = match[1]; + const lineNumber = parseInt(match[2], 10); + const columnNumber = parseInt(match[3], 10); + if (filePath === __filename || filePath === filename) + continue; + const fileName = filePath.split(path.sep).pop(); + return { fileName, filePath, lineNumber, columnNumber }; + } + return null; + }, +};