feat(testrunner): better matchers (#1077)

This patch re-implements matching and reporting for test runner.
Among other improvements:
- test failures now show a short snippet from test
- test failures now explicitly say what received and what was expected
- `expect.toBe()` now does text diff when gets strings as input
- `expect.toEqual` now does object diff
This commit is contained in:
Andrey Lushnikov 2020-02-20 22:55:39 -08:00 committed by GitHub
parent 53a7e342e9
commit 0ded511d0b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 320 additions and 193 deletions

View File

@ -10,6 +10,8 @@ on:
env: env:
CI: true CI: true
# Force terminal colors. @see https://www.npmjs.com/package/colors
FORCE_COLOR: 1
jobs: jobs:
chromium_linux: chromium_linux:

View File

@ -62,6 +62,7 @@
"@types/ws": "^6.0.1", "@types/ws": "^6.0.1",
"@typescript-eslint/eslint-plugin": "^2.6.1", "@typescript-eslint/eslint-plugin": "^2.6.1",
"@typescript-eslint/parser": "^2.6.1", "@typescript-eslint/parser": "^2.6.1",
"colors": "^1.4.0",
"commonmark": "^0.28.1", "commonmark": "^0.28.1",
"cross-env": "^5.0.5", "cross-env": "^5.0.5",
"eslint": "^6.6.0", "eslint": "^6.6.0",

View File

@ -20,6 +20,7 @@ const Diff = require('text-diff');
const PNG = require('pngjs').PNG; const PNG = require('pngjs').PNG;
const jpeg = require('jpeg-js'); const jpeg = require('jpeg-js');
const pixelmatch = require('pixelmatch'); const pixelmatch = require('pixelmatch');
const c = require('colors/safe');
module.exports = {compare}; module.exports = {compare};
@ -51,7 +52,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) {
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer); const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
if (expected.width !== actual.width || expected.height !== actual.height) { if (expected.width !== actual.width || expected.height !== actual.height) {
return { return {
errorMessage: `Sizes differ: expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
}; };
} }
const diff = new PNG({width: expected.width, height: expected.height}); const diff = new PNG({width: expected.width, height: expected.height});
@ -110,23 +111,34 @@ function compare(goldenPath, outputPath, actual, goldenName) {
if (!comparator) { if (!comparator) {
return { return {
pass: false, pass: false,
message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName,
}; };
} }
const result = comparator(actual, expected, mimeType); const result = comparator(actual, expected, mimeType);
if (!result) if (!result)
return { pass: true }; return { pass: true };
ensureOutputDir(); ensureOutputDir();
const output = [
c.red(`GOLDEN FAILED: `) + c.yellow('"' + goldenName + '"'),
];
if (result.errorMessage)
output.push(' ' + result.errorMessage);
output.push('');
output.push(`Expected: ${c.yellow(expectedPath)}`);
if (goldenPath === outputPath) { if (goldenPath === outputPath) {
fs.writeFileSync(addSuffix(actualPath, '-actual'), actual); const filepath = addSuffix(actualPath, '-actual');
fs.writeFileSync(filepath, actual);
output.push(`Received: ${c.yellow(filepath)}`);
} else { } else {
fs.writeFileSync(actualPath, actual); fs.writeFileSync(actualPath, actual);
// Copy expected to the output/ folder for convenience. // Copy expected to the output/ folder for convenience.
fs.writeFileSync(addSuffix(actualPath, '-expected'), expected); fs.writeFileSync(addSuffix(actualPath, '-expected'), expected);
output.push(`Received: ${c.yellow(actualPath)}`);
} }
if (result.diff) { if (result.diff) {
const diffPath = addSuffix(actualPath, '-diff', result.diffExtension); const diffPath = addSuffix(actualPath, '-diff', result.diffExtension);
fs.writeFileSync(diffPath, result.diff); fs.writeFileSync(diffPath, result.diff);
output.push(` Diff: ${c.yellow(diffPath)}`);
} }
let message = goldenName + ' mismatch!'; let message = goldenName + ' mismatch!';
@ -134,7 +146,8 @@ function compare(goldenPath, outputPath, actual, goldenName) {
message += ' ' + result.errorMessage; message += ' ' + result.errorMessage;
return { return {
pass: false, pass: false,
message: message + ' ' + messageSuffix message: message + ' ' + messageSuffix,
formatter: () => output.join('\n'),
}; };
function ensureOutputDir() { function ensureOutputDir() {

View File

@ -124,7 +124,6 @@ if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
new Reporter(testRunner, { new Reporter(testRunner, {
verbose: process.argv.includes('--verbose'), verbose: process.argv.includes('--verbose'),
summary: !process.argv.includes('--verbose'), summary: !process.argv.includes('--verbose'),
projectFolder: utils.projectRoot(),
showSlowTests: process.env.CI ? 5 : 0, showSlowTests: process.env.CI ? 5 : 0,
showSkippedTests: 10, showSkippedTests: 10,
}); });

View File

@ -14,7 +14,11 @@
* limitations under the License. * limitations under the License.
*/ */
module.exports = class Matchers { const {getCallerLocation} = require('./utils.js');
const colors = require('colors/safe');
const Diff = require('text-diff');
class Matchers {
constructor(customMatchers = {}) { constructor(customMatchers = {}) {
this._matchers = {}; this._matchers = {};
Object.assign(this._matchers, DefaultMatchers); Object.assign(this._matchers, DefaultMatchers);
@ -26,99 +30,202 @@ module.exports = class Matchers {
this._matchers[name] = matcher; this._matchers[name] = matcher;
} }
expect(value) { expect(received) {
return new Expect(value, this._matchers); return new Expect(received, this._matchers);
} }
}; };
class MatchError extends Error {
constructor(message, formatter) {
super(message);
this.name = this.constructor.name;
this.formatter = formatter;
this.location = getCallerLocation(__filename);
Error.captureStackTrace(this, this.constructor);
}
}
module.exports = {Matchers, MatchError};
class Expect { class Expect {
constructor(value, matchers) { constructor(received, matchers) {
this.not = {}; this.not = {};
this.not.not = this; this.not.not = this;
for (const matcherName of Object.keys(matchers)) { for (const matcherName of Object.keys(matchers)) {
const matcher = matchers[matcherName]; const matcher = matchers[matcherName];
this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, value); this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received);
this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, value); this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received);
} }
function applyMatcher(matcherName, matcher, inverse, value, ...args) { function applyMatcher(matcherName, matcher, inverse, received, ...args) {
const result = matcher.call(null, value, ...args); const result = matcher.call(null, received, ...args);
const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : ''); const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : '');
if (result.pass === inverse) if (result.pass === inverse)
throw new Error(message); throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received));
} }
} }
} }
function defaultFormatter(received) {
return `Received: ${colors.red(JSON.stringify(received))}`;
}
function stringFormatter(received, expected) {
const diff = new Diff();
const result = diff.main(expected, received);
diff.cleanupSemantic(result);
const highlighted = result.map(([type, text]) => {
if (type === -1)
return colors.bgRed(text);
if (type === 1)
return colors.bgGreen.black(text);
return text;
}).join('');
const output = [
`Expected: ${expected}`,
`Received: ${highlighted}`,
];
for (let i = 0; i < Math.min(expected.length, received.length); ++i) {
if (expected[i] !== received[i]) {
const padding = ' '.repeat('Expected: '.length);
const firstDiffCharacter = '~'.repeat(i) + '^';
output.push(colors.red(padding + firstDiffCharacter));
break;
}
}
return output.join('\n');
}
function objectFormatter(received, expected) {
const receivedLines = received.split('\n');
const expectedLines = expected.split('\n');
const encodingMap = new Map();
const decodingMap = new Map();
const doEncodeLines = (lines) => {
let encoded = '';
for (const line of lines) {
let code = encodingMap.get(line);
if (!code) {
code = String.fromCodePoint(encodingMap.size);
encodingMap.set(line, code);
decodingMap.set(code, line);
}
encoded += code;
}
return encoded;
};
const doDecodeLines = (text) => {
let decoded = [];
for (const codepoint of [...text])
decoded.push(decodingMap.get(codepoint));
return decoded;
}
let receivedEncoded = doEncodeLines(received.split('\n'));
let expectedEncoded = doEncodeLines(expected.split('\n'));
const diff = new Diff();
const result = diff.main(expectedEncoded, receivedEncoded);
diff.cleanupSemantic(result);
const highlighted = result.map(([type, text]) => {
const lines = doDecodeLines(text);
if (type === -1)
return lines.map(line => '- ' + colors.bgRed(line));
if (type === 1)
return lines.map(line => '+ ' + colors.bgGreen.black(line));
return lines.map(line => ' ' + line);
}).flat().join('\n');
return `Received:\n${highlighted}`;
}
function toBeFormatter(received, expected) {
if (typeof expected === 'string' && typeof received === 'string') {
return stringFormatter(JSON.stringify(received), JSON.stringify(expected));
}
return [
`Expected: ${JSON.stringify(expected)}`,
`Received: ${colors.red(JSON.stringify(received))}`,
].join('\n');
}
const DefaultMatchers = { const DefaultMatchers = {
toBe: function(value, other, message) { toBe: function(received, expected, message) {
message = message || `${value} == ${other}`; message = message || `${received} == ${expected}`;
return { pass: value === other, message }; return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) };
}, },
toBeFalsy: function(value, message) { toBeFalsy: function(received, message) {
message = message || `${value}`; message = message || `${received}`;
return { pass: !value, message }; return { pass: !received, message };
}, },
toBeTruthy: function(value, message) { toBeTruthy: function(received, message) {
message = message || `${value}`; message = message || `${received}`;
return { pass: !!value, message }; return { pass: !!received, message };
}, },
toBeGreaterThan: function(value, other, message) { toBeGreaterThan: function(received, other, message) {
message = message || `${value} > ${other}`; message = message || `${received} > ${other}`;
return { pass: value > other, message }; return { pass: received > other, message };
}, },
toBeGreaterThanOrEqual: function(value, other, message) { toBeGreaterThanOrEqual: function(received, other, message) {
message = message || `${value} >= ${other}`; message = message || `${received} >= ${other}`;
return { pass: value >= other, message }; return { pass: received >= other, message };
}, },
toBeLessThan: function(value, other, message) { toBeLessThan: function(received, other, message) {
message = message || `${value} < ${other}`; message = message || `${received} < ${other}`;
return { pass: value < other, message }; return { pass: received < other, message };
}, },
toBeLessThanOrEqual: function(value, other, message) { toBeLessThanOrEqual: function(received, other, message) {
message = message || `${value} <= ${other}`; message = message || `${received} <= ${other}`;
return { pass: value <= other, message }; return { pass: received <= other, message };
}, },
toBeNull: function(value, message) { toBeNull: function(received, message) {
message = message || `${value} == null`; message = message || `${received} == null`;
return { pass: value === null, message }; return { pass: received === null, message };
}, },
toContain: function(value, other, message) { toContain: function(received, other, message) {
message = message || `${value}${other}`; message = message || `${received}${other}`;
return { pass: value.includes(other), message }; return { pass: received.includes(other), message };
}, },
toEqual: function(value, other, message) { toEqual: function(received, other, message) {
const valueJson = stringify(value); let receivedJson = stringify(received);
const otherJson = stringify(other); let otherJson = stringify(other);
message = message || `\n${valueJson}${otherJson}`; let formatter = objectFormatter.bind(null, receivedJson, otherJson);
return { pass: valueJson === otherJson, message }; if (receivedJson.length < 40 && otherJson.length < 40) {
receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' ');
otherJson = otherJson.split('\n').map(line => line.trim()).join(' ');
formatter = stringFormatter.bind(null, receivedJson, otherJson);
}
message = message || `\n${receivedJson}${otherJson}`;
return { pass: receivedJson === otherJson, message, formatter };
}, },
toBeCloseTo: function(value, other, precision, message) { toBeCloseTo: function(received, other, precision, message) {
return { return {
pass: Math.abs(value - other) < Math.pow(10, -precision), pass: Math.abs(received - other) < Math.pow(10, -precision),
message message
}; };
}, },
toBeInstanceOf: function(value, other, message) { toBeInstanceOf: function(received, other, message) {
message = message || `${value.constructor.name} instanceof ${other.name}`; message = message || `${received.constructor.name} instanceof ${other.name}`;
return { pass: value instanceof other, message }; return { pass: received instanceof other, message };
}, },
}; };
function stringify(value) { function stringify(value) {
function stabilize(key, object) { function stabilize(key, object) {
if (typeof object !== 'object' || object === undefined || object === null) if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object))
return object; return object;
const result = {}; const result = {};
for (const key of Object.keys(object).sort()) for (const key of Object.keys(object).sort())

View File

@ -14,24 +14,20 @@
* limitations under the License. * limitations under the License.
*/ */
const RED_COLOR = '\x1b[31m'; const fs = require('fs');
const GREEN_COLOR = '\x1b[32m'; const colors = require('colors/safe');
const GRAY_COLOR = '\x1b[90m'; const {MatchError} = require('./Matchers.js');
const YELLOW_COLOR = '\x1b[33m';
const MAGENTA_COLOR = '\x1b[35m';
const RESET_COLOR = '\x1b[0m';
class Reporter { class Reporter {
constructor(runner, options = {}) { constructor(runner, options = {}) {
const { const {
projectFolder = null,
showSlowTests = 3, showSlowTests = 3,
showSkippedTests = Infinity, showSkippedTests = Infinity,
verbose = false, verbose = false,
summary = true, summary = true,
} = options; } = options;
this._filePathToLines = new Map();
this._runner = runner; this._runner = runner;
this._projectFolder = projectFolder;
this._showSlowTests = showSlowTests; this._showSlowTests = showSlowTests;
this._showSkippedTests = showSkippedTests; this._showSkippedTests = showSkippedTests;
this._verbose = verbose; this._verbose = verbose;
@ -48,19 +44,20 @@ class Reporter {
this._testCounter = 0; this._testCounter = 0;
this._timestamp = Date.now(); this._timestamp = Date.now();
const allTests = this._runner.tests(); const allTests = this._runner.tests();
if (allTests.length === runnableTests.length) if (allTests.length === runnableTests.length) {
console.log(`Running all ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} tests on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
else } else {
console.log(`Running ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} focused tests out of total ${YELLOW_COLOR}${allTests.length}${RESET_COLOR} on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
}
} }
_printTermination(result, message, error) { _printTermination(result, message, error) {
console.log(`${RED_COLOR}## ${result.toUpperCase()} ##${RESET_COLOR}`); console.log(colors.red(`## ${result.toUpperCase()} ##`));
console.log('Message:'); console.log('Message:');
console.log(` ${RED_COLOR}${message}${RESET_COLOR}`); console.log(` ${colors.red(message)}`);
if (error && error.stack) { if (error && error.stack) {
console.log('Stack:'); console.log('Stack:');
console.log(error.stack.split('\n').map(line => ' ' + line).join('\n')); console.log(padLines(error.stack, 2));
} }
console.log('WORKERS STATE'); console.log('WORKERS STATE');
const workerIds = Array.from(this._workersState.keys()); const workerIds = Array.from(this._workersState.keys());
@ -69,23 +66,25 @@ class Reporter {
const {isRunning, test} = this._workersState.get(workerId); const {isRunning, test} = this._workersState.get(workerId);
let description = ''; let description = '';
if (isRunning) if (isRunning)
description = `${YELLOW_COLOR}RUNNING${RESET_COLOR}`; description = colors.yellow('RUNNING');
else if (test.result === 'ok') else if (test.result === 'ok')
description = `${GREEN_COLOR}OK${RESET_COLOR}`; description = colors.green('OK');
else if (test.result === 'skipped') else if (test.result === 'skipped')
description = `${YELLOW_COLOR}SKIPPED${RESET_COLOR}`; description = colors.yellow('SKIPPED');
else if (test.result === 'failed') else if (test.result === 'failed')
description = `${RED_COLOR}FAILED${RESET_COLOR}`; description = colors.red('FAILED');
else if (test.result === 'crashed') else if (test.result === 'crashed')
description = `${RED_COLOR}CRASHED${RESET_COLOR}`; description = colors.red('CRASHED');
else if (test.result === 'timedout') else if (test.result === 'timedout')
description = `${RED_COLOR}TIMEDOUT${RESET_COLOR}`; description = colors.red('TIMEDOUT');
else if (test.result === 'terminated') else if (test.result === 'terminated')
description = `${MAGENTA_COLOR}TERMINATED${RESET_COLOR}`; description = colors.magenta('TERMINATED');
else else
description = `${RED_COLOR}<UNKNOWN>${RESET_COLOR}`; description = colors.red('<UNKNOWN>');
console.log(` ${workerId}: [${description}] ${test.fullName} (${formatTestLocation(test)})`); console.log(` ${workerId}: [${description}] ${test.fullName} (${formatLocation(test.location)})`);
} }
console.log('');
console.log('');
process.exitCode = 2; process.exitCode = 2;
} }
@ -105,24 +104,7 @@ class Reporter {
console.log('\nFailures:'); console.log('\nFailures:');
for (let i = 0; i < failedTests.length; ++i) { for (let i = 0; i < failedTests.length; ++i) {
const test = failedTests[i]; const test = failedTests[i];
console.log(`${i + 1}) ${test.fullName} (${formatTestLocation(test)})`); this._printVerboseTestResult(i + 1, test);
if (test.result === 'timedout') {
console.log(' Message:');
console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`);
} else if (test.result === 'crashed') {
console.log(' Message:');
console.log(` ${RED_COLOR}CRASHED${RESET_COLOR}`);
} else {
console.log(' Message:');
console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`);
console.log(' Stack:');
if (test.error.stack)
console.log(this._beautifyStack(test.error.stack));
}
if (test.output) {
console.log(' Output:');
console.log(test.output.split('\n').map(line => ' ' + line).join('\n'));
}
console.log(''); console.log('');
} }
} }
@ -132,12 +114,12 @@ class Reporter {
if (skippedTests.length > 0) { if (skippedTests.length > 0) {
console.log('\nSkipped:'); console.log('\nSkipped:');
skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => { skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => {
console.log(`${index + 1}) ${test.fullName} (${formatTestLocation(test)})`); console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`);
}); });
} }
if (this._showSkippedTests < skippedTests.length) { if (this._showSkippedTests < skippedTests.length) {
console.log(''); console.log('');
console.log(`... and ${YELLOW_COLOR}${skippedTests.length - this._showSkippedTests}${RESET_COLOR} more skipped tests ...`); console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`);
} }
} }
@ -151,7 +133,7 @@ class Reporter {
for (let i = 0; i < slowTests.length; ++i) { for (let i = 0; i < slowTests.length; ++i) {
const test = slowTests[i]; const test = slowTests[i];
const duration = test.endTimestamp - test.startTimestamp; const duration = test.endTimestamp - test.startTimestamp;
console.log(` (${i + 1}) ${YELLOW_COLOR}${duration / 1000}s${RESET_COLOR} - ${test.fullName} (${formatTestLocation(test)})`); console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName} (${formatLocation(test.location)})`);
} }
} }
@ -160,39 +142,18 @@ class Reporter {
const okTestsLength = executedTests.length - failedTests.length - skippedTests.length; const okTestsLength = executedTests.length - failedTests.length - skippedTests.length;
let summaryText = ''; let summaryText = '';
if (failedTests.length || skippedTests.length) { if (failedTests.length || skippedTests.length) {
const summary = [`ok - ${GREEN_COLOR}${okTestsLength}${RESET_COLOR}`]; const summary = [`ok - ${colors.green(okTestsLength)}`];
if (failedTests.length) if (failedTests.length)
summary.push(`failed - ${RED_COLOR}${failedTests.length}${RESET_COLOR}`); summary.push(`failed - ${colors.red(failedTests.length)}`);
if (skippedTests.length) if (skippedTests.length)
summary.push(`skipped - ${YELLOW_COLOR}${skippedTests.length}${RESET_COLOR}`); summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
summaryText = `(${summary.join(', ')})`; summaryText = ` (${summary.join(', ')})`;
} }
console.log(`\nRan ${executedTests.length} ${summaryText} of ${tests.length} test(s)`); console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`);
const milliseconds = Date.now() - this._timestamp; const milliseconds = Date.now() - this._timestamp;
const seconds = milliseconds / 1000; const seconds = milliseconds / 1000;
console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`); console.log(`Finished in ${colors.yellow(seconds)} seconds`);
}
_beautifyStack(stack) {
if (!this._projectFolder)
return stack;
const lines = stack.split('\n').map(line => ' ' + line);
// Find last stack line that include testrunner code.
let index = 0;
while (index < lines.length && !lines[index].includes(__dirname))
++index;
while (index < lines.length && lines[index].includes(__dirname))
++index;
if (index >= lines.length)
return stack;
const line = lines[index];
const fromIndex = line.lastIndexOf(this._projectFolder) + this._projectFolder.length;
let toIndex = line.lastIndexOf(')');
if (toIndex === -1)
toIndex = line.length;
lines[index] = line.substring(0, fromIndex) + YELLOW_COLOR + line.substring(fromIndex, toIndex) + RESET_COLOR + line.substring(toIndex);
return lines.join('\n');
} }
_onTestStarted(test, workerId) { _onTestStarted(test, workerId) {
@ -203,55 +164,92 @@ class Reporter {
this._workersState.set(workerId, {test, isRunning: false}); this._workersState.set(workerId, {test, isRunning: false});
if (this._verbose) { if (this._verbose) {
++this._testCounter; ++this._testCounter;
let prefix = `${this._testCounter})`; this._printVerboseTestResult(this._testCounter, test, workerId);
if (this._runner.parallel() > 1)
prefix += ` ${GRAY_COLOR}[worker = ${workerId}]${RESET_COLOR}`;
if (test.result === 'ok') {
console.log(`${prefix} ${GREEN_COLOR}[ OK ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
} else if (test.result === 'terminated') {
console.log(`${prefix} ${MAGENTA_COLOR}[ TERMINATED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
} else if (test.result === 'crashed') {
console.log(`${prefix} ${RED_COLOR}[ CRASHED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
} else if (test.result === 'skipped') {
console.log(`${prefix} ${YELLOW_COLOR}[SKIP]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
} else if (test.result === 'failed') {
console.log(`${prefix} ${RED_COLOR}[FAIL]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
console.log(' Message:');
console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`);
console.log(' Stack:');
if (test.error.stack)
console.log(this._beautifyStack(test.error.stack));
if (test.output) {
console.log(' Output:');
console.log(test.output.split('\n').map(line => ' ' + line).join('\n'));
}
} else if (test.result === 'timedout') {
console.log(`${prefix} ${RED_COLOR}[TIME]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
console.log(' Message:');
console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`);
}
} else { } else {
if (test.result === 'ok') if (test.result === 'ok')
process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`); process.stdout.write(colors.green('.'));
else if (test.result === 'skipped') else if (test.result === 'skipped')
process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`); process.stdout.write(colors.yellow('*'));
else if (test.result === 'failed') else if (test.result === 'failed')
process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`); process.stdout.write(colors.red('F'));
else if (test.result === 'crashed') else if (test.result === 'crashed')
process.stdout.write(`${RED_COLOR}C${RESET_COLOR}`); process.stdout.write(colors.red('C'));
else if (test.result === 'terminated') else if (test.result === 'terminated')
process.stdout.write(`${MAGENTA_COLOR}.${RESET_COLOR}`); process.stdout.write(colors.magenta('.'));
else if (test.result === 'timedout') else if (test.result === 'timedout')
process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`); process.stdout.write(colors.red('T'));
}
}
_printVerboseTestResult(resultIndex, test, workerId = undefined) {
let prefix = `${resultIndex})`;
if (this._runner.parallel() > 1 && workerId !== undefined)
prefix += ' ' + colors.gray(`[worker = ${workerId}]`);
if (test.result === 'ok') {
console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'terminated') {
console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'crashed') {
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'skipped') {
console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'timedout') {
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'failed') {
console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName} (${formatLocation(test.location)})`);
if (test.error instanceof MatchError) {
let lines = this._filePathToLines.get(test.error.location.filePath);
if (!lines) {
try {
lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n');
} catch (e) {
lines = [];
}
this._filePathToLines.set(test.error.location.filePath, lines);
}
const lineNumber = test.error.location.lineNumber;
if (lineNumber < lines.length) {
const lineNumberLength = (lineNumber + 1 + '').length;
const FROM = Math.max(test.location.lineNumber - 1, lineNumber - 5);
const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n');
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^';
console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n');
}
console.log(padLines(test.error.formatter(), 4));
console.log('');
} else {
console.log(' Message:');
console.log(` ${colors.red(test.error.message || test.error)}`);
if (test.error.stack) {
console.log(' Stack:');
let stack = test.error.stack;
// Highlight first test location, if any.
const match = stack.match(new RegExp(test.location.filePath + ':(\\d+):(\\d+)'));
if (match) {
const [, line, column] = match;
const fileName = `${test.location.fileName}:${line}:${column}`;
stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName));
}
console.log(padLines(stack, 4));
}
}
if (test.output) {
console.log(' Output:');
console.log(padLines(test.output, 4));
}
} }
} }
} }
function formatTestLocation(test) { function formatLocation(location) {
const location = test.location;
if (!location) if (!location)
return ''; return '';
return `${YELLOW_COLOR}${location.fileName}:${location.lineNumber}:${location.columnNumber}${RESET_COLOR}`; return colors.yellow(`${location.fileName}:${location.lineNumber}:${location.columnNumber}`);
}
function padLines(text, spaces = 0) {
const indent = ' '.repeat(spaces);
return text.split('\n').map(line => indent + line).join('\n');
} }
module.exports = Reporter; module.exports = Reporter;

View File

@ -23,6 +23,7 @@ const Multimap = require('./Multimap');
const fs = require('fs'); const fs = require('fs');
const {SourceMapSupport} = require('./SourceMapSupport'); const {SourceMapSupport} = require('./SourceMapSupport');
const debug = require('debug'); const debug = require('debug');
const {getCallerLocation} = require('./utils');
const INFINITE_TIMEOUT = 2147483647; const INFINITE_TIMEOUT = 2147483647;
@ -41,7 +42,7 @@ class UserCallback {
}); });
this.timeout = timeout; this.timeout = timeout;
this.location = this._getLocation(); this.location = getCallerLocation(__filename);
} }
async run(...args) { async run(...args) {
@ -62,35 +63,6 @@ class UserCallback {
} }
} }
_getLocation() {
const error = new Error();
const stackFrames = error.stack.split('\n').slice(1);
// Find first stackframe that doesn't point to this file.
for (let frame of stackFrames) {
frame = frame.trim();
if (!frame.startsWith('at '))
return null;
if (frame.endsWith(')')) {
const from = frame.indexOf('(');
frame = frame.substring(from + 1, frame.length - 1);
} else {
frame = frame.substring('at '.length);
}
const match = frame.match(/^(.*):(\d+):(\d+)$/);
if (!match)
return null;
const filePath = match[1];
const lineNumber = parseInt(match[2], 10);
const columnNumber = parseInt(match[3], 10);
if (filePath === __filename)
continue;
const fileName = filePath.split(path.sep).pop();
return { fileName, filePath, lineNumber, columnNumber };
}
return null;
}
terminate() { terminate() {
this._terminateCallback(TerminatedError); this._terminateCallback(TerminatedError);
} }
@ -119,6 +91,7 @@ class Test {
this.declaredMode = declaredMode; this.declaredMode = declaredMode;
this._userCallback = new UserCallback(callback, timeout); this._userCallback = new UserCallback(callback, timeout);
this.location = this._userCallback.location; this.location = this._userCallback.location;
this.timeout = timeout;
// Test results // Test results
this.result = null; this.result = null;
@ -422,7 +395,10 @@ class TestRunner extends EventEmitter {
const runnableTests = this._runnableTests(); const runnableTests = this._runnableTests();
this.emit(TestRunner.Events.Started, runnableTests); this.emit(TestRunner.Events.Started, runnableTests);
this._runningPass = new TestPass(this, this._rootSuite, runnableTests, this._parallel, this._breakOnFailure); this._runningPass = new TestPass(this, this._rootSuite, runnableTests, this._parallel, this._breakOnFailure);
const termination = await this._runningPass.run(); const termination = await this._runningPass.run().catch(e => {
console.error(e);
throw e;
});
this._runningPass = null; this._runningPass = null;
const result = {}; const result = {};
if (termination) { if (termination) {

View File

@ -16,6 +16,6 @@
const TestRunner = require('./TestRunner'); const TestRunner = require('./TestRunner');
const Reporter = require('./Reporter'); const Reporter = require('./Reporter');
const Matchers = require('./Matchers'); const {Matchers} = require('./Matchers');
module.exports = { TestRunner, Reporter, Matchers }; module.exports = { TestRunner, Reporter, Matchers };

View File

@ -8,7 +8,6 @@ require('./testrunner.spec.js').addTests({testRunner, expect});
new Reporter(testRunner, { new Reporter(testRunner, {
verbose: process.argv.includes('--verbose'), verbose: process.argv.includes('--verbose'),
summary: true, summary: true,
projectFolder: require('path').join(__dirname, '..'),
showSlowTests: 0, showSlowTests: 0,
}); });
testRunner.run(); testRunner.run();

32
utils/testrunner/utils.js Normal file
View File

@ -0,0 +1,32 @@
const path = require('path');
module.exports = {
getCallerLocation: function(filename) {
const error = new Error();
const stackFrames = error.stack.split('\n').slice(1);
// Find first stackframe that doesn't point to this file.
for (let frame of stackFrames) {
frame = frame.trim();
if (!frame.startsWith('at '))
return null;
if (frame.endsWith(')')) {
const from = frame.indexOf('(');
frame = frame.substring(from + 1, frame.length - 1);
} else {
frame = frame.substring('at '.length);
}
const match = frame.match(/^(.*):(\d+):(\d+)$/);
if (!match)
return null;
const filePath = match[1];
const lineNumber = parseInt(match[2], 10);
const columnNumber = parseInt(match[3], 10);
if (filePath === __filename || filePath === filename)
continue;
const fileName = filePath.split(path.sep).pop();
return { fileName, filePath, lineNumber, columnNumber };
}
return null;
},
};