mirror of
https://github.com/microsoft/playwright.git
synced 2025-01-03 17:09:01 +03:00
feat(testrunner): better matchers (#1077)
This patch re-implements matching and reporting for test runner. Among other improvements: - test failures now show a short snippet from test - test failures now explicitly say what received and what was expected - `expect.toBe()` now does text diff when gets strings as input - `expect.toEqual` now does object diff
This commit is contained in:
Andrey Lushnikov
GitHub
parent
53a7e342e9
commit
0ded511d0b
2
.github/workflows/tests.yml
vendored
2
.github/workflows/tests.yml
vendored
@ -10,6 +10,8 @@ on:
|
||||
|
||||
env:
|
||||
CI: true
|
||||
# Force terminal colors. @see https://www.npmjs.com/package/colors
|
||||
FORCE_COLOR: 1
|
||||
|
||||
jobs:
|
||||
chromium_linux:
|
||||
|
||||
@ -62,6 +62,7 @@
|
||||
"@types/ws": "^6.0.1",
|
||||
"@typescript-eslint/eslint-plugin": "^2.6.1",
|
||||
"@typescript-eslint/parser": "^2.6.1",
|
||||
"colors": "^1.4.0",
|
||||
"commonmark": "^0.28.1",
|
||||
"cross-env": "^5.0.5",
|
||||
"eslint": "^6.6.0",
|
||||
|
||||
@ -20,6 +20,7 @@ const Diff = require('text-diff');
|
||||
const PNG = require('pngjs').PNG;
|
||||
const jpeg = require('jpeg-js');
|
||||
const pixelmatch = require('pixelmatch');
|
||||
const c = require('colors/safe');
|
||||
|
||||
module.exports = {compare};
|
||||
|
||||
@ -51,7 +52,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) {
|
||||
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
|
||||
if (expected.width !== actual.width || expected.height !== actual.height) {
|
||||
return {
|
||||
errorMessage: `Sizes differ: expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
|
||||
errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
|
||||
};
|
||||
}
|
||||
const diff = new PNG({width: expected.width, height: expected.height});
|
||||
@ -110,23 +111,34 @@ function compare(goldenPath, outputPath, actual, goldenName) {
|
||||
if (!comparator) {
|
||||
return {
|
||||
pass: false,
|
||||
message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName
|
||||
message: 'Failed to find comparator with type ' + mimeType + ': ' + goldenName,
|
||||
};
|
||||
}
|
||||
const result = comparator(actual, expected, mimeType);
|
||||
if (!result)
|
||||
return { pass: true };
|
||||
ensureOutputDir();
|
||||
const output = [
|
||||
c.red(`GOLDEN FAILED: `) + c.yellow('"' + goldenName + '"'),
|
||||
];
|
||||
if (result.errorMessage)
|
||||
output.push(' ' + result.errorMessage);
|
||||
output.push('');
|
||||
output.push(`Expected: ${c.yellow(expectedPath)}`);
|
||||
if (goldenPath === outputPath) {
|
||||
fs.writeFileSync(addSuffix(actualPath, '-actual'), actual);
|
||||
const filepath = addSuffix(actualPath, '-actual');
|
||||
fs.writeFileSync(filepath, actual);
|
||||
output.push(`Received: ${c.yellow(filepath)}`);
|
||||
} else {
|
||||
fs.writeFileSync(actualPath, actual);
|
||||
// Copy expected to the output/ folder for convenience.
|
||||
fs.writeFileSync(addSuffix(actualPath, '-expected'), expected);
|
||||
output.push(`Received: ${c.yellow(actualPath)}`);
|
||||
}
|
||||
if (result.diff) {
|
||||
const diffPath = addSuffix(actualPath, '-diff', result.diffExtension);
|
||||
fs.writeFileSync(diffPath, result.diff);
|
||||
output.push(` Diff: ${c.yellow(diffPath)}`);
|
||||
}
|
||||
|
||||
let message = goldenName + ' mismatch!';
|
||||
@ -134,7 +146,8 @@ function compare(goldenPath, outputPath, actual, goldenName) {
|
||||
message += ' ' + result.errorMessage;
|
||||
return {
|
||||
pass: false,
|
||||
message: message + ' ' + messageSuffix
|
||||
message: message + ' ' + messageSuffix,
|
||||
formatter: () => output.join('\n'),
|
||||
};
|
||||
|
||||
function ensureOutputDir() {
|
||||
|
||||
@ -124,7 +124,6 @@ if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
|
||||
new Reporter(testRunner, {
|
||||
verbose: process.argv.includes('--verbose'),
|
||||
summary: !process.argv.includes('--verbose'),
|
||||
projectFolder: utils.projectRoot(),
|
||||
showSlowTests: process.env.CI ? 5 : 0,
|
||||
showSkippedTests: 10,
|
||||
});
|
||||
|
||||
@ -14,7 +14,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
module.exports = class Matchers {
|
||||
const {getCallerLocation} = require('./utils.js');
|
||||
const colors = require('colors/safe');
|
||||
const Diff = require('text-diff');
|
||||
|
||||
class Matchers {
|
||||
constructor(customMatchers = {}) {
|
||||
this._matchers = {};
|
||||
Object.assign(this._matchers, DefaultMatchers);
|
||||
@ -26,99 +30,202 @@ module.exports = class Matchers {
|
||||
this._matchers[name] = matcher;
|
||||
}
|
||||
|
||||
expect(value) {
|
||||
return new Expect(value, this._matchers);
|
||||
expect(received) {
|
||||
return new Expect(received, this._matchers);
|
||||
}
|
||||
};
|
||||
|
||||
class MatchError extends Error {
|
||||
constructor(message, formatter) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
this.formatter = formatter;
|
||||
this.location = getCallerLocation(__filename);
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {Matchers, MatchError};
|
||||
|
||||
class Expect {
|
||||
constructor(value, matchers) {
|
||||
constructor(received, matchers) {
|
||||
this.not = {};
|
||||
this.not.not = this;
|
||||
for (const matcherName of Object.keys(matchers)) {
|
||||
const matcher = matchers[matcherName];
|
||||
this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, value);
|
||||
this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, value);
|
||||
this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received);
|
||||
this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received);
|
||||
}
|
||||
|
||||
function applyMatcher(matcherName, matcher, inverse, value, ...args) {
|
||||
const result = matcher.call(null, value, ...args);
|
||||
function applyMatcher(matcherName, matcher, inverse, received, ...args) {
|
||||
const result = matcher.call(null, received, ...args);
|
||||
const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : '');
|
||||
if (result.pass === inverse)
|
||||
throw new Error(message);
|
||||
throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function defaultFormatter(received) {
|
||||
return `Received: ${colors.red(JSON.stringify(received))}`;
|
||||
}
|
||||
|
||||
function stringFormatter(received, expected) {
|
||||
const diff = new Diff();
|
||||
const result = diff.main(expected, received);
|
||||
diff.cleanupSemantic(result);
|
||||
const highlighted = result.map(([type, text]) => {
|
||||
if (type === -1)
|
||||
return colors.bgRed(text);
|
||||
if (type === 1)
|
||||
return colors.bgGreen.black(text);
|
||||
return text;
|
||||
}).join('');
|
||||
const output = [
|
||||
`Expected: ${expected}`,
|
||||
`Received: ${highlighted}`,
|
||||
];
|
||||
for (let i = 0; i < Math.min(expected.length, received.length); ++i) {
|
||||
if (expected[i] !== received[i]) {
|
||||
const padding = ' '.repeat('Expected: '.length);
|
||||
const firstDiffCharacter = '~'.repeat(i) + '^';
|
||||
output.push(colors.red(padding + firstDiffCharacter));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return output.join('\n');
|
||||
}
|
||||
|
||||
function objectFormatter(received, expected) {
|
||||
const receivedLines = received.split('\n');
|
||||
const expectedLines = expected.split('\n');
|
||||
const encodingMap = new Map();
|
||||
const decodingMap = new Map();
|
||||
|
||||
const doEncodeLines = (lines) => {
|
||||
let encoded = '';
|
||||
for (const line of lines) {
|
||||
let code = encodingMap.get(line);
|
||||
if (!code) {
|
||||
code = String.fromCodePoint(encodingMap.size);
|
||||
encodingMap.set(line, code);
|
||||
decodingMap.set(code, line);
|
||||
}
|
||||
encoded += code;
|
||||
}
|
||||
return encoded;
|
||||
};
|
||||
|
||||
const doDecodeLines = (text) => {
|
||||
let decoded = [];
|
||||
for (const codepoint of [...text])
|
||||
decoded.push(decodingMap.get(codepoint));
|
||||
return decoded;
|
||||
}
|
||||
|
||||
let receivedEncoded = doEncodeLines(received.split('\n'));
|
||||
let expectedEncoded = doEncodeLines(expected.split('\n'));
|
||||
|
||||
const diff = new Diff();
|
||||
const result = diff.main(expectedEncoded, receivedEncoded);
|
||||
diff.cleanupSemantic(result);
|
||||
|
||||
const highlighted = result.map(([type, text]) => {
|
||||
const lines = doDecodeLines(text);
|
||||
if (type === -1)
|
||||
return lines.map(line => '- ' + colors.bgRed(line));
|
||||
if (type === 1)
|
||||
return lines.map(line => '+ ' + colors.bgGreen.black(line));
|
||||
return lines.map(line => ' ' + line);
|
||||
}).flat().join('\n');
|
||||
return `Received:\n${highlighted}`;
|
||||
}
|
||||
|
||||
function toBeFormatter(received, expected) {
|
||||
if (typeof expected === 'string' && typeof received === 'string') {
|
||||
return stringFormatter(JSON.stringify(received), JSON.stringify(expected));
|
||||
}
|
||||
return [
|
||||
`Expected: ${JSON.stringify(expected)}`,
|
||||
`Received: ${colors.red(JSON.stringify(received))}`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const DefaultMatchers = {
|
||||
toBe: function(value, other, message) {
|
||||
message = message || `${value} == ${other}`;
|
||||
return { pass: value === other, message };
|
||||
toBe: function(received, expected, message) {
|
||||
message = message || `${received} == ${expected}`;
|
||||
return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) };
|
||||
},
|
||||
|
||||
toBeFalsy: function(value, message) {
|
||||
message = message || `${value}`;
|
||||
return { pass: !value, message };
|
||||
toBeFalsy: function(received, message) {
|
||||
message = message || `${received}`;
|
||||
return { pass: !received, message };
|
||||
},
|
||||
|
||||
toBeTruthy: function(value, message) {
|
||||
message = message || `${value}`;
|
||||
return { pass: !!value, message };
|
||||
toBeTruthy: function(received, message) {
|
||||
message = message || `${received}`;
|
||||
return { pass: !!received, message };
|
||||
},
|
||||
|
||||
toBeGreaterThan: function(value, other, message) {
|
||||
message = message || `${value} > ${other}`;
|
||||
return { pass: value > other, message };
|
||||
toBeGreaterThan: function(received, other, message) {
|
||||
message = message || `${received} > ${other}`;
|
||||
return { pass: received > other, message };
|
||||
},
|
||||
|
||||
toBeGreaterThanOrEqual: function(value, other, message) {
|
||||
message = message || `${value} >= ${other}`;
|
||||
return { pass: value >= other, message };
|
||||
toBeGreaterThanOrEqual: function(received, other, message) {
|
||||
message = message || `${received} >= ${other}`;
|
||||
return { pass: received >= other, message };
|
||||
},
|
||||
|
||||
toBeLessThan: function(value, other, message) {
|
||||
message = message || `${value} < ${other}`;
|
||||
return { pass: value < other, message };
|
||||
toBeLessThan: function(received, other, message) {
|
||||
message = message || `${received} < ${other}`;
|
||||
return { pass: received < other, message };
|
||||
},
|
||||
|
||||
toBeLessThanOrEqual: function(value, other, message) {
|
||||
message = message || `${value} <= ${other}`;
|
||||
return { pass: value <= other, message };
|
||||
toBeLessThanOrEqual: function(received, other, message) {
|
||||
message = message || `${received} <= ${other}`;
|
||||
return { pass: received <= other, message };
|
||||
},
|
||||
|
||||
toBeNull: function(value, message) {
|
||||
message = message || `${value} == null`;
|
||||
return { pass: value === null, message };
|
||||
toBeNull: function(received, message) {
|
||||
message = message || `${received} == null`;
|
||||
return { pass: received === null, message };
|
||||
},
|
||||
|
||||
toContain: function(value, other, message) {
|
||||
message = message || `${value} ⊇ ${other}`;
|
||||
return { pass: value.includes(other), message };
|
||||
toContain: function(received, other, message) {
|
||||
message = message || `${received} ⊇ ${other}`;
|
||||
return { pass: received.includes(other), message };
|
||||
},
|
||||
|
||||
toEqual: function(value, other, message) {
|
||||
const valueJson = stringify(value);
|
||||
const otherJson = stringify(other);
|
||||
message = message || `\n${valueJson} ≈ ${otherJson}`;
|
||||
return { pass: valueJson === otherJson, message };
|
||||
toEqual: function(received, other, message) {
|
||||
let receivedJson = stringify(received);
|
||||
let otherJson = stringify(other);
|
||||
let formatter = objectFormatter.bind(null, receivedJson, otherJson);
|
||||
if (receivedJson.length < 40 && otherJson.length < 40) {
|
||||
receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' ');
|
||||
otherJson = otherJson.split('\n').map(line => line.trim()).join(' ');
|
||||
formatter = stringFormatter.bind(null, receivedJson, otherJson);
|
||||
}
|
||||
message = message || `\n${receivedJson} ≈ ${otherJson}`;
|
||||
return { pass: receivedJson === otherJson, message, formatter };
|
||||
},
|
||||
|
||||
toBeCloseTo: function(value, other, precision, message) {
|
||||
toBeCloseTo: function(received, other, precision, message) {
|
||||
return {
|
||||
pass: Math.abs(value - other) < Math.pow(10, -precision),
|
||||
pass: Math.abs(received - other) < Math.pow(10, -precision),
|
||||
message
|
||||
};
|
||||
},
|
||||
|
||||
toBeInstanceOf: function(value, other, message) {
|
||||
message = message || `${value.constructor.name} instanceof ${other.name}`;
|
||||
return { pass: value instanceof other, message };
|
||||
toBeInstanceOf: function(received, other, message) {
|
||||
message = message || `${received.constructor.name} instanceof ${other.name}`;
|
||||
return { pass: received instanceof other, message };
|
||||
},
|
||||
};
|
||||
|
||||
function stringify(value) {
|
||||
function stabilize(key, object) {
|
||||
if (typeof object !== 'object' || object === undefined || object === null)
|
||||
if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object))
|
||||
return object;
|
||||
const result = {};
|
||||
for (const key of Object.keys(object).sort())
|
||||
|
||||
@ -14,24 +14,20 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const RED_COLOR = '\x1b[31m';
|
||||
const GREEN_COLOR = '\x1b[32m';
|
||||
const GRAY_COLOR = '\x1b[90m';
|
||||
const YELLOW_COLOR = '\x1b[33m';
|
||||
const MAGENTA_COLOR = '\x1b[35m';
|
||||
const RESET_COLOR = '\x1b[0m';
|
||||
const fs = require('fs');
|
||||
const colors = require('colors/safe');
|
||||
const {MatchError} = require('./Matchers.js');
|
||||
|
||||
class Reporter {
|
||||
constructor(runner, options = {}) {
|
||||
const {
|
||||
projectFolder = null,
|
||||
showSlowTests = 3,
|
||||
showSkippedTests = Infinity,
|
||||
verbose = false,
|
||||
summary = true,
|
||||
} = options;
|
||||
this._filePathToLines = new Map();
|
||||
this._runner = runner;
|
||||
this._projectFolder = projectFolder;
|
||||
this._showSlowTests = showSlowTests;
|
||||
this._showSkippedTests = showSkippedTests;
|
||||
this._verbose = verbose;
|
||||
@ -48,19 +44,20 @@ class Reporter {
|
||||
this._testCounter = 0;
|
||||
this._timestamp = Date.now();
|
||||
const allTests = this._runner.tests();
|
||||
if (allTests.length === runnableTests.length)
|
||||
console.log(`Running all ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} tests on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`);
|
||||
else
|
||||
console.log(`Running ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} focused tests out of total ${YELLOW_COLOR}${allTests.length}${RESET_COLOR} on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`);
|
||||
if (allTests.length === runnableTests.length) {
|
||||
console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
|
||||
} else {
|
||||
console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
|
||||
}
|
||||
}
|
||||
|
||||
_printTermination(result, message, error) {
|
||||
console.log(`${RED_COLOR}## ${result.toUpperCase()} ##${RESET_COLOR}`);
|
||||
console.log(colors.red(`## ${result.toUpperCase()} ##`));
|
||||
console.log('Message:');
|
||||
console.log(` ${RED_COLOR}${message}${RESET_COLOR}`);
|
||||
console.log(` ${colors.red(message)}`);
|
||||
if (error && error.stack) {
|
||||
console.log('Stack:');
|
||||
console.log(error.stack.split('\n').map(line => ' ' + line).join('\n'));
|
||||
console.log(padLines(error.stack, 2));
|
||||
}
|
||||
console.log('WORKERS STATE');
|
||||
const workerIds = Array.from(this._workersState.keys());
|
||||
@ -69,23 +66,25 @@ class Reporter {
|
||||
const {isRunning, test} = this._workersState.get(workerId);
|
||||
let description = '';
|
||||
if (isRunning)
|
||||
description = `${YELLOW_COLOR}RUNNING${RESET_COLOR}`;
|
||||
description = colors.yellow('RUNNING');
|
||||
else if (test.result === 'ok')
|
||||
description = `${GREEN_COLOR}OK${RESET_COLOR}`;
|
||||
description = colors.green('OK');
|
||||
else if (test.result === 'skipped')
|
||||
description = `${YELLOW_COLOR}SKIPPED${RESET_COLOR}`;
|
||||
description = colors.yellow('SKIPPED');
|
||||
else if (test.result === 'failed')
|
||||
description = `${RED_COLOR}FAILED${RESET_COLOR}`;
|
||||
description = colors.red('FAILED');
|
||||
else if (test.result === 'crashed')
|
||||
description = `${RED_COLOR}CRASHED${RESET_COLOR}`;
|
||||
description = colors.red('CRASHED');
|
||||
else if (test.result === 'timedout')
|
||||
description = `${RED_COLOR}TIMEDOUT${RESET_COLOR}`;
|
||||
description = colors.red('TIMEDOUT');
|
||||
else if (test.result === 'terminated')
|
||||
description = `${MAGENTA_COLOR}TERMINATED${RESET_COLOR}`;
|
||||
description = colors.magenta('TERMINATED');
|
||||
else
|
||||
description = `${RED_COLOR}<UNKNOWN>${RESET_COLOR}`;
|
||||
console.log(` ${workerId}: [${description}] ${test.fullName} (${formatTestLocation(test)})`);
|
||||
description = colors.red('<UNKNOWN>');
|
||||
console.log(` ${workerId}: [${description}] ${test.fullName} (${formatLocation(test.location)})`);
|
||||
}
|
||||
console.log('');
|
||||
console.log('');
|
||||
process.exitCode = 2;
|
||||
}
|
||||
|
||||
@ -105,24 +104,7 @@ class Reporter {
|
||||
console.log('\nFailures:');
|
||||
for (let i = 0; i < failedTests.length; ++i) {
|
||||
const test = failedTests[i];
|
||||
console.log(`${i + 1}) ${test.fullName} (${formatTestLocation(test)})`);
|
||||
if (test.result === 'timedout') {
|
||||
console.log(' Message:');
|
||||
console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`);
|
||||
} else if (test.result === 'crashed') {
|
||||
console.log(' Message:');
|
||||
console.log(` ${RED_COLOR}CRASHED${RESET_COLOR}`);
|
||||
} else {
|
||||
console.log(' Message:');
|
||||
console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`);
|
||||
console.log(' Stack:');
|
||||
if (test.error.stack)
|
||||
console.log(this._beautifyStack(test.error.stack));
|
||||
}
|
||||
if (test.output) {
|
||||
console.log(' Output:');
|
||||
console.log(test.output.split('\n').map(line => ' ' + line).join('\n'));
|
||||
}
|
||||
this._printVerboseTestResult(i + 1, test);
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
@ -132,12 +114,12 @@ class Reporter {
|
||||
if (skippedTests.length > 0) {
|
||||
console.log('\nSkipped:');
|
||||
skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => {
|
||||
console.log(`${index + 1}) ${test.fullName} (${formatTestLocation(test)})`);
|
||||
console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`);
|
||||
});
|
||||
}
|
||||
if (this._showSkippedTests < skippedTests.length) {
|
||||
console.log('');
|
||||
console.log(`... and ${YELLOW_COLOR}${skippedTests.length - this._showSkippedTests}${RESET_COLOR} more skipped tests ...`);
|
||||
console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -151,7 +133,7 @@ class Reporter {
|
||||
for (let i = 0; i < slowTests.length; ++i) {
|
||||
const test = slowTests[i];
|
||||
const duration = test.endTimestamp - test.startTimestamp;
|
||||
console.log(` (${i + 1}) ${YELLOW_COLOR}${duration / 1000}s${RESET_COLOR} - ${test.fullName} (${formatTestLocation(test)})`);
|
||||
console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName} (${formatLocation(test.location)})`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -160,39 +142,18 @@ class Reporter {
|
||||
const okTestsLength = executedTests.length - failedTests.length - skippedTests.length;
|
||||
let summaryText = '';
|
||||
if (failedTests.length || skippedTests.length) {
|
||||
const summary = [`ok - ${GREEN_COLOR}${okTestsLength}${RESET_COLOR}`];
|
||||
const summary = [`ok - ${colors.green(okTestsLength)}`];
|
||||
if (failedTests.length)
|
||||
summary.push(`failed - ${RED_COLOR}${failedTests.length}${RESET_COLOR}`);
|
||||
summary.push(`failed - ${colors.red(failedTests.length)}`);
|
||||
if (skippedTests.length)
|
||||
summary.push(`skipped - ${YELLOW_COLOR}${skippedTests.length}${RESET_COLOR}`);
|
||||
summaryText = `(${summary.join(', ')})`;
|
||||
summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
|
||||
summaryText = ` (${summary.join(', ')})`;
|
||||
}
|
||||
|
||||
console.log(`\nRan ${executedTests.length} ${summaryText} of ${tests.length} test(s)`);
|
||||
console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`);
|
||||
const milliseconds = Date.now() - this._timestamp;
|
||||
const seconds = milliseconds / 1000;
|
||||
console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`);
|
||||
}
|
||||
|
||||
_beautifyStack(stack) {
|
||||
if (!this._projectFolder)
|
||||
return stack;
|
||||
const lines = stack.split('\n').map(line => ' ' + line);
|
||||
// Find last stack line that include testrunner code.
|
||||
let index = 0;
|
||||
while (index < lines.length && !lines[index].includes(__dirname))
|
||||
++index;
|
||||
while (index < lines.length && lines[index].includes(__dirname))
|
||||
++index;
|
||||
if (index >= lines.length)
|
||||
return stack;
|
||||
const line = lines[index];
|
||||
const fromIndex = line.lastIndexOf(this._projectFolder) + this._projectFolder.length;
|
||||
let toIndex = line.lastIndexOf(')');
|
||||
if (toIndex === -1)
|
||||
toIndex = line.length;
|
||||
lines[index] = line.substring(0, fromIndex) + YELLOW_COLOR + line.substring(fromIndex, toIndex) + RESET_COLOR + line.substring(toIndex);
|
||||
return lines.join('\n');
|
||||
console.log(`Finished in ${colors.yellow(seconds)} seconds`);
|
||||
}
|
||||
|
||||
_onTestStarted(test, workerId) {
|
||||
@ -203,55 +164,92 @@ class Reporter {
|
||||
this._workersState.set(workerId, {test, isRunning: false});
|
||||
if (this._verbose) {
|
||||
++this._testCounter;
|
||||
let prefix = `${this._testCounter})`;
|
||||
if (this._runner.parallel() > 1)
|
||||
prefix += ` ${GRAY_COLOR}[worker = ${workerId}]${RESET_COLOR}`;
|
||||
if (test.result === 'ok') {
|
||||
console.log(`${prefix} ${GREEN_COLOR}[ OK ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
} else if (test.result === 'terminated') {
|
||||
console.log(`${prefix} ${MAGENTA_COLOR}[ TERMINATED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
} else if (test.result === 'crashed') {
|
||||
console.log(`${prefix} ${RED_COLOR}[ CRASHED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
} else if (test.result === 'skipped') {
|
||||
console.log(`${prefix} ${YELLOW_COLOR}[SKIP]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
} else if (test.result === 'failed') {
|
||||
console.log(`${prefix} ${RED_COLOR}[FAIL]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
console.log(' Message:');
|
||||
console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`);
|
||||
console.log(' Stack:');
|
||||
if (test.error.stack)
|
||||
console.log(this._beautifyStack(test.error.stack));
|
||||
if (test.output) {
|
||||
console.log(' Output:');
|
||||
console.log(test.output.split('\n').map(line => ' ' + line).join('\n'));
|
||||
}
|
||||
} else if (test.result === 'timedout') {
|
||||
console.log(`${prefix} ${RED_COLOR}[TIME]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`);
|
||||
console.log(' Message:');
|
||||
console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`);
|
||||
}
|
||||
this._printVerboseTestResult(this._testCounter, test, workerId);
|
||||
} else {
|
||||
if (test.result === 'ok')
|
||||
process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`);
|
||||
process.stdout.write(colors.green('.'));
|
||||
else if (test.result === 'skipped')
|
||||
process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`);
|
||||
process.stdout.write(colors.yellow('*'));
|
||||
else if (test.result === 'failed')
|
||||
process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`);
|
||||
process.stdout.write(colors.red('F'));
|
||||
else if (test.result === 'crashed')
|
||||
process.stdout.write(`${RED_COLOR}C${RESET_COLOR}`);
|
||||
process.stdout.write(colors.red('C'));
|
||||
else if (test.result === 'terminated')
|
||||
process.stdout.write(`${MAGENTA_COLOR}.${RESET_COLOR}`);
|
||||
process.stdout.write(colors.magenta('.'));
|
||||
else if (test.result === 'timedout')
|
||||
process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`);
|
||||
process.stdout.write(colors.red('T'));
|
||||
}
|
||||
}
|
||||
|
||||
_printVerboseTestResult(resultIndex, test, workerId = undefined) {
|
||||
let prefix = `${resultIndex})`;
|
||||
if (this._runner.parallel() > 1 && workerId !== undefined)
|
||||
prefix += ' ' + colors.gray(`[worker = ${workerId}]`);
|
||||
if (test.result === 'ok') {
|
||||
console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
} else if (test.result === 'terminated') {
|
||||
console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
} else if (test.result === 'crashed') {
|
||||
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
} else if (test.result === 'skipped') {
|
||||
console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
} else if (test.result === 'timedout') {
|
||||
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
} else if (test.result === 'failed') {
|
||||
console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName} (${formatLocation(test.location)})`);
|
||||
if (test.error instanceof MatchError) {
|
||||
let lines = this._filePathToLines.get(test.error.location.filePath);
|
||||
if (!lines) {
|
||||
try {
|
||||
lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n');
|
||||
} catch (e) {
|
||||
lines = [];
|
||||
}
|
||||
this._filePathToLines.set(test.error.location.filePath, lines);
|
||||
}
|
||||
const lineNumber = test.error.location.lineNumber;
|
||||
if (lineNumber < lines.length) {
|
||||
const lineNumberLength = (lineNumber + 1 + '').length;
|
||||
const FROM = Math.max(test.location.lineNumber - 1, lineNumber - 5);
|
||||
const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n');
|
||||
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^';
|
||||
console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n');
|
||||
}
|
||||
console.log(padLines(test.error.formatter(), 4));
|
||||
console.log('');
|
||||
} else {
|
||||
console.log(' Message:');
|
||||
console.log(` ${colors.red(test.error.message || test.error)}`);
|
||||
if (test.error.stack) {
|
||||
console.log(' Stack:');
|
||||
let stack = test.error.stack;
|
||||
// Highlight first test location, if any.
|
||||
const match = stack.match(new RegExp(test.location.filePath + ':(\\d+):(\\d+)'));
|
||||
if (match) {
|
||||
const [, line, column] = match;
|
||||
const fileName = `${test.location.fileName}:${line}:${column}`;
|
||||
stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName));
|
||||
}
|
||||
console.log(padLines(stack, 4));
|
||||
}
|
||||
}
|
||||
if (test.output) {
|
||||
console.log(' Output:');
|
||||
console.log(padLines(test.output, 4));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function formatTestLocation(test) {
|
||||
const location = test.location;
|
||||
function formatLocation(location) {
|
||||
if (!location)
|
||||
return '';
|
||||
return `${YELLOW_COLOR}${location.fileName}:${location.lineNumber}:${location.columnNumber}${RESET_COLOR}`;
|
||||
return colors.yellow(`${location.fileName}:${location.lineNumber}:${location.columnNumber}`);
|
||||
}
|
||||
|
||||
function padLines(text, spaces = 0) {
|
||||
const indent = ' '.repeat(spaces);
|
||||
return text.split('\n').map(line => indent + line).join('\n');
|
||||
}
|
||||
|
||||
module.exports = Reporter;
|
||||
|
||||
@ -23,6 +23,7 @@ const Multimap = require('./Multimap');
|
||||
const fs = require('fs');
|
||||
const {SourceMapSupport} = require('./SourceMapSupport');
|
||||
const debug = require('debug');
|
||||
const {getCallerLocation} = require('./utils');
|
||||
|
||||
const INFINITE_TIMEOUT = 2147483647;
|
||||
|
||||
@ -41,7 +42,7 @@ class UserCallback {
|
||||
});
|
||||
|
||||
this.timeout = timeout;
|
||||
this.location = this._getLocation();
|
||||
this.location = getCallerLocation(__filename);
|
||||
}
|
||||
|
||||
async run(...args) {
|
||||
@ -62,35 +63,6 @@ class UserCallback {
|
||||
}
|
||||
}
|
||||
|
||||
_getLocation() {
|
||||
const error = new Error();
|
||||
const stackFrames = error.stack.split('\n').slice(1);
|
||||
// Find first stackframe that doesn't point to this file.
|
||||
for (let frame of stackFrames) {
|
||||
frame = frame.trim();
|
||||
if (!frame.startsWith('at '))
|
||||
return null;
|
||||
if (frame.endsWith(')')) {
|
||||
const from = frame.indexOf('(');
|
||||
frame = frame.substring(from + 1, frame.length - 1);
|
||||
} else {
|
||||
frame = frame.substring('at '.length);
|
||||
}
|
||||
|
||||
const match = frame.match(/^(.*):(\d+):(\d+)$/);
|
||||
if (!match)
|
||||
return null;
|
||||
const filePath = match[1];
|
||||
const lineNumber = parseInt(match[2], 10);
|
||||
const columnNumber = parseInt(match[3], 10);
|
||||
if (filePath === __filename)
|
||||
continue;
|
||||
const fileName = filePath.split(path.sep).pop();
|
||||
return { fileName, filePath, lineNumber, columnNumber };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
terminate() {
|
||||
this._terminateCallback(TerminatedError);
|
||||
}
|
||||
@ -119,6 +91,7 @@ class Test {
|
||||
this.declaredMode = declaredMode;
|
||||
this._userCallback = new UserCallback(callback, timeout);
|
||||
this.location = this._userCallback.location;
|
||||
this.timeout = timeout;
|
||||
|
||||
// Test results
|
||||
this.result = null;
|
||||
@ -422,7 +395,10 @@ class TestRunner extends EventEmitter {
|
||||
const runnableTests = this._runnableTests();
|
||||
this.emit(TestRunner.Events.Started, runnableTests);
|
||||
this._runningPass = new TestPass(this, this._rootSuite, runnableTests, this._parallel, this._breakOnFailure);
|
||||
const termination = await this._runningPass.run();
|
||||
const termination = await this._runningPass.run().catch(e => {
|
||||
console.error(e);
|
||||
throw e;
|
||||
});
|
||||
this._runningPass = null;
|
||||
const result = {};
|
||||
if (termination) {
|
||||
|
||||
@ -16,6 +16,6 @@
|
||||
|
||||
const TestRunner = require('./TestRunner');
|
||||
const Reporter = require('./Reporter');
|
||||
const Matchers = require('./Matchers');
|
||||
const {Matchers} = require('./Matchers');
|
||||
|
||||
module.exports = { TestRunner, Reporter, Matchers };
|
||||
|
||||
@ -8,7 +8,6 @@ require('./testrunner.spec.js').addTests({testRunner, expect});
|
||||
new Reporter(testRunner, {
|
||||
verbose: process.argv.includes('--verbose'),
|
||||
summary: true,
|
||||
projectFolder: require('path').join(__dirname, '..'),
|
||||
showSlowTests: 0,
|
||||
});
|
||||
testRunner.run();
|
||||
|
||||
32
utils/testrunner/utils.js
Normal file
32
utils/testrunner/utils.js
Normal file
@ -0,0 +1,32 @@
|
||||
const path = require('path');
|
||||
|
||||
module.exports = {
|
||||
getCallerLocation: function(filename) {
|
||||
const error = new Error();
|
||||
const stackFrames = error.stack.split('\n').slice(1);
|
||||
// Find first stackframe that doesn't point to this file.
|
||||
for (let frame of stackFrames) {
|
||||
frame = frame.trim();
|
||||
if (!frame.startsWith('at '))
|
||||
return null;
|
||||
if (frame.endsWith(')')) {
|
||||
const from = frame.indexOf('(');
|
||||
frame = frame.substring(from + 1, frame.length - 1);
|
||||
} else {
|
||||
frame = frame.substring('at '.length);
|
||||
}
|
||||
|
||||
const match = frame.match(/^(.*):(\d+):(\d+)$/);
|
||||
if (!match)
|
||||
return null;
|
||||
const filePath = match[1];
|
||||
const lineNumber = parseInt(match[2], 10);
|
||||
const columnNumber = parseInt(match[3], 10);
|
||||
if (filePath === __filename || filePath === filename)
|
||||
continue;
|
||||
const fileName = filePath.split(path.sep).pop();
|
||||
return { fileName, filePath, lineNumber, columnNumber };
|
||||
}
|
||||
return null;
|
||||
},
|
||||
};
|
||||
Loading…
Reference in New Issue
Block a user