mirror of
https://github.com/microsoft/playwright.git
synced 2024-12-14 21:53:35 +03:00
chore(testrunner): introduce TestRun (#1582)
This commit is contained in:
Dmitry Gozman
GitHub
parent
5499b1844d
commit
a41836b1f1
10
test/test.js
10
test/test.js
@ -43,17 +43,17 @@ const testRunner = new TestRunner({
|
||||
breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1,
|
||||
installCommonHelpers: false
|
||||
});
|
||||
testRunner.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
|
||||
testRunner.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip));
|
||||
testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true));
|
||||
testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
|
||||
testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
|
||||
testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
|
||||
testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
|
||||
testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
|
||||
testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
|
||||
testRunner.testAttribute('focus', t => t.setMode(t.Modes.Focus));
|
||||
testRunner.suiteAttribute('focus', s => s.setMode(s.Modes.Focus));
|
||||
testRunner.testAttribute('focus', t => t.setFocused(true));
|
||||
testRunner.suiteAttribute('focus', s => s.setFocused(true));
|
||||
testRunner.testAttribute('debug', t => {
|
||||
t.setMode(t.Modes.Focus);
|
||||
t.setFocused(true);
|
||||
t.setTimeout(100000000);
|
||||
|
||||
let session;
|
||||
|
||||
@ -39,34 +39,23 @@ class Reporter {
|
||||
runner.on('testfinished', this._onTestFinished.bind(this));
|
||||
}
|
||||
|
||||
_onStarted(runnableTests) {
|
||||
_onStarted(testRuns) {
|
||||
this._testCounter = 0;
|
||||
this._timestamp = Date.now();
|
||||
const allTests = this._runner.tests();
|
||||
if (allTests.length === runnableTests.length) {
|
||||
console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
|
||||
if (!this._runner.hasFocusedTestsOrSuites()) {
|
||||
console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
|
||||
} else {
|
||||
console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
|
||||
console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
|
||||
console.log('');
|
||||
const focusedSuites = this._runner.focusedSuites().map(suite => ({
|
||||
id: suite.location().filePath + ':' + suite.location().lineNumber + ':' + suite.location().columnNumber,
|
||||
fullName: suite.fullName(),
|
||||
location: suite.location(),
|
||||
}));
|
||||
const focusedTests = this._runner.focusedTests().map(test => ({
|
||||
id: test.location().filePath + ':' + test.location().lineNumber + ':' + test.location().columnNumber,
|
||||
fullName: test.fullName(),
|
||||
location: test.location(),
|
||||
}));
|
||||
const focusedEntities = new Map([
|
||||
...focusedSuites.map(suite => ([suite.id, suite])),
|
||||
...focusedTests.map(test => ([test.id, test])),
|
||||
]);
|
||||
if (focusedEntities.size) {
|
||||
const focusedEntities = [
|
||||
...this._runner.suites().filter(suite => suite.focused()),
|
||||
...this._runner.tests().filter(test => test.focused()),
|
||||
];
|
||||
if (focusedEntities.length) {
|
||||
console.log('Focused Suites and Tests:');
|
||||
const entities = [...focusedEntities.values()];
|
||||
for (let i = 0; i < entities.length; ++i)
|
||||
console.log(` ${i + 1}) ${entities[i].fullName} (${formatLocation(entities[i].location)})`);
|
||||
for (let i = 0; i < focusedEntities.length; ++i)
|
||||
console.log(` ${i + 1}) ${focusedEntities[i].fullName()} (${formatLocation(focusedEntities[i].location())})`);
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
@ -80,168 +69,164 @@ class Reporter {
|
||||
}
|
||||
|
||||
for (let i = 0; i < result.errors.length; i++) {
|
||||
const { message, error, workerId, tests } = result.errors[i];
|
||||
const { message, error, runs } = result.errors[i];
|
||||
console.log(`\n${colors.magenta('NON-TEST ERROR #' + i)}: ${message}`);
|
||||
if (error && error.stack)
|
||||
console.log(padLines(error.stack, 2));
|
||||
const lastTests = tests.slice(tests.length - Math.min(10, tests.length));
|
||||
if (lastTests.length)
|
||||
const lastRuns = runs.slice(runs.length - Math.min(10, runs.length));
|
||||
if (lastRuns.length)
|
||||
console.log(`WORKER STATE`);
|
||||
for (let j = 0; j < lastTests.length; j++)
|
||||
this._printVerboseTestResult(j, lastTests[j], workerId);
|
||||
for (let j = 0; j < lastRuns.length; j++)
|
||||
this._printVerboseTestRunResult(j, lastRuns[j]);
|
||||
}
|
||||
console.log('');
|
||||
console.log('');
|
||||
}
|
||||
|
||||
_onFinished(result) {
|
||||
this._printTestResults();
|
||||
this._printTestResults(result);
|
||||
if (!result.ok())
|
||||
this._printFailedResult(result);
|
||||
process.exitCode = result.exitCode;
|
||||
}
|
||||
|
||||
_printTestResults() {
|
||||
_printTestResults(result) {
|
||||
// 2 newlines after completing all tests.
|
||||
console.log('\n');
|
||||
|
||||
const failedTests = this._runner.failedTests();
|
||||
if (this._summary && failedTests.length > 0) {
|
||||
const runs = result.runs;
|
||||
const failedRuns = runs.filter(run => run.isFailure());
|
||||
const executedRuns = runs.filter(run => run.result());
|
||||
const okRuns = runs.filter(run => run.ok());
|
||||
const skippedRuns = runs.filter(run => run.result() === 'skipped');
|
||||
const markedAsFailingRuns = runs.filter(run => run.result() === 'markedAsFailing');
|
||||
|
||||
if (this._summary && failedRuns.length > 0) {
|
||||
console.log('\nFailures:');
|
||||
for (let i = 0; i < failedTests.length; ++i) {
|
||||
const test = failedTests[i];
|
||||
this._printVerboseTestResult(i + 1, test);
|
||||
for (let i = 0; i < failedRuns.length; ++i) {
|
||||
this._printVerboseTestRunResult(i + 1, failedRuns[i]);
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
|
||||
const skippedTests = this._runner.skippedTests();
|
||||
const markedAsFailingTests = this._runner.markedAsFailingTests();
|
||||
if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) {
|
||||
if (markedAsFailingTests.length > 0) {
|
||||
if (this._showMarkedAsFailingTests && this._summary && markedAsFailingRuns.length) {
|
||||
if (markedAsFailingRuns.length > 0) {
|
||||
console.log('\nMarked as failing:');
|
||||
markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => {
|
||||
console.log(`${index + 1}) ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
markedAsFailingRuns.slice(0, this._showMarkedAsFailingTests).forEach((testRun, index) => {
|
||||
console.log(`${index + 1}) ${testRun.test().fullName()} (${formatLocation(testRun.test().location())})`);
|
||||
});
|
||||
}
|
||||
if (this._showMarkedAsFailingTests < markedAsFailingTests.length) {
|
||||
if (this._showMarkedAsFailingTests < markedAsFailingRuns.length) {
|
||||
console.log('');
|
||||
console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
|
||||
console.log(`... and ${colors.yellow(markedAsFailingRuns.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
|
||||
}
|
||||
}
|
||||
|
||||
if (this._showSlowTests) {
|
||||
const slowTests = this._runner.passedTests().sort((a, b) => {
|
||||
const aDuration = a.endTimestamp - a.startTimestamp;
|
||||
const bDuration = b.endTimestamp - b.startTimestamp;
|
||||
return bDuration - aDuration;
|
||||
}).slice(0, this._showSlowTests);
|
||||
const slowRuns = okRuns.sort((a, b) => b.duration() - a.duration()).slice(0, this._showSlowTests);
|
||||
console.log(`\nSlowest tests:`);
|
||||
for (let i = 0; i < slowTests.length; ++i) {
|
||||
const test = slowTests[i];
|
||||
const duration = test.endTimestamp - test.startTimestamp;
|
||||
console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
for (let i = 0; i < slowRuns.length; ++i) {
|
||||
const run = slowRuns[i];
|
||||
console.log(` (${i + 1}) ${colors.yellow((run.duration() / 1000) + 's')} - ${run.test().fullName()} (${formatLocation(run.test().location())})`);
|
||||
}
|
||||
}
|
||||
|
||||
const tests = this._runner.tests();
|
||||
const executedTests = tests.filter(test => test.result);
|
||||
const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length;
|
||||
let summaryText = '';
|
||||
if (failedTests.length || markedAsFailingTests.length) {
|
||||
const summary = [`ok - ${colors.green(okTestsLength)}`];
|
||||
if (failedTests.length)
|
||||
summary.push(`failed - ${colors.red(failedTests.length)}`);
|
||||
if (markedAsFailingTests.length)
|
||||
summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`);
|
||||
if (skippedTests.length)
|
||||
summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
|
||||
if (failedRuns.length || markedAsFailingRuns.length) {
|
||||
const summary = [`ok - ${colors.green(okRuns.length)}`];
|
||||
if (failedRuns.length)
|
||||
summary.push(`failed - ${colors.red(failedRuns.length)}`);
|
||||
if (markedAsFailingRuns.length)
|
||||
summary.push(`marked as failing - ${colors.yellow(markedAsFailingRuns.length)}`);
|
||||
if (skippedRuns.length)
|
||||
summary.push(`skipped - ${colors.yellow(skippedRuns.length)}`);
|
||||
summaryText = ` (${summary.join(', ')})`;
|
||||
}
|
||||
|
||||
console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`);
|
||||
console.log(`\nRan ${executedRuns.length}${summaryText} of ${runs.length} test${runs.length > 1 ? 's' : ''}`);
|
||||
const milliseconds = Date.now() - this._timestamp;
|
||||
const seconds = milliseconds / 1000;
|
||||
console.log(`Finished in ${colors.yellow(seconds)} seconds`);
|
||||
}
|
||||
|
||||
_onTestStarted(test, workerId) {
|
||||
_onTestStarted(testRun) {
|
||||
}
|
||||
|
||||
_onTestFinished(test, workerId) {
|
||||
_onTestFinished(testRun) {
|
||||
if (this._verbose) {
|
||||
++this._testCounter;
|
||||
this._printVerboseTestResult(this._testCounter, test, workerId);
|
||||
this._printVerboseTestRunResult(this._testCounter, testRun);
|
||||
} else {
|
||||
if (test.result === 'ok')
|
||||
if (testRun.result() === 'ok')
|
||||
process.stdout.write(colors.green('\u00B7'));
|
||||
else if (test.result === 'skipped')
|
||||
else if (testRun.result() === 'skipped')
|
||||
process.stdout.write(colors.yellow('\u00B7'));
|
||||
else if (test.result === 'markedAsFailing')
|
||||
else if (testRun.result() === 'markedAsFailing')
|
||||
process.stdout.write(colors.yellow('\u00D7'));
|
||||
else if (test.result === 'failed')
|
||||
else if (testRun.result() === 'failed')
|
||||
process.stdout.write(colors.red('F'));
|
||||
else if (test.result === 'crashed')
|
||||
else if (testRun.result() === 'crashed')
|
||||
process.stdout.write(colors.red('C'));
|
||||
else if (test.result === 'terminated')
|
||||
else if (testRun.result() === 'terminated')
|
||||
process.stdout.write(colors.magenta('.'));
|
||||
else if (test.result === 'timedout')
|
||||
else if (testRun.result() === 'timedout')
|
||||
process.stdout.write(colors.red('T'));
|
||||
}
|
||||
}
|
||||
|
||||
_printVerboseTestResult(resultIndex, test, workerId = undefined) {
|
||||
_printVerboseTestRunResult(resultIndex, testRun) {
|
||||
const test = testRun.test();
|
||||
let prefix = `${resultIndex})`;
|
||||
if (this._runner.parallel() > 1 && workerId !== undefined)
|
||||
prefix += ' ' + colors.gray(`[worker = ${workerId}]`);
|
||||
if (test.result === 'ok') {
|
||||
if (this._runner.parallel() > 1)
|
||||
prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`);
|
||||
if (testRun.result() === 'ok') {
|
||||
console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
} else if (test.result === 'terminated') {
|
||||
} else if (testRun.result() === 'terminated') {
|
||||
console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
} else if (test.result === 'crashed') {
|
||||
} else if (testRun.result() === 'crashed') {
|
||||
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
} else if (test.result === 'skipped') {
|
||||
} else if (test.result === 'markedAsFailing') {
|
||||
} else if (testRun.result() === 'skipped') {
|
||||
} else if (testRun.result() === 'markedAsFailing') {
|
||||
console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
} else if (test.result === 'timedout') {
|
||||
} else if (testRun.result() === 'timedout') {
|
||||
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
if (test.output) {
|
||||
if (testRun.output) {
|
||||
console.log(' Output:');
|
||||
for (const line of test.output)
|
||||
for (const line of testRun.output)
|
||||
console.log(' ' + line);
|
||||
}
|
||||
} else if (test.result === 'failed') {
|
||||
} else if (testRun.result() === 'failed') {
|
||||
console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`);
|
||||
if (test.error instanceof MatchError) {
|
||||
let lines = this._filePathToLines.get(test.error.location.filePath);
|
||||
if (testRun.error() instanceof MatchError) {
|
||||
let lines = this._filePathToLines.get(testRun.error().location.filePath);
|
||||
if (!lines) {
|
||||
try {
|
||||
lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n');
|
||||
lines = fs.readFileSync(testRun.error().location.filePath, 'utf8').split('\n');
|
||||
} catch (e) {
|
||||
lines = [];
|
||||
}
|
||||
this._filePathToLines.set(test.error.location.filePath, lines);
|
||||
this._filePathToLines.set(testRun.error().location.filePath, lines);
|
||||
}
|
||||
const lineNumber = test.error.location.lineNumber;
|
||||
const lineNumber = testRun.error().location.lineNumber;
|
||||
if (lineNumber < lines.length) {
|
||||
const lineNumberLength = (lineNumber + 1 + '').length;
|
||||
const FROM = Math.max(test.location().lineNumber - 1, lineNumber - 5);
|
||||
const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n');
|
||||
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^';
|
||||
const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(testRun.error().location.columnNumber - 1) + '^';
|
||||
console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n');
|
||||
}
|
||||
console.log(padLines(test.error.formatter(), 4));
|
||||
console.log(padLines(testRun.error().formatter(), 4));
|
||||
console.log('');
|
||||
} else {
|
||||
console.log(' Message:');
|
||||
let message = '' + (test.error.message || test.error);
|
||||
if (test.error.stack && message.includes(test.error.stack))
|
||||
message = message.substring(0, message.indexOf(test.error.stack));
|
||||
let message = '' + (testRun.error().message || testRun.error());
|
||||
if (testRun.error().stack && message.includes(testRun.error().stack))
|
||||
message = message.substring(0, message.indexOf(testRun.error().stack));
|
||||
if (message)
|
||||
console.log(` ${colors.red(message)}`);
|
||||
if (test.error.stack) {
|
||||
if (testRun.error().stack) {
|
||||
console.log(' Stack:');
|
||||
let stack = test.error.stack;
|
||||
let stack = testRun.error().stack;
|
||||
// Highlight first test location, if any.
|
||||
const match = stack.match(new RegExp(test.location().filePath + ':(\\d+):(\\d+)'));
|
||||
if (match) {
|
||||
@ -252,9 +237,9 @@ class Reporter {
|
||||
console.log(padLines(stack, 4));
|
||||
}
|
||||
}
|
||||
if (test.output) {
|
||||
if (testRun.output) {
|
||||
console.log(' Output:');
|
||||
for (const line of test.output)
|
||||
for (const line of testRun.output)
|
||||
console.log(' ' + line);
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,12 +37,6 @@ function runUserCallback(callback, timeout, args) {
|
||||
return { promise, terminate };
|
||||
}
|
||||
|
||||
const TestMode = {
|
||||
Run: 'run',
|
||||
Skip: 'skip',
|
||||
Focus: 'focus',
|
||||
};
|
||||
|
||||
const TestExpectation = {
|
||||
Ok: 'ok',
|
||||
Fail: 'fail',
|
||||
@ -72,7 +66,8 @@ class Test {
|
||||
this._suite = suite;
|
||||
this._name = name;
|
||||
this._fullName = (suite.fullName() + ' ' + name).trim();
|
||||
this._mode = TestMode.Run;
|
||||
this._skipped = false;
|
||||
this._focused = false;
|
||||
this._expectation = TestExpectation.Ok;
|
||||
this._body = callback;
|
||||
this._location = location;
|
||||
@ -80,26 +75,9 @@ class Test {
|
||||
this._repeat = 1;
|
||||
this._hooks = [];
|
||||
|
||||
// Test results. TODO: make these private.
|
||||
this.result = null;
|
||||
this.error = null;
|
||||
this.startTimestamp = 0;
|
||||
this.endTimestamp = 0;
|
||||
|
||||
this.Modes = { ...TestMode };
|
||||
this.Expectations = { ...TestExpectation };
|
||||
}
|
||||
|
||||
_clone() {
|
||||
// TODO: introduce TestRun instead?
|
||||
const test = new Test(this._suite, this._name, this._body, this._location);
|
||||
test._timeout = this._timeout;
|
||||
test._mode = this._mode;
|
||||
test._expectation = this._expectation;
|
||||
test._hooks = this._hooks.slice();
|
||||
return test;
|
||||
}
|
||||
|
||||
suite() {
|
||||
return this._suite;
|
||||
}
|
||||
@ -120,13 +98,20 @@ class Test {
|
||||
return this._body;
|
||||
}
|
||||
|
||||
mode() {
|
||||
return this._mode;
|
||||
skipped() {
|
||||
return this._skipped;
|
||||
}
|
||||
|
||||
setMode(mode) {
|
||||
if (this._mode !== TestMode.Focus)
|
||||
this._mode = mode;
|
||||
setSkipped(skipped) {
|
||||
this._skipped = skipped;
|
||||
}
|
||||
|
||||
focused() {
|
||||
return this._focused;
|
||||
}
|
||||
|
||||
setFocused(focused) {
|
||||
this._focused = focused;
|
||||
}
|
||||
|
||||
timeout() {
|
||||
@ -171,24 +156,16 @@ class Suite {
|
||||
this._parentSuite = parentSuite;
|
||||
this._name = name;
|
||||
this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim();
|
||||
this._mode = TestMode.Run;
|
||||
this._skipped = false;
|
||||
this._focused = false;
|
||||
this._expectation = TestExpectation.Ok;
|
||||
this._location = location;
|
||||
this._repeat = 1;
|
||||
this._hooks = [];
|
||||
|
||||
this.Modes = { ...TestMode };
|
||||
this.Expectations = { ...TestExpectation };
|
||||
}
|
||||
|
||||
_clone() {
|
||||
// TODO: introduce TestRun instead?
|
||||
const suite = new Suite(this._parentSuite, this._name, this._location);
|
||||
suite._mode = this._mode;
|
||||
suite._expectation = this._expectation;
|
||||
return suite;
|
||||
}
|
||||
|
||||
parentSuite() {
|
||||
return this._parentSuite;
|
||||
}
|
||||
@ -201,13 +178,20 @@ class Suite {
|
||||
return this._fullName;
|
||||
}
|
||||
|
||||
mode() {
|
||||
return this._mode;
|
||||
skipped() {
|
||||
return this._skipped;
|
||||
}
|
||||
|
||||
setMode(mode) {
|
||||
if (this._mode !== TestMode.Focus)
|
||||
this._mode = mode;
|
||||
setSkipped(skipped) {
|
||||
this._skipped = skipped;
|
||||
}
|
||||
|
||||
focused() {
|
||||
return this._focused;
|
||||
}
|
||||
|
||||
setFocused(focused) {
|
||||
this._focused = focused;
|
||||
}
|
||||
|
||||
location() {
|
||||
@ -251,12 +235,56 @@ class Suite {
|
||||
}
|
||||
}
|
||||
|
||||
class TestRun {
|
||||
constructor(test) {
|
||||
this._test = test;
|
||||
this._result = null;
|
||||
this._error = null;
|
||||
this._startTimestamp = 0;
|
||||
this._endTimestamp = 0;
|
||||
this._workerId = null;
|
||||
}
|
||||
|
||||
finished() {
|
||||
return this._result !== null && this._result !== 'running';
|
||||
}
|
||||
|
||||
isFailure() {
|
||||
return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed;
|
||||
}
|
||||
|
||||
ok() {
|
||||
return this._result === TestResult.Ok;
|
||||
}
|
||||
|
||||
result() {
|
||||
return this._result;
|
||||
}
|
||||
|
||||
error() {
|
||||
return this._error;
|
||||
}
|
||||
|
||||
duration() {
|
||||
return this._endTimestamp - this._startTimestamp;
|
||||
}
|
||||
|
||||
test() {
|
||||
return this._test;
|
||||
}
|
||||
|
||||
workerId() {
|
||||
return this._workerId;
|
||||
}
|
||||
}
|
||||
|
||||
class Result {
|
||||
constructor() {
|
||||
this.result = TestResult.Ok;
|
||||
this.exitCode = 0;
|
||||
this.message = '';
|
||||
this.errors = [];
|
||||
this.runs = [];
|
||||
}
|
||||
|
||||
setResult(result, message) {
|
||||
@ -275,11 +303,9 @@ class Result {
|
||||
}
|
||||
|
||||
addError(message, error, worker) {
|
||||
const data = { message, error, tests: [] };
|
||||
if (worker) {
|
||||
data.workerId = worker._workerId;
|
||||
data.tests = worker._runTests.slice();
|
||||
}
|
||||
const data = { message, error, runs: [] };
|
||||
if (worker)
|
||||
data.runs = worker._runs.slice();
|
||||
this.errors.push(data);
|
||||
}
|
||||
|
||||
@ -297,7 +323,7 @@ class TestWorker {
|
||||
this._workerId = workerId;
|
||||
this._runningTestTerminate = null;
|
||||
this._runningHookTerminate = null;
|
||||
this._runTests = [];
|
||||
this._runs = [];
|
||||
}
|
||||
|
||||
terminate(terminateHooks) {
|
||||
@ -308,36 +334,34 @@ class TestWorker {
|
||||
this._runningHookTerminate();
|
||||
}
|
||||
|
||||
_markTerminated(test) {
|
||||
_markTerminated(testRun) {
|
||||
if (!this._terminating)
|
||||
return false;
|
||||
test.result = TestResult.Terminated;
|
||||
testRun._result = TestResult.Terminated;
|
||||
return true;
|
||||
}
|
||||
|
||||
async runTest(test) {
|
||||
this._runTests.push(test);
|
||||
async run(testRun) {
|
||||
this._runs.push(testRun);
|
||||
|
||||
if (this._markTerminated(test))
|
||||
return;
|
||||
|
||||
let skipped = test.mode() === TestMode.Skip;
|
||||
const test = testRun.test();
|
||||
let skipped = test.skipped() && !test.focused();
|
||||
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
||||
skipped = skipped || (suite.mode() === TestMode.Skip);
|
||||
skipped = skipped || (suite.skipped() && !suite.focused());
|
||||
if (skipped) {
|
||||
await this._testPass._willStartTest(this, test);
|
||||
test.result = TestResult.Skipped;
|
||||
await this._testPass._didFinishTest(this, test);
|
||||
await this._willStartTestRun(testRun);
|
||||
testRun._result = TestResult.Skipped;
|
||||
await this._didFinishTestRun(testRun);
|
||||
return;
|
||||
}
|
||||
|
||||
let expectedToFail = test.expectation() === TestExpectation.Fail;
|
||||
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
||||
expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail);
|
||||
if (expectedToFail && test.mode() !== TestMode.Focus) {
|
||||
await this._testPass._willStartTest(this, test);
|
||||
test.result = TestResult.MarkedAsFailing;
|
||||
await this._testPass._didFinishTest(this, test);
|
||||
if (expectedToFail && !test.focused()) {
|
||||
await this._willStartTestRun(testRun);
|
||||
testRun._result = TestResult.MarkedAsFailing;
|
||||
await this._didFinishTestRun(testRun);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -351,80 +375,80 @@ class TestWorker {
|
||||
common++;
|
||||
|
||||
while (this._suiteStack.length > common) {
|
||||
if (this._markTerminated(test))
|
||||
if (this._markTerminated(testRun))
|
||||
return;
|
||||
const suite = this._suiteStack.pop();
|
||||
for (const hook of suite.hooks('afterAll')) {
|
||||
if (!await this._runHook(test, hook, suite.fullName()))
|
||||
if (!await this._runHook(testRun, hook, suite.fullName()))
|
||||
return;
|
||||
}
|
||||
}
|
||||
while (this._suiteStack.length < suiteStack.length) {
|
||||
if (this._markTerminated(test))
|
||||
if (this._markTerminated(testRun))
|
||||
return;
|
||||
const suite = suiteStack[this._suiteStack.length];
|
||||
this._suiteStack.push(suite);
|
||||
for (const hook of suite.hooks('beforeAll')) {
|
||||
if (!await this._runHook(test, hook, suite.fullName()))
|
||||
if (!await this._runHook(testRun, hook, suite.fullName()))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (this._markTerminated(test))
|
||||
if (this._markTerminated(testRun))
|
||||
return;
|
||||
|
||||
// From this point till the end, we have to run all hooks
|
||||
// no matter what happens.
|
||||
|
||||
await this._testPass._willStartTest(this, test);
|
||||
await this._willStartTestRun(testRun);
|
||||
for (const suite of this._suiteStack) {
|
||||
for (const hook of suite.hooks('beforeEach'))
|
||||
await this._runHook(test, hook, suite.fullName(), true);
|
||||
await this._runHook(testRun, hook, suite.fullName(), true);
|
||||
}
|
||||
for (const hook of test.hooks('before'))
|
||||
await this._runHook(test, hook, test.fullName(), true);
|
||||
await this._runHook(testRun, hook, test.fullName(), true);
|
||||
|
||||
if (!test.error && !this._markTerminated(test)) {
|
||||
await this._testPass._willStartTestBody(this, test);
|
||||
if (!testRun._error && !this._markTerminated(testRun)) {
|
||||
await this._willStartTestBody(testRun);
|
||||
const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]);
|
||||
this._runningTestTerminate = terminate;
|
||||
test.error = await promise;
|
||||
testRun._error = await promise;
|
||||
this._runningTestTerminate = null;
|
||||
if (test.error && test.error.stack)
|
||||
await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(test.error);
|
||||
if (!test.error)
|
||||
test.result = TestResult.Ok;
|
||||
else if (test.error === TimeoutError)
|
||||
test.result = TestResult.TimedOut;
|
||||
else if (test.error === TerminatedError)
|
||||
test.result = TestResult.Terminated;
|
||||
if (testRun._error && testRun._error.stack)
|
||||
await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error);
|
||||
if (!testRun._error)
|
||||
testRun._result = TestResult.Ok;
|
||||
else if (testRun._error === TimeoutError)
|
||||
testRun._result = TestResult.TimedOut;
|
||||
else if (testRun._error === TerminatedError)
|
||||
testRun._result = TestResult.Terminated;
|
||||
else
|
||||
test.result = TestResult.Failed;
|
||||
await this._testPass._didFinishTestBody(this, test);
|
||||
testRun._result = TestResult.Failed;
|
||||
await this._didFinishTestBody(testRun);
|
||||
}
|
||||
|
||||
for (const hook of test.hooks('after'))
|
||||
await this._runHook(test, hook, test.fullName(), true);
|
||||
await this._runHook(testRun, hook, test.fullName(), true);
|
||||
for (const suite of this._suiteStack.slice().reverse()) {
|
||||
for (const hook of suite.hooks('afterEach'))
|
||||
await this._runHook(test, hook, suite.fullName(), true);
|
||||
await this._runHook(testRun, hook, suite.fullName(), true);
|
||||
}
|
||||
await this._testPass._didFinishTest(this, test);
|
||||
await this._didFinishTestRun(testRun);
|
||||
}
|
||||
|
||||
async _runHook(test, hook, fullName, passTest = false) {
|
||||
await this._testPass._willStartHook(this, hook, fullName);
|
||||
async _runHook(testRun, hook, fullName, passTest = false) {
|
||||
await this._willStartHook(hook, fullName);
|
||||
const timeout = this._testPass._runner._timeout;
|
||||
const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, test] : [this._state]);
|
||||
const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, testRun.test()] : [this._state]);
|
||||
this._runningHookTerminate = terminate;
|
||||
let error = await promise;
|
||||
this._runningHookTerminate = null;
|
||||
|
||||
if (error) {
|
||||
const locationString = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`;
|
||||
if (test.result !== TestResult.Terminated) {
|
||||
if (testRun && testRun._result !== TestResult.Terminated) {
|
||||
// Prefer terminated result over any hook failures.
|
||||
test.result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed;
|
||||
testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed;
|
||||
}
|
||||
let message;
|
||||
if (error === TimeoutError) {
|
||||
@ -439,20 +463,56 @@ class TestWorker {
|
||||
await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
|
||||
message = `${locationString} - FAILED while running "${hook.name}" in suite "${fullName}": `;
|
||||
}
|
||||
await this._testPass._didFailHook(this, hook, fullName, message, error);
|
||||
test.error = error;
|
||||
await this._didFailHook(hook, fullName, message, error);
|
||||
if (testRun)
|
||||
testRun._error = error;
|
||||
return false;
|
||||
}
|
||||
|
||||
await this._testPass._didCompleteHook(this, hook, fullName);
|
||||
await this._didCompleteHook(hook, fullName);
|
||||
return true;
|
||||
}
|
||||
|
||||
async _willStartTestRun(testRun) {
|
||||
testRun._startTimestamp = Date.now();
|
||||
testRun._workerId = this._workerId;
|
||||
this._testPass._runner.emit(TestRunner.Events.TestStarted, testRun);
|
||||
}
|
||||
|
||||
async _didFinishTestRun(testRun) {
|
||||
testRun._endTimestamp = Date.now();
|
||||
testRun._workerId = this._workerId;
|
||||
this._testPass._runner.emit(TestRunner.Events.TestFinished, testRun);
|
||||
}
|
||||
|
||||
async _willStartTestBody(testRun) {
|
||||
debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`);
|
||||
}
|
||||
|
||||
async _didFinishTestBody(testRun) {
|
||||
debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`);
|
||||
}
|
||||
|
||||
async _willStartHook(hook, fullName) {
|
||||
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
}
|
||||
|
||||
async _didFailHook(hook, fullName, message, error) {
|
||||
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
if (message)
|
||||
this._testPass._result.addError(message, error, this);
|
||||
this._testPass._result.setResult(TestResult.Crashed, message);
|
||||
}
|
||||
|
||||
async _didCompleteHook(hook, fullName) {
|
||||
debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
}
|
||||
|
||||
async shutdown() {
|
||||
while (this._suiteStack.length > 0) {
|
||||
const suite = this._suiteStack.pop();
|
||||
for (const hook of suite.hooks('afterAll'))
|
||||
await this._runHook({}, hook, suite.fullName());
|
||||
await this._runHook(null, hook, suite.fullName());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -469,7 +529,7 @@ class TestPass {
|
||||
this._terminating = false;
|
||||
}
|
||||
|
||||
async run(testList) {
|
||||
async run(testRuns) {
|
||||
const terminations = [
|
||||
createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'),
|
||||
createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'),
|
||||
@ -480,24 +540,21 @@ class TestPass {
|
||||
for (const termination of terminations)
|
||||
process.on(termination.event, termination.handler);
|
||||
|
||||
for (const test of testList) {
|
||||
test.result = null;
|
||||
test.error = null;
|
||||
}
|
||||
this._result = new Result();
|
||||
this._result.runs = testRuns;
|
||||
|
||||
const parallel = Math.min(this._parallel, testList.length);
|
||||
const parallel = Math.min(this._parallel, testRuns.length);
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < parallel; ++i) {
|
||||
const initialTestIndex = i * Math.floor(testList.length / parallel);
|
||||
workerPromises.push(this._runWorker(initialTestIndex, testList, i));
|
||||
const initialTestRunIndex = i * Math.floor(testRuns.length / parallel);
|
||||
workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i));
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
for (const termination of terminations)
|
||||
process.removeListener(termination.event, termination.handler);
|
||||
|
||||
if (this._runner.failedTests().length)
|
||||
if (testRuns.some(run => run.isFailure()))
|
||||
this._result.setResult(TestResult.Failed, '');
|
||||
return this._result;
|
||||
|
||||
@ -510,25 +567,25 @@ class TestPass {
|
||||
}
|
||||
}
|
||||
|
||||
async _runWorker(testIndex, testList, parallelIndex) {
|
||||
async _runWorker(testRunIndex, testRuns, parallelIndex) {
|
||||
let worker = new TestWorker(this, this._nextWorkerId++, parallelIndex);
|
||||
this._workers[parallelIndex] = worker;
|
||||
while (!worker._terminating) {
|
||||
let skipped = 0;
|
||||
while (skipped < testList.length && testList[testIndex].result !== null) {
|
||||
testIndex = (testIndex + 1) % testList.length;
|
||||
while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) {
|
||||
testRunIndex = (testRunIndex + 1) % testRuns.length;
|
||||
skipped++;
|
||||
}
|
||||
const test = testList[testIndex];
|
||||
if (test.result !== null) {
|
||||
const testRun = testRuns[testRunIndex];
|
||||
if (testRun._result !== null) {
|
||||
// All tests have been run.
|
||||
break;
|
||||
}
|
||||
|
||||
// Mark as running so that other workers do not run it again.
|
||||
test.result = 'running';
|
||||
await worker.runTest(test);
|
||||
if (isTestFailure(test.result)) {
|
||||
testRun._result = 'running';
|
||||
await worker.run(testRun);
|
||||
if (testRun.isFailure()) {
|
||||
// Something went wrong during test run, let's use a fresh worker.
|
||||
await worker.shutdown();
|
||||
if (this._breakOnFailure) {
|
||||
@ -556,39 +613,6 @@ class TestPass {
|
||||
this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null);
|
||||
}
|
||||
}
|
||||
|
||||
async _willStartTest(worker, test) {
|
||||
test.startTimestamp = Date.now();
|
||||
this._runner.emit(TestRunner.Events.TestStarted, test, worker._workerId);
|
||||
}
|
||||
|
||||
async _didFinishTest(worker, test) {
|
||||
test.endTimestamp = Date.now();
|
||||
this._runner.emit(TestRunner.Events.TestFinished, test, worker._workerId);
|
||||
}
|
||||
|
||||
async _willStartTestBody(worker, test) {
|
||||
debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
|
||||
}
|
||||
|
||||
async _didFinishTestBody(worker, test) {
|
||||
debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
|
||||
}
|
||||
|
||||
async _willStartHook(worker, hook, fullName) {
|
||||
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
}
|
||||
|
||||
async _didFailHook(worker, hook, fullName, message, error) {
|
||||
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
if (message)
|
||||
this._result.addError(message, error, worker);
|
||||
this._result.setResult(TestResult.Crashed, message);
|
||||
}
|
||||
|
||||
async _didCompleteHook(worker, hook, fullName) {
|
||||
debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`);
|
||||
}
|
||||
}
|
||||
|
||||
class TestRunner extends EventEmitter {
|
||||
@ -625,10 +649,10 @@ class TestRunner extends EventEmitter {
|
||||
this.it = this._testBuilder([]);
|
||||
|
||||
if (installCommonHelpers) {
|
||||
this.fdescribe = this.describe.setup(s => s.setMode(s.Modes.Focus));
|
||||
this.xdescribe = this.describe.setup(s => s.setMode(s.Modes.Skip));
|
||||
this.fit = this.it.setup(t => t.setMode(t.Modes.Focus));
|
||||
this.xit = this.it.setup(t => t.setMode(t.Modes.Skip));
|
||||
this.fdescribe = this.describe.setup(s => s.setFocused(true));
|
||||
this.xdescribe = this.describe.setup(s => s.setSkipped(true));
|
||||
this.fit = this.it.setup(t => t.setFocused(true));
|
||||
this.xit = this.it.setup(t => t.setSkipped(true));
|
||||
}
|
||||
}
|
||||
|
||||
@ -638,12 +662,10 @@ class TestRunner extends EventEmitter {
|
||||
const suite = new Suite(this._currentSuite, name, location);
|
||||
for (const { callback, args } of callbacks)
|
||||
callback(suite, ...args);
|
||||
for (let i = 0; i < suite.repeat(); i++) {
|
||||
this._currentSuite = suite._clone();
|
||||
callback(...suiteArgs);
|
||||
this._suites.push(this._currentSuite);
|
||||
this._currentSuite = this._currentSuite.parentSuite();
|
||||
}
|
||||
this._currentSuite = suite;
|
||||
callback(...suiteArgs);
|
||||
this._suites.push(suite);
|
||||
this._currentSuite = suite.parentSuite();
|
||||
}, {
|
||||
get: (obj, prop) => {
|
||||
if (prop === 'setup')
|
||||
@ -664,8 +686,7 @@ class TestRunner extends EventEmitter {
|
||||
test.setTimeout(this._timeout);
|
||||
for (const { callback, args } of callbacks)
|
||||
callback(test, ...args);
|
||||
for (let i = 0; i < test.repeat(); i++)
|
||||
this._tests.push(test._clone());
|
||||
this._tests.push(test);
|
||||
}, {
|
||||
get: (obj, prop) => {
|
||||
if (prop === 'setup')
|
||||
@ -697,11 +718,19 @@ class TestRunner extends EventEmitter {
|
||||
|
||||
async run(options = {}) {
|
||||
const { totalTimeout = 0 } = options;
|
||||
const runnableTests = this.runnableTests();
|
||||
this.emit(TestRunner.Events.Started, runnableTests);
|
||||
const testRuns = [];
|
||||
for (const test of this._testsToRun()) {
|
||||
let repeat = test.repeat();
|
||||
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
||||
repeat *= suite.repeat();
|
||||
for (let i = 0; i < repeat; i++)
|
||||
testRuns.push(new TestRun(test));
|
||||
}
|
||||
this.emit(TestRunner.Events.Started, testRuns);
|
||||
|
||||
let result = new Result();
|
||||
let result;
|
||||
if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) {
|
||||
result = new Result();
|
||||
result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI');
|
||||
} else {
|
||||
this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
|
||||
@ -712,7 +741,7 @@ class TestRunner extends EventEmitter {
|
||||
}, totalTimeout);
|
||||
}
|
||||
try {
|
||||
result = await this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; });
|
||||
result = await this._runningPass.run(testRuns).catch(e => { console.error(e); throw e; });
|
||||
} finally {
|
||||
this._runningPass = null;
|
||||
clearTimeout(timeoutId);
|
||||
@ -722,13 +751,13 @@ class TestRunner extends EventEmitter {
|
||||
return result;
|
||||
}
|
||||
|
||||
runnableTests() {
|
||||
_testsToRun() {
|
||||
if (!this.hasFocusedTestsOrSuites())
|
||||
return this._tests.slice();
|
||||
return this._tests;
|
||||
const notFocusedSuites = new Set();
|
||||
// Mark parent suites of focused tests as not focused.
|
||||
for (const test of this._tests) {
|
||||
if (test.mode() === TestMode.Focus) {
|
||||
if (test.focused()) {
|
||||
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
||||
notFocusedSuites.add(suite);
|
||||
}
|
||||
@ -736,9 +765,9 @@ class TestRunner extends EventEmitter {
|
||||
// Pick all tests that are focused or belong to focused suites.
|
||||
const tests = [];
|
||||
for (const test of this._tests) {
|
||||
let focused = test.mode() === TestMode.Focus;
|
||||
let focused = test.focused();
|
||||
for (let suite = test.suite(); suite; suite = suite.parentSuite())
|
||||
focused = focused || (suite.mode() === TestMode.Focus && !notFocusedSuites.has(suite));
|
||||
focused = focused || (suite.focused() && !notFocusedSuites.has(suite));
|
||||
if (focused)
|
||||
tests.push(test);
|
||||
}
|
||||
@ -755,22 +784,14 @@ class TestRunner extends EventEmitter {
|
||||
return this._timeout;
|
||||
}
|
||||
|
||||
focusedSuites() {
|
||||
return this._suites.filter(suite => suite.mode() === TestMode.Focus);
|
||||
}
|
||||
|
||||
focusedTests() {
|
||||
return this._tests.filter(test => test.mode() === TestMode.Focus);
|
||||
}
|
||||
|
||||
hasFocusedTestsOrSuites() {
|
||||
return !!this.focusedTests().length || !!this.focusedSuites().length;
|
||||
return this._tests.some(test => test.focused()) || this._suites.some(suite => suite.focused());
|
||||
}
|
||||
|
||||
focusMatchingTests(fullNameRegex) {
|
||||
for (const test of this._tests) {
|
||||
if (fullNameRegex.test(test.fullName()))
|
||||
test.setMode(TestMode.Focus);
|
||||
test.setFocused(true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -778,20 +799,8 @@ class TestRunner extends EventEmitter {
|
||||
return this._tests.slice();
|
||||
}
|
||||
|
||||
failedTests() {
|
||||
return this._tests.filter(test => isTestFailure(test.result));
|
||||
}
|
||||
|
||||
passedTests() {
|
||||
return this._tests.filter(test => test.result === TestResult.Ok);
|
||||
}
|
||||
|
||||
skippedTests() {
|
||||
return this._tests.filter(test => test.result === TestResult.Skipped);
|
||||
}
|
||||
|
||||
markedAsFailingTests() {
|
||||
return this._tests.filter(test => test.result === TestResult.MarkedAsFailing);
|
||||
suites() {
|
||||
return this._suites.slice();
|
||||
}
|
||||
|
||||
parallel() {
|
||||
|
||||
@ -19,12 +19,21 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const test = t.tests()[0];
|
||||
expect(test.name()).toBe('uno');
|
||||
expect(test.fullName()).toBe('uno');
|
||||
expect(test.mode()).toBe('run');
|
||||
expect(test.focused()).toBe(false);
|
||||
expect(test.skipped()).toBe(false);
|
||||
expect(test.location().filePath).toEqual(__filename);
|
||||
expect(test.location().fileName).toEqual('testrunner.spec.js');
|
||||
expect(test.location().lineNumber).toBeTruthy();
|
||||
expect(test.location().columnNumber).toBeTruthy();
|
||||
});
|
||||
it('should run a test', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', () => {});
|
||||
const result = await t.run();
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('ok');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TestRunner.xit', () => {
|
||||
@ -35,8 +44,16 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const test = t.tests()[0];
|
||||
expect(test.name()).toBe('uno');
|
||||
expect(test.fullName()).toBe('uno');
|
||||
expect(test.mode()).toBe('skip');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
expect(test.focused()).toBe(false);
|
||||
expect(test.skipped()).toBe(true);
|
||||
});
|
||||
it('should not run a skipped test', async() => {
|
||||
const t = newTestRunner();
|
||||
t.xit('uno', () => {});
|
||||
const result = await t.run();
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('skipped');
|
||||
});
|
||||
});
|
||||
|
||||
@ -48,8 +65,16 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const test = t.tests()[0];
|
||||
expect(test.name()).toBe('uno');
|
||||
expect(test.fullName()).toBe('uno');
|
||||
expect(test.mode()).toBe('focus');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
expect(test.focused()).toBe(true);
|
||||
expect(test.skipped()).toBe(false);
|
||||
});
|
||||
it('should run a focused test', async() => {
|
||||
const t = newTestRunner();
|
||||
t.fit('uno', () => {});
|
||||
const result = await t.run();
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('ok');
|
||||
});
|
||||
it('should run a failed focused test', async() => {
|
||||
const t = newTestRunner();
|
||||
@ -57,10 +82,11 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
t.fit.setup(t => t.setExpectation(t.Expectations.Fail))('uno', () => {
|
||||
run = true; throw new Error('failure');
|
||||
});
|
||||
expect(t.tests().length).toBe(1);
|
||||
await t.run();
|
||||
const result = await t.run();
|
||||
expect(run).toBe(true);
|
||||
expect(t.failedTests()[0].name()).toBe('uno');
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
@ -74,11 +100,12 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const test = t.tests()[0];
|
||||
expect(test.name()).toBe('uno');
|
||||
expect(test.fullName()).toBe('suite uno');
|
||||
expect(test.mode()).toBe('run');
|
||||
expect(test.focused()).toBe(false);
|
||||
expect(test.skipped()).toBe(false);
|
||||
expect(test.suite().name()).toBe('suite');
|
||||
expect(test.suite().fullName()).toBe('suite');
|
||||
expect(test.suite().mode()).toBe('run');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
expect(test.suite().focused()).toBe(false);
|
||||
expect(test.suite().skipped()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@ -90,20 +117,22 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
});
|
||||
expect(t.tests().length).toBe(1);
|
||||
const test = t.tests()[0];
|
||||
expect(test.mode()).toBe('run');
|
||||
expect(test.suite().mode()).toBe('skip');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
expect(test.focused()).toBe(false);
|
||||
expect(test.skipped()).toBe(false);
|
||||
expect(test.suite().focused()).toBe(false);
|
||||
expect(test.suite().skipped()).toBe(true);
|
||||
});
|
||||
it('focused tests inside a skipped suite are not run', async() => {
|
||||
const t = newTestRunner();
|
||||
let run = false;
|
||||
t.xdescribe('suite', () => {
|
||||
t.fit('uno', () => {});
|
||||
t.fit('uno', () => { run = true; });
|
||||
});
|
||||
expect(t.tests().length).toBe(1);
|
||||
const test = t.tests()[0];
|
||||
expect(test.mode()).toBe('focus');
|
||||
expect(test.suite().mode()).toBe('skip');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
const result = await t.run();
|
||||
expect(run).toBe(false);
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('skipped');
|
||||
});
|
||||
});
|
||||
|
||||
@ -115,20 +144,20 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
});
|
||||
expect(t.tests().length).toBe(1);
|
||||
const test = t.tests()[0];
|
||||
expect(test.mode()).toBe('run');
|
||||
expect(test.suite().mode()).toBe('focus');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
expect(test.focused()).toBe(false);
|
||||
expect(test.skipped()).toBe(false);
|
||||
expect(test.suite().focused()).toBe(true);
|
||||
expect(test.suite().skipped()).toBe(false);
|
||||
});
|
||||
it('skipped tests inside a focused suite should not be run', async() => {
|
||||
const t = newTestRunner();
|
||||
t.fdescribe('suite', () => {
|
||||
t.xit('uno', () => {});
|
||||
});
|
||||
expect(t.tests().length).toBe(1);
|
||||
const test = t.tests()[0];
|
||||
expect(test.mode()).toBe('skip');
|
||||
expect(test.suite().mode()).toBe('focus');
|
||||
expect(t.runnableTests()).toEqual([test]);
|
||||
const result = await t.run();
|
||||
expect(result.runs.length).toBe(1);
|
||||
expect(result.runs[0].test()).toBe(t.tests()[0]);
|
||||
expect(result.runs[0].result()).toBe('skipped');
|
||||
});
|
||||
it('should run all "run" tests inside a focused suite', async() => {
|
||||
const log = [];
|
||||
@ -176,10 +205,8 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
t.testModifier('foo', (t, ...args) => {
|
||||
log.push('foo');
|
||||
|
||||
expect(t.Modes.Run).toBeTruthy();
|
||||
expect(t.Modes.Skip).toBeTruthy();
|
||||
expect(t.Modes.Focus).toBeTruthy();
|
||||
expect(t.mode()).toBe(t.Modes.Run);
|
||||
expect(t.focused()).toBe(false);
|
||||
expect(t.skipped()).toBe(false);
|
||||
expect(t.Expectations.Ok).toBeTruthy();
|
||||
expect(t.Expectations.Fail).toBeTruthy();
|
||||
expect(t.expectation()).toBe(t.Expectations.Ok);
|
||||
@ -190,7 +217,7 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
expect(args[0]).toBe('uno');
|
||||
expect(args[1]).toBe('dos');
|
||||
|
||||
t.setMode(t.Modes.Focus);
|
||||
t.setFocused(true);
|
||||
t.setExpectation(t.Expectations.Fail);
|
||||
t.setTimeout(234);
|
||||
t.setRepeat(42);
|
||||
@ -198,9 +225,9 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
|
||||
t.testAttribute('bar', t => {
|
||||
log.push('bar');
|
||||
expect(t.mode()).toBe(t.Modes.Focus);
|
||||
t.setMode(t.Modes.Skip);
|
||||
expect(t.mode()).toBe(t.Modes.Focus);
|
||||
t.setSkipped(true);
|
||||
expect(t.focused()).toBe(true);
|
||||
expect(t.skipped()).toBe(true);
|
||||
expect(t.expectation()).toBe(t.Expectations.Fail);
|
||||
expect(t.timeout()).toBe(234);
|
||||
expect(t.repeat()).toBe(42);
|
||||
@ -323,15 +350,15 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const t = newTestRunner({timeout: 10000});
|
||||
t.afterEach(() => { throw new Error('crash!'); });
|
||||
t.it('uno', () => { t.terminate(); });
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('terminated');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('terminated');
|
||||
});
|
||||
it('should report as terminated when terminated during hook', async() => {
|
||||
const t = newTestRunner({timeout: 10000});
|
||||
t.afterEach(() => { t.terminate(); });
|
||||
t.it('uno', () => { });
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('terminated');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('terminated');
|
||||
});
|
||||
it('should unwind hooks properly when crashed', async() => {
|
||||
const log = [];
|
||||
@ -389,8 +416,8 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
t.it.repeat(3)('uno', () => test++);
|
||||
});
|
||||
await t.run();
|
||||
expect(suite).toBe(2);
|
||||
expect(beforeAll).toBe(2);
|
||||
expect(suite).toBe(1);
|
||||
expect(beforeAll).toBe(1);
|
||||
expect(beforeEach).toBe(6);
|
||||
expect(test).toBe(6);
|
||||
});
|
||||
@ -513,7 +540,7 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
const t = newTestRunner({timeout: 10000});
|
||||
t.it('uno', async () => { await new Promise(() => {}); });
|
||||
const result = await t.run({totalTimeout: 1});
|
||||
expect(t.tests()[0].result).toBe('terminated');
|
||||
expect(result.runs[0].result()).toBe('terminated');
|
||||
expect(result.message).toContain('Total timeout');
|
||||
});
|
||||
});
|
||||
@ -603,92 +630,64 @@ module.exports.addTests = function({testRunner, expect}) {
|
||||
});
|
||||
});
|
||||
|
||||
describe('TestRunner.passedTests', () => {
|
||||
it('should work', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', () => {});
|
||||
await t.run();
|
||||
expect(t.failedTests().length).toBe(0);
|
||||
expect(t.skippedTests().length).toBe(0);
|
||||
expect(t.passedTests().length).toBe(1);
|
||||
const [test] = t.passedTests();
|
||||
expect(test.result).toBe('ok');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TestRunner.failedTests', () => {
|
||||
describe('TestRunner result', () => {
|
||||
it('should work for both throwing and timeouting tests', async() => {
|
||||
const t = newTestRunner({timeout: 1});
|
||||
t.it('uno', () => { throw new Error('boo');});
|
||||
t.it('dos', () => new Promise(() => {}));
|
||||
await t.run();
|
||||
expect(t.skippedTests().length).toBe(0);
|
||||
expect(t.passedTests().length).toBe(0);
|
||||
expect(t.failedTests().length).toBe(2);
|
||||
const [test1, test2] = t.failedTests();
|
||||
expect(test1.result).toBe('failed');
|
||||
expect(test2.result).toBe('timedout');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('failed');
|
||||
expect(result.runs[1].result()).toBe('timedout');
|
||||
});
|
||||
it('should report crashed tests', async() => {
|
||||
const t = newTestRunner();
|
||||
t.beforeEach(() => { throw new Error('woof');});
|
||||
t.it('uno', () => {});
|
||||
await t.run();
|
||||
expect(t.failedTests().length).toBe(1);
|
||||
expect(t.failedTests()[0].result).toBe('crashed');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('crashed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TestRunner.skippedTests', () => {
|
||||
it('should work for both throwing and timeouting tests', async() => {
|
||||
it('skipped should work for both throwing and timeouting tests', async() => {
|
||||
const t = newTestRunner({timeout: 1});
|
||||
t.xit('uno', () => { throw new Error('boo');});
|
||||
await t.run();
|
||||
expect(t.skippedTests().length).toBe(1);
|
||||
expect(t.passedTests().length).toBe(0);
|
||||
expect(t.failedTests().length).toBe(0);
|
||||
const [test] = t.skippedTests();
|
||||
expect(test.result).toBe('skipped');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('skipped');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Test.result', () => {
|
||||
it('should return OK', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', () => {});
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('ok');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('ok');
|
||||
});
|
||||
it('should return TIMEDOUT', async() => {
|
||||
const t = newTestRunner({timeout: 1});
|
||||
t.it('uno', async() => new Promise(() => {}));
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('timedout');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('timedout');
|
||||
});
|
||||
it('should return SKIPPED', async() => {
|
||||
const t = newTestRunner();
|
||||
t.xit('uno', () => {});
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('skipped');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('skipped');
|
||||
});
|
||||
it('should return FAILED', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', async() => Promise.reject('woof'));
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('failed');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('failed');
|
||||
});
|
||||
it('should return TERMINATED', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', async() => t.terminate());
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('terminated');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('terminated');
|
||||
});
|
||||
it('should return CRASHED', async() => {
|
||||
const t = newTestRunner();
|
||||
t.it('uno', () => {});
|
||||
t.afterEach(() => {throw new Error('foo');});
|
||||
await t.run();
|
||||
expect(t.tests()[0].result).toBe('crashed');
|
||||
const result = await t.run();
|
||||
expect(result.runs[0].result()).toBe('crashed');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user