diff --git a/package.json b/package.json index 76850f280a..d1fa6a712f 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,8 @@ "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", "doc-no-channel": "node utils/doclint/cli.js --no-channel", - "test-infra": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js", + "test-infra": "node test/runner utils/doclint/check_public_api/test/test.js && node test/runner utils/doclint/preprocessor/test.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run doc-no-channel && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra", - "debug-test": "node --inspect-brk test/test.js", "clean": "rimraf lib && rimraf types", "prepare": "node install-from-github.js", "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types", diff --git a/utils/testrunner/GoldenUtils.js b/test/runner/GoldenUtils.js similarity index 97% rename from utils/testrunner/GoldenUtils.js rename to test/runner/GoldenUtils.js index eb54cd92d8..af723822b6 100644 --- a/utils/testrunner/GoldenUtils.js +++ b/test/runner/GoldenUtils.js @@ -92,8 +92,8 @@ function compare(actual, golden) { const goldenPath = path.normalize(golden.goldenPath); const outputPath = path.normalize(golden.outputPath); const goldenName = golden.goldenName; - const expectedPath = path.join(goldenPath, goldenName); - const actualPath = path.join(outputPath, goldenName); + const expectedPath = path.resolve(goldenPath, goldenName); + const actualPath = path.resolve(outputPath, goldenName); const messageSuffix = 'Output is saved in "' + path.basename(outputPath + '" directory'); diff --git a/test/runner/worker.js b/test/runner/worker.js index eda692f2ca..1eb56fbf3e 100644 --- a/test/runner/worker.js +++ b/test/runner/worker.js @@ -19,7 +19,7 @@ const Mocha = require('mocha'); const { registerWorkerFixture } = require('./fixturePool'); const { fixturesUI, fixturePool } = require('./fixturesUI'); const { gracefullyCloseAll } = require('../../lib/server/processLauncher'); -const GoldenUtils = require('../../utils/testrunner/GoldenUtils'); +const GoldenUtils = require('./GoldenUtils'); const browserName = process.env.BROWSER || 'chromium'; const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index ee17e5f9f6..09af938006 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -21,70 +21,58 @@ const Source = require('../../Source'); const mdBuilder = require('../MDBuilder'); const jsBuilder = require('../JSBuilder'); -const TestRunner = require('../../../testrunner/'); -const runner = new TestRunner({ - goldenPath: __dirname, - outputPath: __dirname -}); - -const {describe, xdescribe, fdescribe} = runner.api(); -const {it, fit, xit} = runner.api(); -const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); -const {expect} = runner.api(); - -let browser; -let page; - -beforeAll(async function() { - browser = await playwright.chromium.launch(); - page = await browser.newPage(); -}); - -afterAll(async function() { +registerWorkerFixture('page', async({}, test) => { + const browser = await playwright.chromium.launch(); + const page = await browser.newPage(); + await test(page); await browser.close(); }); describe('checkPublicAPI', function() { - it('diff-classes', testLint); - it('diff-methods', testLint); - it('diff-properties', testLint); - it('diff-arguments', testLint); - it('diff-events', testLint); - it('check-duplicates', testLint); - it('check-sorting', testLint); - it('check-returns', testLint); - it('check-nullish', testLint); - it('js-builder-common', testJSBuilder); - it('js-builder-inheritance', testJSBuilder); - it('md-builder-common', testMDBuilder); - it('md-builder-comments', testMDBuilder); + testLint('diff-classes'); + testLint('diff-methods'); + testLint('diff-properties'); + testLint('diff-arguments'); + testLint('diff-events'); + testLint('check-duplicates'); + testLint('check-sorting'); + testLint('check-returns'); + testLint('check-nullish'); + testJSBuilder('js-builder-common'); + testJSBuilder('js-builder-inheritance'); + testMDBuilder('md-builder-common'); + testMDBuilder('md-builder-comments'); }); -runner.run(); - -async function testLint(state, testRun) { - const dirPath = path.join(__dirname, testRun.test().name()); - const mdSources = await Source.readdir(dirPath, '.md'); - const tsSources = await Source.readdir(dirPath, '.ts'); - const jsSources = await Source.readdir(dirPath, '.js'); - const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources)); - const errors = messages.map(message => message.text); - expect(errors.join('\n')).toBeGolden(path.join(testRun.test().name(), 'result.txt')); +async function testLint(name) { + it(name, async({page}) => { + const dirPath = path.join(__dirname, name); + const mdSources = await Source.readdir(dirPath, '.md'); + const tsSources = await Source.readdir(dirPath, '.ts'); + const jsSources = await Source.readdir(dirPath, '.js'); + const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources)); + const errors = messages.map(message => message.text); + expect(errors.join('\n')).toBeGolden(path.join(dirPath, 'result.txt')); + }); } -async function testMDBuilder(state, testRun) { - const dirPath = path.join(__dirname, testRun.test().name()); - const sources = await Source.readdir(dirPath, '.md'); - const {documentation} = await mdBuilder(page, sources); - expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); +async function testMDBuilder(name) { + it(name, async({page}) => { + const dirPath = path.join(__dirname, name); + const sources = await Source.readdir(dirPath, '.md'); + const {documentation} = await mdBuilder(page, sources); + expect(serialize(documentation)).toBeGolden(path.join(dirPath, 'result.txt')); + }); } -async function testJSBuilder(state, testRun) { - const dirPath = path.join(__dirname, testRun.test().name()); - const jsSources = await Source.readdir(dirPath, '.js'); - const tsSources = await Source.readdir(dirPath, '.ts'); - const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources)); - expect(serialize(documentation)).toBeGolden(path.join(testRun.test().name(), 'result.txt')); +async function testJSBuilder(name) { + it(name, async() => { + const dirPath = path.join(__dirname, name); + const jsSources = await Source.readdir(dirPath, '.js'); + const tsSources = await Source.readdir(dirPath, '.ts'); + const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources)); + expect(serialize(documentation)).toBeGolden(path.join(dirPath, 'result.txt')); + }); } /** diff --git a/utils/doclint/preprocessor/test.js b/utils/doclint/preprocessor/test.js index 94a8645c92..01d8c0cb4a 100644 --- a/utils/doclint/preprocessor/test.js +++ b/utils/doclint/preprocessor/test.js @@ -14,15 +14,8 @@ * limitations under the License. */ -const {runCommands, ensureTipOfTreeAPILinks} = require('.'); +const {runCommands} = require('.'); const Source = require('../Source'); -const TestRunner = require('../../testrunner/'); -const runner = new TestRunner(); - -const {describe, xdescribe, fdescribe} = runner.api(); -const {it, fit, xit} = runner.api(); -const {beforeAll, beforeEach, afterAll, afterEach} = runner.api(); -const {expect} = runner.api(); describe('runCommands', function() { const OPTIONS_REL = { @@ -202,5 +195,3 @@ describe('runCommands', function() { }); }); -runner.run(); - diff --git a/utils/testrunner/.npmignore b/utils/testrunner/.npmignore deleted file mode 100644 index 7b4e15d4a0..0000000000 --- a/utils/testrunner/.npmignore +++ /dev/null @@ -1,13 +0,0 @@ -# exclude all examples and README.md -examples/ -README.md - -# repeats from .gitignore -node_modules -.npmignore -.DS_Store -*.swp -*.pyc -.vscode -package-lock.json -yarn.lock diff --git a/utils/testrunner/Location.js b/utils/testrunner/Location.js deleted file mode 100644 index 2a6eeacd3d..0000000000 --- a/utils/testrunner/Location.js +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); - -// Hack for our own tests. -const testRunnerTestFile = path.join(__dirname, 'test', 'testrunner.spec.js'); - -class Location { - constructor() { - this._fileName = ''; - this._filePath = ''; - this._lineNumber = 0; - this._columnNumber = 0; - } - - fileName() { - return this._fileName; - } - - filePath() { - return this._filePath; - } - - lineNumber() { - return this._lineNumber; - } - - columnNumber() { - return this._columnNumber; - } - - toString() { - return this._fileName + ':' + this._lineNumber; - } - - toDetailedString() { - return this._fileName + ':' + this._lineNumber + ':' + this._columnNumber; - } - - static getCallerLocation(ignorePrefix = __dirname) { - const error = new Error(); - const stackFrames = error.stack.split('\n').slice(1); - const location = new Location(); - // Find first stackframe that doesn't point to this file. - for (let frame of stackFrames) { - frame = frame.trim(); - if (!frame.startsWith('at ')) - return null; - if (frame.endsWith(')')) { - const from = frame.indexOf('('); - frame = frame.substring(from + 1, frame.length - 1); - } else { - frame = frame.substring('at '.length); - } - - const match = frame.match(/^(.*):(\d+):(\d+)$/); - if (!match) - return null; - const filePath = match[1]; - if (filePath === __filename || (filePath.startsWith(ignorePrefix) && filePath !== testRunnerTestFile)) - continue; - - location._filePath = filePath; - location._fileName = filePath.split(path.sep).pop(); - location._lineNumber = parseInt(match[2], 10); - location._columnNumber = parseInt(match[3], 10); - return location; - } - return location; - } -} - -module.exports = Location; diff --git a/utils/testrunner/Matchers.js b/utils/testrunner/Matchers.js deleted file mode 100644 index e9b64de90f..0000000000 --- a/utils/testrunner/Matchers.js +++ /dev/null @@ -1,247 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const Location = require('./Location.js'); -const colors = require('colors/safe'); -const Diff = require('text-diff'); -const GoldenUtils = require('./GoldenUtils'); - -class Matchers { - constructor(config) { - this.expect = this.expect.bind(this); - - this._matchers = { - toBe: function(received, expected, message) { - message = message || `${received} == ${expected}`; - return { pass: received === expected, message, formatter: toBeFormatter.bind(null, received, expected) }; - }, - - toBeFalsy: function(received, message) { - message = message || `${received}`; - return { pass: !received, message }; - }, - - toBeTruthy: function(received, message) { - message = message || `${received}`; - return { pass: !!received, message }; - }, - - toBeGreaterThan: function(received, other, message) { - message = message || `${received} > ${other}`; - return { pass: received > other, message }; - }, - - toBeGreaterThanOrEqual: function(received, other, message) { - message = message || `${received} >= ${other}`; - return { pass: received >= other, message }; - }, - - toBeLessThan: function(received, other, message) { - message = message || `${received} < ${other}`; - return { pass: received < other, message }; - }, - - toBeLessThanOrEqual: function(received, other, message) { - message = message || `${received} <= ${other}`; - return { pass: received <= other, message }; - }, - - toBeNull: function(received, message) { - message = message || `${received} == null`; - return { pass: received === null, message }; - }, - - toContain: function(received, other, message) { - message = message || `${received} ⊇ ${other}`; - return { pass: received.includes(other), message }; - }, - - toEqual: function(received, other, message) { - let receivedJson = stringify(received); - let otherJson = stringify(other); - let formatter = objectFormatter.bind(null, receivedJson, otherJson); - if (receivedJson.length < 40 && otherJson.length < 40) { - receivedJson = receivedJson.split('\n').map(line => line.trim()).join(' '); - otherJson = otherJson.split('\n').map(line => line.trim()).join(' '); - formatter = stringFormatter.bind(null, receivedJson, otherJson); - } - message = message || `\n${receivedJson} ≈ ${otherJson}`; - return { pass: receivedJson === otherJson, message, formatter }; - }, - - toBeCloseTo: function(received, other, precision, message) { - return { - pass: Math.abs(received - other) < Math.pow(10, -precision), - message - }; - }, - - toBeInstanceOf: function(received, other, message) { - message = message || `${received.constructor.name} instanceof ${other.name}`; - return { pass: received instanceof other, message }; - }, - - toBeGolden: function(received, goldenName) { - return GoldenUtils.compare(received, { - goldenPath: config.goldenPath, - outputPath: config.outputPath, - goldenName - }); - }, - - toMatch: function(received, other, message) { - message = message || `${received}`; - return { pass: received.match(other), message }; - } - } - } - - expect(received) { - return new Expect(received, this._matchers); - } -}; - -class MatchError extends Error { - constructor(message, formatter) { - super(message); - this.name = this.constructor.name; - this.formatter = formatter; - this.location = Location.getCallerLocation(); - Error.captureStackTrace(this, this.constructor); - } -} - -module.exports = {Matchers, MatchError}; - -class Expect { - constructor(received, matchers) { - this.not = {}; - this.not.not = this; - for (const matcherName of Object.keys(matchers)) { - const matcher = matchers[matcherName]; - this[matcherName] = applyMatcher.bind(null, matcherName, matcher, false /* inverse */, received); - this.not[matcherName] = applyMatcher.bind(null, matcherName, matcher, true /* inverse */, received); - } - - function applyMatcher(matcherName, matcher, inverse, received, ...args) { - const result = matcher.call(null, received, ...args); - const message = `expect.${inverse ? 'not.' : ''}${matcherName} failed` + (result.message ? `: ${result.message}` : ''); - if (result.pass === inverse) - throw new MatchError(message, result.formatter || defaultFormatter.bind(null, received)); - } - } -} - -function defaultFormatter(received) { - return `Received: ${colors.red(JSON.stringify(received))}`; -} - -function stringFormatter(received, expected) { - const diff = new Diff(); - const result = diff.main(expected, received); - diff.cleanupSemantic(result); - const highlighted = result.map(([type, text]) => { - if (type === -1) - return colors.bgRed(text); - if (type === 1) - return colors.bgGreen.black(text); - return text; - }).join(''); - const output = [ - `Expected: ${expected}`, - `Received: ${received}`, - ` Diff: ${highlighted}`, - ]; - for (let i = 0; i < Math.min(expected.length, received.length); ++i) { - if (expected[i] !== received[i]) { - const padding = ' '.repeat(' Diff: '.length); - const firstDiffCharacter = '~'.repeat(i) + '^'; - output.push(colors.red(padding + firstDiffCharacter)); - break; - } - } - return output.join('\n'); -} - -function objectFormatter(received, expected) { - const encodingMap = new Map(); - const decodingMap = new Map(); - - const doEncodeLines = (lines) => { - let encoded = ''; - for (const line of lines) { - let code = encodingMap.get(line); - if (!code) { - code = String.fromCodePoint(encodingMap.size); - encodingMap.set(line, code); - decodingMap.set(code, line); - } - encoded += code; - } - return encoded; - }; - - const doDecodeLines = (text) => { - let decoded = []; - for (const codepoint of [...text]) - decoded.push(decodingMap.get(codepoint)); - return decoded; - } - - let receivedEncoded = doEncodeLines(received.split('\n')); - let expectedEncoded = doEncodeLines(expected.split('\n')); - - const diff = new Diff(); - const result = diff.main(expectedEncoded, receivedEncoded); - diff.cleanupSemantic(result); - - const highlighted = result.map(([type, text]) => { - const lines = doDecodeLines(text); - if (type === -1) - return lines.map(line => '- ' + colors.bgRed(line)); - if (type === 1) - return lines.map(line => '+ ' + colors.bgGreen.black(line)); - return lines.map(line => ' ' + line); - }); - - const flattened = []; - for (const list of highlighted) - flattened.push(...list); - return `Received:\n${flattened.join('\n')}`; -} - -function toBeFormatter(received, expected) { - if (typeof expected === 'string' && typeof received === 'string') { - return stringFormatter(JSON.stringify(received), JSON.stringify(expected)); - } - return [ - `Expected: ${JSON.stringify(expected)}`, - `Received: ${colors.red(JSON.stringify(received))}`, - ].join('\n'); -} - -function stringify(value) { - function stabilize(key, object) { - if (typeof object !== 'object' || object === undefined || object === null || Array.isArray(object)) - return object; - const result = {}; - for (const key of Object.keys(object).sort()) - result[key] = object[key]; - return result; - } - - return JSON.stringify(stabilize(null, value), stabilize, 2); -} diff --git a/utils/testrunner/README.md b/utils/testrunner/README.md deleted file mode 100644 index 9bf5fc6e99..0000000000 --- a/utils/testrunner/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# TestRunner - -This test runner is used internally by Playwright to test Playwright itself. - -- testrunner is a *library*: tests are `node.js` scripts -- parallel wrt IO operations -- supports async/await -- modular -- well-isolated state per execution thread - -### Example - -Save the following as `test.js` and run using `node`: - -```sh -node test.js -``` - -```js -const {TestRunner, Reporter, Matchers} = require('.'); - -// Runner holds and runs all the tests -const runner = new TestRunner({ - parallel: 2, // run 2 parallel threads - timeout: 1000, // setup timeout of 1 second per test -}); -// Simple expect-like matchers -const {expect} = new Matchers(); - -// Extract jasmine-like DSL into the global namespace -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -// Test hooks can be async. -beforeAll(async state => { - state.parallelIndex; // either 0 or 1 in this example, depending on the executing thread - state.foo = 'bar'; // set state for every test -}); - -describe('math', () => { - it('to be sane', async (state, test) => { - state.parallelIndex; // Very first test will always be ran by the 0's thread - state.foo; // this will be 'bar' - expect(2 + 2).toBe(4); - }); -}); - -// Reporter subscribes to TestRunner events and displays information in terminal -const reporter = new Reporter(runner); - -// Run all tests. -runner.run(); -``` - diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js deleted file mode 100644 index 803cc820de..0000000000 --- a/utils/testrunner/Reporter.js +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const fs = require('fs'); -const path = require('path'); -const colors = require('colors/safe'); -const {MatchError} = require('./Matchers.js'); - -class Reporter { - constructor(delegate, options = {}) { - const { - showSlowTests = 3, - showMarkedAsFailingTests = Infinity, - verbose = false, - summary = true, - lineBreak = 0, - } = options; - this._filePathToLines = new Map(); - this._delegate = delegate; - this._showSlowTests = showSlowTests; - this._showMarkedAsFailingTests = showMarkedAsFailingTests; - this._verbose = verbose; - this._summary = summary; - this._lineBreak = lineBreak; - this._testCounter = 0; - } - - onStarted(testRuns) { - this._testCounter = 0; - this._timestamp = Date.now(); - if (!this._delegate.hasFocusedTestsOrSuitesOrFiles()) { - console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}:\n`); - } else { - console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(this._delegate.testCount())} on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}`); - console.log(''); - const focusedFilePaths = this._delegate.focusedFilePaths(); - if (focusedFilePaths.length) { - console.log('Focused Files:'); - for (let i = 0; i < focusedFilePaths.length; ++i) - console.log(` ${i + 1}) ${colors.yellow(path.basename(focusedFilePaths[i]))}`); - console.log(''); - } - const focusedEntities = [ - ...this._delegate.focusedSuites(), - ...this._delegate.focusedTests(), - ]; - - if (focusedEntities.length) { - console.log('Focused Suites and Tests:'); - for (let i = 0; i < focusedEntities.length; ++i) - console.log(` ${i + 1}) ${focusedEntities[i].fullName()} (${formatLocation(focusedEntities[i].location())})`); - console.log(''); - } - } - } - - _printFailedResult(result) { - console.log(colors.red(`## ${result.result.toUpperCase()} ##`)); - if (result.message) { - console.log('Message:'); - console.log(` ${colors.red(result.message)}`); - } - - for (let i = 0; i < result.errors.length; i++) { - const { message, error, runs } = result.errors[i]; - console.log(`\n${colors.magenta('NON-TEST ERROR #' + i)}: ${message}`); - if (error && error.stack) - console.log(padLines(error.stack, 2)); - const lastRuns = runs.slice(runs.length - Math.min(10, runs.length)); - if (lastRuns.length) - console.log(`WORKER STATE`); - for (let j = 0; j < lastRuns.length; j++) - this._printVerboseTestRunResult(j, lastRuns[j]); - } - console.log(''); - console.log(''); - } - - onFinished(result) { - this._printTestResults(result); - if (!result.ok()) - this._printFailedResult(result); - process.exitCode = result.exitCode; - } - - _printTestResults(result) { - // 2 newlines after completing all tests. - console.log('\n'); - - const runs = result.runs; - const failedRuns = runs.filter(run => run.isFailure()); - const executedRuns = runs.filter(run => run.result()); - const okRuns = runs.filter(run => run.ok()); - const skippedRuns = runs.filter(run => run.result() === 'skipped'); - const markedAsFailingRuns = runs.filter(run => run.result() === 'markedAsFailing'); - - if (this._summary && failedRuns.length > 0) { - console.log('\nFailures:'); - for (let i = 0; i < failedRuns.length; ++i) { - this._printVerboseTestRunResult(i + 1, failedRuns[i]); - console.log(''); - } - } - - if (this._showMarkedAsFailingTests && this._summary && markedAsFailingRuns.length) { - if (markedAsFailingRuns.length > 0) { - console.log('\nMarked as failing:'); - markedAsFailingRuns.slice(0, this._showMarkedAsFailingTests).forEach((testRun, index) => { - console.log(`${index + 1}) ${testRun.test().fullName()} (${formatLocation(testRun.test().location())})`); - }); - } - if (this._showMarkedAsFailingTests < markedAsFailingRuns.length) { - console.log(''); - console.log(`... and ${colors.yellow(markedAsFailingRuns.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`); - } - } - - if (this._showSlowTests) { - const slowRuns = okRuns.sort((a, b) => b.duration() - a.duration()).slice(0, this._showSlowTests); - console.log(`\nSlowest tests:`); - for (let i = 0; i < slowRuns.length; ++i) { - const run = slowRuns[i]; - console.log(` (${i + 1}) ${colors.yellow((run.duration() / 1000) + 's')} - ${run.test().fullName()} (${formatLocation(run.test().location())})`); - } - } - - let summaryText = ''; - if (failedRuns.length || markedAsFailingRuns.length) { - const summary = [`ok - ${colors.green(okRuns.length)}`]; - if (failedRuns.length) - summary.push(`failed - ${colors.red(failedRuns.length)}`); - if (markedAsFailingRuns.length) - summary.push(`marked as failing - ${colors.yellow(markedAsFailingRuns.length)}`); - if (skippedRuns.length) - summary.push(`skipped - ${colors.yellow(skippedRuns.length)}`); - summaryText = ` (${summary.join(', ')})`; - } - - console.log(`\nRan ${executedRuns.length}${summaryText} of ${runs.length} test${runs.length > 1 ? 's' : ''}`); - const milliseconds = Date.now() - this._timestamp; - const seconds = milliseconds / 1000; - console.log(`Finished in ${colors.yellow(seconds)} seconds`); - } - - onTestRunStarted(testRun) { - } - - onTestRunFinished(testRun) { - ++this._testCounter; - if (this._verbose) { - this._printVerboseTestRunResult(this._testCounter, testRun); - } else { - if (testRun.result() === 'ok') - process.stdout.write(colors.green('\u00B7')); - else if (testRun.result() === 'skipped') - process.stdout.write(colors.yellow('\u00B7')); - else if (testRun.result() === 'markedAsFailing') - process.stdout.write(colors.yellow('\u00D7')); - else if (testRun.result() === 'failed') - process.stdout.write(colors.red('F')); - else if (testRun.result() === 'crashed') - process.stdout.write(colors.red('C')); - else if (testRun.result() === 'terminated') - process.stdout.write(colors.magenta('.')); - else if (testRun.result() === 'timedout') - process.stdout.write(colors.red('T')); - if (this._lineBreak && !(this._testCounter % this._lineBreak)) - process.stdout.write('\n'); - } - } - - _printVerboseTestRunResult(resultIndex, testRun) { - const test = testRun.test(); - let prefix = `${resultIndex})`; - if (this._delegate.parallel() > 1) - prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`); - if (testRun.result() === 'ok') { - console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (testRun.result() === 'terminated') { - console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (testRun.result() === 'crashed') { - console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (testRun.result() === 'skipped') { - } else if (testRun.result() === 'markedAsFailing') { - console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (testRun.result() === 'timedout') { - console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`); - const output = testRun.output(); - if (output.length) { - console.log(' Output:'); - for (const line of output) - console.log(' ' + line); - } - } else if (testRun.result() === 'failed') { - console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`); - if (testRun.error() instanceof MatchError) { - const location = testRun.error().location; - let lines = this._filePathToLines.get(location.filePath()); - if (!lines) { - try { - lines = fs.readFileSync(location.filePath(), 'utf8').split('\n'); - } catch (e) { - lines = []; - } - this._filePathToLines.set(location.filePath(), lines); - } - const lineNumber = location.lineNumber(); - if (lineNumber < lines.length) { - const lineNumberLength = (lineNumber + 1 + '').length; - const FROM = Math.max(0, lineNumber - 5); - const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n'); - const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(location.columnNumber() - 1) + '^'; - console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n'); - } - console.log(padLines(testRun.error().formatter(), 4)); - console.log(''); - } else { - console.log(' Message:'); - let message = '' + (testRun.error().message || testRun.error()); - if (testRun.error().stack && message.includes(testRun.error().stack)) - message = message.substring(0, message.indexOf(testRun.error().stack)); - if (message) - console.log(` ${colors.red(message)}`); - if (testRun.error().stack) { - console.log(' Stack:'); - let stack = testRun.error().stack; - // Highlight first test location, if any. - const match = stack.match(new RegExp(test.location().filePath() + ':(\\d+):(\\d+)')); - if (match) { - const [, line, column] = match; - const fileName = `${test.location().fileName()}:${line}:${column}`; - stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName)); - } - console.log(padLines(stack, 4)); - } - } - const output = testRun.output(); - if (output.length) { - console.log(' Output:'); - for (const line of output) - console.log(' ' + line); - } - } - } -} - -function formatLocation(location) { - if (!location) - return ''; - return colors.yellow(`${location.toDetailedString()}`); -} - -function padLines(text, spaces = 0) { - const indent = ' '.repeat(spaces); - return text.split('\n').map(line => indent + line).join('\n'); -} - -module.exports = Reporter; diff --git a/utils/testrunner/SourceMap.js b/utils/testrunner/SourceMap.js deleted file mode 100644 index c919715198..0000000000 --- a/utils/testrunner/SourceMap.js +++ /dev/null @@ -1,460 +0,0 @@ -/* - * Copyright (C) 2012 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -const path = require('path'); - -function upperBound(array, object, comparator, left, right) { - function defaultComparator(a, b) { - return a < b ? -1 : (a > b ? 1 : 0); - } - comparator = comparator || defaultComparator; - let l = left || 0; - let r = right !== undefined ? right : array.length; - while (l < r) { - const m = (l + r) >> 1; - if (comparator(object, array[m]) >= 0) { - l = m + 1; - } else { - r = m; - } - } - return r; -} - -/** - * @interface - */ -class SourceMap { - /** - * @return {string} - */ - compiledURL() { - } - - /** - * @return {string} - */ - url() { - } - - /** - * @return {!Array} - */ - sourceURLs() { - } - - /** - * @param {string} sourceURL - * @return {?string} - */ - embeddedContentByURL(sourceURL) { - } - - /** - * @param {number} lineNumber in compiled resource - * @param {number} columnNumber in compiled resource - * @return {?SourceMapEntry} - */ - findEntry(lineNumber, columnNumber) { - } - - /** - * @param {string} sourceURL - * @param {number} lineNumber - * @param {number} columnNumber - * @return {?SourceMapEntry} - */ - sourceLineMapping(sourceURL, lineNumber, columnNumber) { - } - - /** - * @return {!Array} - */ - mappings() { - } - - dispose() { - } -} - -/** - * @unrestricted - */ -class SourceMapV3 { - constructor() { - /** @type {number} */ this.version; - /** @type {string|undefined} */ this.file; - /** @type {!Array.} */ this.sources; - /** @type {!Array.|undefined} */ this.sections; - /** @type {string} */ this.mappings; - /** @type {string|undefined} */ this.sourceRoot; - /** @type {!Array.|undefined} */ this.names; - } -} - -/** - * @unrestricted - */ -SourceMapV3.Section = class { - constructor() { - /** @type {!SourceMapV3} */ this.map; - /** @type {!SourceMapV3.Offset} */ this.offset; - } -}; - -/** - * @unrestricted - */ -SourceMapV3.Offset = class { - constructor() { - /** @type {number} */ this.line; - /** @type {number} */ this.column; - } -}; - -/** - * @unrestricted - */ -class SourceMapEntry { - /** - * @param {number} lineNumber - * @param {number} columnNumber - * @param {string=} sourceURL - * @param {number=} sourceLineNumber - * @param {number=} sourceColumnNumber - * @param {string=} name - */ - constructor(lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber, name) { - this.lineNumber = lineNumber; - this.columnNumber = columnNumber; - this.sourceURL = sourceURL; - this.sourceLineNumber = sourceLineNumber; - this.sourceColumnNumber = sourceColumnNumber; - this.name = name; - } - - /** - * @param {!SourceMapEntry} entry1 - * @param {!SourceMapEntry} entry2 - * @return {number} - */ - static compare(entry1, entry2) { - if (entry1.lineNumber !== entry2.lineNumber) { - return entry1.lineNumber - entry2.lineNumber; - } - return entry1.columnNumber - entry2.columnNumber; - } -} - -/** - * @implements {SourceMap} - * @unrestricted - */ -class TextSourceMap { - /** - * Implements Source Map V3 model. See https://github.com/google/closure-compiler/wiki/Source-Maps - * for format description. - * @param {string} compiledURL - * @param {string} sourceMappingURL - * @param {!SourceMapV3} payload - */ - constructor(compiledURL, sourceMappingURL, payload) { - if (!TextSourceMap._base64Map) { - const base64Digits = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - TextSourceMap._base64Map = {}; - for (let i = 0; i < base64Digits.length; ++i) { - TextSourceMap._base64Map[base64Digits.charAt(i)] = i; - } - } - - this._json = payload; - this._compiledURL = compiledURL; - this._sourceMappingURL = sourceMappingURL; - this._baseURL = sourceMappingURL.startsWith('data:') ? compiledURL : sourceMappingURL; - - /** @type {?Array} */ - this._mappings = null; - /** @type {!Map} */ - this._sourceInfos = new Map(); - if (this._json.sections) { - const sectionWithURL = !!this._json.sections.find(section => !!section.url); - if (sectionWithURL) { - cosole.warn(`SourceMap "${sourceMappingURL}" contains unsupported "URL" field in one of its sections.`); - } - } - this._eachSection(this._parseSources.bind(this)); - } - - /** - * @override - * @return {string} - */ - compiledURL() { - return this._compiledURL; - } - - /** - * @override - * @return {string} - */ - url() { - return this._sourceMappingURL; - } - - /** - * @override - * @return {!Array.} - */ - sourceURLs() { - return Array.from(this._sourceInfos.keys()); - } - - /** - * @override - * @param {string} sourceURL - * @return {?string} - */ - embeddedContentByURL(sourceURL) { - if (!this._sourceInfos.has(sourceURL)) { - return null; - } - return this._sourceInfos.get(sourceURL).content; - } - - /** - * @override - * @param {number} lineNumber in compiled resource - * @param {number} columnNumber in compiled resource - * @return {?SourceMapEntry} - */ - findEntry(lineNumber, columnNumber) { - const mappings = this.mappings(); - const index = upperBound(mappings, undefined, (unused, entry) => lineNumber - entry.lineNumber || columnNumber - entry.columnNumber); - return index ? mappings[index - 1] : null; - } - - /** - * @override - * @return {!Array} - */ - mappings() { - if (this._mappings === null) { - this._mappings = []; - this._eachSection(this._parseMap.bind(this)); - this._json = null; - } - return /** @type {!Array} */ (this._mappings); - } - - /** - * @param {function(!SourceMapV3, number, number)} callback - */ - _eachSection(callback) { - if (!this._json.sections) { - callback(this._json, 0, 0); - return; - } - for (const section of this._json.sections) { - callback(section.map, section.offset.line, section.offset.column); - } - } - - /** - * @param {!SourceMapV3} sourceMap - */ - _parseSources(sourceMap) { - const sourcesList = []; - let sourceRoot = sourceMap.sourceRoot || ''; - if (sourceRoot && !sourceRoot.endsWith('/')) { - sourceRoot += '/'; - } - for (let i = 0; i < sourceMap.sources.length; ++i) { - const href = sourceRoot + sourceMap.sources[i]; - let url = path.resolve(path.dirname(this._baseURL), href); - const source = sourceMap.sourcesContent && sourceMap.sourcesContent[i]; - if (url === this._compiledURL && source) { - url += '? [sm]'; - } - this._sourceInfos.set(url, new TextSourceMap.SourceInfo(source, null)); - sourcesList.push(url); - } - sourceMap[TextSourceMap._sourcesListSymbol] = sourcesList; - } - - /** - * @param {!SourceMapV3} map - * @param {number} lineNumber - * @param {number} columnNumber - */ - _parseMap(map, lineNumber, columnNumber) { - let sourceIndex = 0; - let sourceLineNumber = 0; - let sourceColumnNumber = 0; - let nameIndex = 0; - const sources = map[TextSourceMap._sourcesListSymbol]; - const names = map.names || []; - const stringCharIterator = new TextSourceMap.StringCharIterator(map.mappings); - let sourceURL = sources[sourceIndex]; - - while (true) { - if (stringCharIterator.peek() === ',') { - stringCharIterator.next(); - } else { - while (stringCharIterator.peek() === ';') { - lineNumber += 1; - columnNumber = 0; - stringCharIterator.next(); - } - if (!stringCharIterator.hasNext()) { - break; - } - } - - columnNumber += this._decodeVLQ(stringCharIterator); - if (!stringCharIterator.hasNext() || this._isSeparator(stringCharIterator.peek())) { - this._mappings.push(new SourceMapEntry(lineNumber, columnNumber)); - continue; - } - - const sourceIndexDelta = this._decodeVLQ(stringCharIterator); - if (sourceIndexDelta) { - sourceIndex += sourceIndexDelta; - sourceURL = sources[sourceIndex]; - } - sourceLineNumber += this._decodeVLQ(stringCharIterator); - sourceColumnNumber += this._decodeVLQ(stringCharIterator); - - if (!stringCharIterator.hasNext() || this._isSeparator(stringCharIterator.peek())) { - this._mappings.push( - new SourceMapEntry(lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber)); - continue; - } - - nameIndex += this._decodeVLQ(stringCharIterator); - this._mappings.push(new SourceMapEntry( - lineNumber, columnNumber, sourceURL, sourceLineNumber, sourceColumnNumber, names[nameIndex])); - } - - // As per spec, mappings are not necessarily sorted. - this._mappings.sort(SourceMapEntry.compare); - } - - /** - * @param {string} char - * @return {boolean} - */ - _isSeparator(char) { - return char === ',' || char === ';'; - } - - /** - * @param {!TextSourceMap.StringCharIterator} stringCharIterator - * @return {number} - */ - _decodeVLQ(stringCharIterator) { - // Read unsigned value. - let result = 0; - let shift = 0; - let digit; - do { - digit = TextSourceMap._base64Map[stringCharIterator.next()]; - result += (digit & TextSourceMap._VLQ_BASE_MASK) << shift; - shift += TextSourceMap._VLQ_BASE_SHIFT; - } while (digit & TextSourceMap._VLQ_CONTINUATION_MASK); - - // Fix the sign. - const negative = result & 1; - result >>= 1; - return negative ? -result : result; - } - - /** - * @override - */ - dispose() { - } -} - -TextSourceMap._VLQ_BASE_SHIFT = 5; -TextSourceMap._VLQ_BASE_MASK = (1 << 5) - 1; -TextSourceMap._VLQ_CONTINUATION_MASK = 1 << 5; - -/** - * @unrestricted - */ -TextSourceMap.StringCharIterator = class { - /** - * @param {string} string - */ - constructor(string) { - this._string = string; - this._position = 0; - } - - /** - * @return {string} - */ - next() { - return this._string.charAt(this._position++); - } - - /** - * @return {string} - */ - peek() { - return this._string.charAt(this._position); - } - - /** - * @return {boolean} - */ - hasNext() { - return this._position < this._string.length; - } -}; - -/** - * @unrestricted - */ -TextSourceMap.SourceInfo = class { - /** - * @param {?string} content - * @param {?Array} reverseMappings - */ - constructor(content, reverseMappings) { - this.content = content; - this.reverseMappings = reverseMappings; - } -}; - -TextSourceMap._sourcesListSymbol = Symbol('sourcesList'); - -module.exports = {TextSourceMap}; diff --git a/utils/testrunner/SourceMapSupport.js b/utils/testrunner/SourceMapSupport.js deleted file mode 100644 index 2cabccc005..0000000000 --- a/utils/testrunner/SourceMapSupport.js +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const fs = require('fs'); -const path = require('path'); -const {TextSourceMap} = require('./SourceMap'); -const util = require('util'); - -const readFileAsync = util.promisify(fs.readFile.bind(fs)); - -class SourceMapSupport { - constructor() { - this._sourceMapPromises = new Map(); - } - - async rewriteStackTraceWithSourceMaps(error) { - if (!error.stack || typeof error.stack !== 'string') - return; - const stackFrames = error.stack.split('\n'); - for (let i = 0; i < stackFrames.length; ++i) { - const stackFrame = stackFrames[i]; - - let match = stackFrame.match(/\((.*):(\d+):(\d+)\)$/); - if (!match) - match = stackFrame.match(/^\s*at (.*):(\d+):(\d+)$/); - if (!match) - continue; - const filePath = match[1]; - const sourceMap = await this._maybeLoadSourceMapForPath(filePath); - if (!sourceMap) - continue; - const compiledLineNumber = parseInt(match[2], 10); - const compiledColumnNumber = parseInt(match[3], 10); - if (isNaN(compiledLineNumber) || isNaN(compiledColumnNumber)) - continue; - const entry = sourceMap.findEntry(compiledLineNumber, compiledColumnNumber); - if (!entry) - continue; - stackFrames[i] = stackFrame.replace(filePath + ':' + compiledLineNumber + ':' + compiledColumnNumber, entry.sourceURL + ':' + entry.sourceLineNumber + ':' + entry.sourceColumnNumber); - } - error.stack = stackFrames.join('\n'); - } - - async _maybeLoadSourceMapForPath(filePath) { - let sourceMapPromise = this._sourceMapPromises.get(filePath); - if (sourceMapPromise === undefined) { - sourceMapPromise = this._loadSourceMapForPath(filePath); - this._sourceMapPromises.set(filePath, sourceMapPromise); - } - return sourceMapPromise; - } - - async _loadSourceMapForPath(filePath) { - try { - const fileContent = await readFileAsync(filePath, 'utf8'); - const magicCommentLine = fileContent.trim().split('\n').pop().trim(); - const magicCommentMatch = magicCommentLine.match('^//#\\s*sourceMappingURL\\s*=(.*)$'); - if (!magicCommentMatch) - return null; - const sourceMappingURL = magicCommentMatch[1].trim(); - - const sourceMapPath = path.resolve(path.dirname(filePath), sourceMappingURL); - const json = JSON.parse(await readFileAsync(sourceMapPath, 'utf8')); - return new TextSourceMap(filePath, sourceMapPath, json); - } catch(e) { - return null; - } - } -} - -module.exports = {SourceMapSupport}; diff --git a/utils/testrunner/Test.js b/utils/testrunner/Test.js deleted file mode 100644 index bd528180b1..0000000000 --- a/utils/testrunner/Test.js +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const Location = require('./Location'); - -const TestExpectation = { - Ok: 'ok', - Fail: 'fail', -}; - -class Test { - constructor(suite, name, callback, location) { - this._suite = suite; - this._name = name; - this._fullName = (suite.fullName() + ' ' + name).trim(); - this._skipped = false; - this._expectation = TestExpectation.Ok; - this._body = callback; - this._location = location; - this._timeout = 100000000; - this._environments = []; - this.Expectations = { ...TestExpectation }; - } - - titles() { - if (!this._name) - return this._suite.titles(); - return [...this._suite.titles(), this._name]; - } - - suite() { - return this._suite; - } - - name() { - return this._name; - } - - fullName() { - return this._fullName; - } - - location() { - return this._location; - } - - body() { - return this._body; - } - - skipped() { - return this._skipped; - } - - setSkipped(skipped) { - this._skipped = skipped; - return this; - } - - timeout() { - return this._timeout; - } - - setTimeout(timeout) { - this._timeout = timeout; - return this; - } - - expectation() { - return this._expectation; - } - - setExpectation(expectation) { - this._expectation = expectation; - return this; - } - - addEnvironment(environment) { - this._environments.push(environment); - return this; - } - - removeEnvironment(environment) { - const index = this._environments.indexOf(environment); - if (index === -1) - throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); - this._environments.splice(index, 1); - return this; - } -} - -class Suite { - constructor(parentSuite, name, location) { - this._parentSuite = parentSuite; - this._name = name; - const fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); - this._fullName = fullName; - this._location = location; - this._skipped = false; - this._expectation = TestExpectation.Ok; - - this._defaultEnvironment = { - name() { return fullName; }, - }; - - this._environments = [this._defaultEnvironment]; - this.Expectations = { ...TestExpectation }; - } - - _addHook(name, callback) { - if (this._defaultEnvironment[name]) - throw new Error(`ERROR: cannot re-assign hook "${name}" for suite "${this._fullName}"`); - this._defaultEnvironment[name] = callback; - } - - beforeEach(callback) { this._addHook('beforeEach', callback); } - afterEach(callback) { this._addHook('afterEach', callback); } - beforeAll(callback) { this._addHook('beforeAll', callback); } - afterAll(callback) { this._addHook('afterAll', callback); } - globalSetup(callback) { this._addHook('globalSetup', callback); } - globalTeardown(callback) { this._addHook('globalTeardown', callback); } - - titles() { - if (!this._parentSuite) - return this._name ? [this._name] : []; - return this._name ? [...this._parentSuite.titles(), this._name] : this._parentSuite.titles(); - } - - parentSuite() { return this._parentSuite; } - - name() { return this._name; } - - fullName() { return this._fullName; } - - skipped() { return this._skipped; } - - setSkipped(skipped) { - this._skipped = skipped; - return this; - } - - location() { return this._location; } - - expectation() { return this._expectation; } - - setExpectation(expectation) { - this._expectation = expectation; - return this; - } - - addEnvironment(environment) { - this._environments.push(environment); - return this; - } - - removeEnvironment(environment) { - const index = this._environments.indexOf(environment); - if (index === -1) - throw new Error(`Environment "${environment.name()}" cannot be removed because it was not added to the suite "${this.fullName()}"`); - this._environments.splice(index, 1); - return this; - } -} - -module.exports = { TestExpectation, Test, Suite }; diff --git a/utils/testrunner/TestCollector.js b/utils/testrunner/TestCollector.js deleted file mode 100644 index ec52299403..0000000000 --- a/utils/testrunner/TestCollector.js +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const Location = require('./Location'); -const { Test, Suite } = require('./Test'); -const { TestRun } = require('./TestRunner'); - -class FocusedFilter { - constructor() { - this._focusedTests = new Set(); - this._focusedSuites = new Set(); - this._focusedFilePaths = new Set(); - } - - focusTest(test) { this._focusedTests.add(test); } - focusSuite(suite) { this._focusedSuites.add(suite); } - focusFilePath(filePath) { this._focusedFilePaths.add(filePath); } - - hasFocusedTestsOrSuitesOrFiles() { - return !!this._focusedTests.size || !!this._focusedSuites.size || !!this._focusedFilePaths.size; - } - - focusedTests(tests) { - return tests.filter(test => this._focusedTests.has(test)); - } - - focusedSuites(suites) { - return suites.filter(suite => this._focusedSuites.has(suite)); - } - - focusedFilePaths(filePaths) { - return filePaths.filter(filePath => this._focusedFilePaths.has(filePath)); - } - - filter(tests) { - if (!this.hasFocusedTestsOrSuitesOrFiles()) - return tests; - - const ignoredSuites = new Set(); - const ignoredFilePaths = new Set(); - for (const test of tests) { - if (this._focusedTests.has(test)) { - // Focused tests should be run even if skipped. - test.setSkipped(false); - // TODO: remove next line once we run failing tests. - test.setExpectation(test.Expectations.Ok); - ignoredFilePaths.add(test.location().filePath()); - } - for (let suite = test.suite(); suite; suite = suite.parentSuite()) { - if (this._focusedSuites.has(suite)) { - // Focused suites should be run even if skipped. - suite.setSkipped(false); - // TODO: remove next line once we run failing tests. - suite.setExpectation(suite.Expectations.Ok); - } - // Mark parent suites of focused tests as ignored. - if (this._focusedTests.has(test)) - ignoredSuites.add(suite); - } - } - // Pick all tests that are focused or belong to focused suites. - const result = []; - for (const test of tests) { - let focused = this._focusedTests.has(test) || (this._focusedFilePaths.has(test.location().filePath()) && !ignoredFilePaths.has(test.location().filePath())); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - focused = focused || (this._focusedSuites.has(suite) && !ignoredSuites.has(suite)); - if (focused) - result.push(test); - } - return result; - } -} - -class Repeater { - constructor() { - this._repeatCount = new Map(); - } - - repeat(testOrSuite, count) { - this._repeatCount.set(testOrSuite, count); - } - - _get(testOrSuite) { - const repeat = this._repeatCount.get(testOrSuite); - return repeat === undefined ? 1 : repeat; - } - - createTestRuns(tests) { - const suiteToChildren = new Map(); - const rootSuites = new Set(); - for (const test of tests) { - let children = suiteToChildren.get(test.suite()); - if (!children) { - children = new Set(); - suiteToChildren.set(test.suite(), children); - } - children.add(test); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) { - let children = suiteToChildren.get(suite.parentSuite()); - if (!children) { - children = new Set(); - suiteToChildren.set(suite.parentSuite(), children); - } - children.add(suite); - // Add root suites. - if (!suite.parentSuite()) - rootSuites.add(suite); - } - } - - const collectTests = (testOrSuite) => { - const testOrder = []; - if (testOrSuite instanceof Test) { - testOrder.push(testOrSuite); - } else { - for (const child of suiteToChildren.get(testOrSuite)) - testOrder.push(...collectTests(child)); - } - const repeat = this._repeatCount.has(testOrSuite) ? this._repeatCount.get(testOrSuite) : 1; - const result = []; - for (let i = 0; i < repeat; ++i) - result.push(...testOrder); - return result; - } - - const testOrder = []; - for (const rootSuite of rootSuites) - testOrder.push(...collectTests(rootSuite)); - return testOrder.map(test => new TestRun(test)); - - } -} - -function specBuilder(modifiers, attributes, specCallback) { - function builder(specs) { - return new Proxy((...args) => specCallback(specs, ...args), { - get: (obj, prop) => { - if (modifiers.has(prop)) - return (...args) => builder([...specs, { callback: modifiers.get(prop), args }]); - if (attributes.has(prop)) - return builder([...specs, { callback: attributes.get(prop), args: [] }]); - return obj[prop]; - }, - }); - } - return builder([]); -} - -class TestCollector { - constructor(options = {}) { - let { timeout = 10 * 1000 } = options; - if (timeout === 0) - timeout = 100000000; // Inifinite timeout. - - this._tests = []; - this._suites = []; - this._suiteModifiers = new Map(); - this._suiteAttributes = new Map(); - this._testModifiers = new Map(); - this._testAttributes = new Map(); - this._testCallbackWrappers = []; - this._api = {}; - - this._currentSuite = new Suite(null, '', new Location()); - this._rootSuite = this._currentSuite; - - this._api.describe = specBuilder(this._suiteModifiers, this._suiteAttributes, (specs, name, suiteCallback, ...suiteArgs) => { - const location = Location.getCallerLocation(); - const suite = new Suite(this._currentSuite, name, location); - for (const { callback, args } of specs) - callback(suite, ...args); - this._currentSuite = suite; - suiteCallback(...suiteArgs); - this._suites.push(suite); - this._currentSuite = suite.parentSuite(); - }); - this._api.it = specBuilder(this._testModifiers, this._testAttributes, (specs, name, testCallback) => { - const location = Location.getCallerLocation(); - for (const wrapper of this._testCallbackWrappers) - testCallback = wrapper(testCallback); - const test = new Test(this._currentSuite, name, testCallback, location); - test.setTimeout(timeout); - for (const { callback, args } of specs) - callback(test, ...args); - this._tests.push(test); - }); - this._api.beforeAll = callback => this._currentSuite.beforeAll(callback); - this._api.beforeEach = callback => this._currentSuite.beforeEach(callback); - this._api.afterAll = callback => this._currentSuite.afterAll(callback); - this._api.afterEach = callback => this._currentSuite.afterEach(callback); - this._api.globalSetup = callback => this._currentSuite.globalSetup(callback); - this._api.globalTeardown = callback => this._currentSuite.globalTeardown(callback); - } - - useEnvironment(environment) { - return this._currentSuite.addEnvironment(environment); - } - - addTestCallbackWrapper(wrapper) { - this._testCallbackWrappers.push(wrapper); - } - - addTestModifier(name, callback) { - this._testModifiers.set(name, callback); - } - - addTestAttribute(name, callback) { - this._testAttributes.set(name, callback); - } - - addSuiteModifier(name, callback) { - this._suiteModifiers.set(name, callback); - } - - addSuiteAttribute(name, callback) { - this._suiteAttributes.set(name, callback); - } - - api() { - return this._api; - } - - tests() { - return this._tests; - } - - suites() { - return this._suites; - } - - filePaths() { - const filePaths = new Set(); - for (const test of this._tests) - filePaths.add(test.location().filePath()); - for (const suite of this._suites) - filePaths.add(suite.location().filePath()); - return [...filePaths]; - } - - rootSuite() { - return this._rootSuite; - } -} - -module.exports = { TestCollector, specBuilder, FocusedFilter, Repeater }; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js deleted file mode 100644 index f790d0231d..0000000000 --- a/utils/testrunner/TestRunner.js +++ /dev/null @@ -1,581 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * Modifications copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { SourceMapSupport } = require('./SourceMapSupport'); -const debug = require('debug'); -const { TestExpectation } = require('./Test'); - -const TimeoutError = new Error('Timeout'); -const TerminatedError = new Error('Terminated'); - -function runUserCallback(callback, timeout, args) { - let terminateCallback; - let timeoutId; - const promise = Promise.race([ - Promise.resolve().then(callback.bind(null, ...args)).then(() => null).catch(e => e), - new Promise(resolve => { - timeoutId = setTimeout(resolve.bind(null, TimeoutError), timeout); - }), - new Promise(resolve => terminateCallback = resolve), - ]).catch(e => e).finally(() => clearTimeout(timeoutId)); - const terminate = () => terminateCallback(TerminatedError); - return { promise, terminate }; -} - -const TestResult = { - Ok: 'ok', - MarkedAsFailing: 'markedAsFailing', // User marked as failed - Skipped: 'skipped', // User marked as skipped - Failed: 'failed', // Exception happened during running - TimedOut: 'timedout', // Timeout Exceeded while running - Terminated: 'terminated', // Execution terminated - Crashed: 'crashed', // If testrunner crashed due to this test -}; - -function isEmptyEnvironment(env) { - return !env.afterEach && !env.afterAll && !env.beforeEach && !env.beforeAll && - !env.globalSetup && !env.globalTeardown; -} - -class TestRun { - constructor(test) { - this._test = test; - this._result = null; - this._error = null; - this._startTimestamp = 0; - this._endTimestamp = 0; - this._workerId = null; - this._output = []; - - this._environments = test._environments.filter(env => !isEmptyEnvironment(env)).reverse(); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - this._environments.push(...suite._environments.filter(env => !isEmptyEnvironment(env)).reverse()); - this._environments.reverse(); - } - - finished() { - return this._result !== null && this._result !== 'running'; - } - - isFailure() { - return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed; - } - - ok() { - return this._result === TestResult.Ok; - } - - result() { - return this._result; - } - - error() { - return this._error; - } - - duration() { - return this._endTimestamp - this._startTimestamp; - } - - test() { - return this._test; - } - - workerId() { - return this._workerId; - } - - log(log) { - this._output.push(log); - } - - output() { - return this._output; - } -} - -class Result { - constructor() { - this.result = TestResult.Ok; - this.exitCode = 0; - this.message = ''; - this.errors = []; - this.runs = []; - } - - setResult(result, message) { - if (!this.ok()) - return; - this.result = result; - this.message = message || ''; - if (result === TestResult.Ok) - this.exitCode = 0; - else if (result === TestResult.Terminated) - this.exitCode = 130; - else if (result === TestResult.Crashed) - this.exitCode = 2; - else - this.exitCode = 1; - } - - addError(message, error, worker) { - const data = { message, error, runs: [] }; - if (worker) - data.runs = worker._runs.slice(); - this.errors.push(data); - } - - ok() { - return this.result === TestResult.Ok; - } -} - -class TestWorker { - constructor(testRunner, hookRunner, workerId, parallelIndex) { - this._testRunner = testRunner; - this._hookRunner = hookRunner; - this._state = { parallelIndex }; - this._environmentStack = []; - this._terminating = false; - this._workerId = workerId; - this._runningTestTerminate = null; - this._runs = []; - } - - terminate(terminateHooks) { - this._terminating = true; - if (this._runningTestTerminate) - this._runningTestTerminate(); - this._hookRunner.terminateWorker(this); - } - - _markTerminated(testRun) { - if (!this._terminating) - return false; - testRun._result = TestResult.Terminated; - return true; - } - - async run(testRun) { - this._runs.push(testRun); - - const test = testRun.test(); - let skipped = test.skipped(); - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - skipped = skipped || suite.skipped(); - if (skipped) { - await this._willStartTestRun(testRun); - testRun._result = TestResult.Skipped; - await this._didFinishTestRun(testRun); - return; - } - - let expectedToFail = test.expectation() === TestExpectation.Fail; - for (let suite = test.suite(); suite; suite = suite.parentSuite()) - expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail); - if (expectedToFail) { - await this._willStartTestRun(testRun); - testRun._result = TestResult.MarkedAsFailing; - await this._didFinishTestRun(testRun); - return; - } - - const environmentStack = testRun._environments; - let common = 0; - while (common < environmentStack.length && this._environmentStack[common] === environmentStack[common]) - common++; - - while (this._environmentStack.length > common) { - if (this._markTerminated(testRun)) - return; - const environment = this._environmentStack.pop(); - if (!await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, testRun)) - return; - if (!await this._hookRunner.maybeRunGlobalTeardown(environment)) - return; - } - while (this._environmentStack.length < environmentStack.length) { - if (this._markTerminated(testRun)) - return; - const environment = environmentStack[this._environmentStack.length]; - this._environmentStack.push(environment); - if (!await this._hookRunner.maybeRunGlobalSetup(environment)) - return; - if (!await this._hookRunner.runHook(environment, 'beforeAll', [this._state], this, testRun)) - return; - } - - if (this._markTerminated(testRun)) - return; - - // From this point till the end, we have to run all hooks - // no matter what happens. - - await this._willStartTestRun(testRun); - for (const environment of this._environmentStack) { - await this._hookRunner.runHook(environment, 'beforeEach', [this._state, testRun], this, testRun); - } - - if (!testRun._error && !this._markTerminated(testRun)) { - await this._willStartTestBody(testRun); - const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, testRun]); - this._runningTestTerminate = terminate; - testRun._error = await promise; - this._runningTestTerminate = null; - if (testRun._error && testRun._error.stack) - await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error); - if (!testRun._error) - testRun._result = TestResult.Ok; - else if (testRun._error === TimeoutError) - testRun._result = TestResult.TimedOut; - else if (testRun._error === TerminatedError) - testRun._result = TestResult.Terminated; - else - testRun._result = TestResult.Failed; - await this._didFinishTestBody(testRun); - } - - for (const environment of this._environmentStack.slice().reverse()) - await this._hookRunner.runHook(environment, 'afterEach', [this._state, testRun], this, testRun); - await this._didFinishTestRun(testRun); - } - - async _willStartTestRun(testRun) { - testRun._startTimestamp = Date.now(); - testRun._workerId = this._workerId; - await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunStarted, [testRun]); - } - - async _didFinishTestRun(testRun) { - testRun._endTimestamp = Date.now(); - testRun._workerId = this._workerId; - - this._hookRunner.markFinishedTestRun(testRun); - await this._testRunner._runDelegateCallback(this._testRunner._delegate.onTestRunFinished, [testRun]); - } - - async _willStartTestBody(testRun) { - debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location()})`); - } - - async _didFinishTestBody(testRun) { - debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location()})`); - } - - async shutdown() { - while (this._environmentStack.length > 0) { - const environment = this._environmentStack.pop(); - await this._hookRunner.runHook(environment, 'afterAll', [this._state], this, null); - await this._hookRunner.maybeRunGlobalTeardown(environment); - } - } -} - -class HookRunner { - constructor(testRunner, testRuns) { - this._testRunner = testRunner; - this._runningHookTerminations = new Map(); - - this._environmentToGlobalState = new Map(); - for (const testRun of testRuns) { - for (const env of testRun._environments) { - let globalState = this._environmentToGlobalState.get(env); - if (!globalState) { - globalState = { - pendingTestRuns: new Set(), - globalSetupPromise: null, - globalTeardownPromise: null, - }; - this._environmentToGlobalState.set(env, globalState); - } - globalState.pendingTestRuns.add(testRun); - } - } - } - - terminateWorker(worker) { - let termination = this._runningHookTerminations.get(worker); - this._runningHookTerminations.delete(worker); - if (termination) - termination(); - } - - terminateAll() { - for (const termination of this._runningHookTerminations.values()) - termination(); - this._runningHookTerminations.clear(); - } - - markFinishedTestRun(testRun) { - for (const environment of testRun._environments) { - const globalState = this._environmentToGlobalState.get(environment); - globalState.pendingTestRuns.delete(testRun); - } - } - - async _runHookInternal(worker, testRun, hook, fullName, hookArgs = []) { - await this._willStartHook(worker, testRun, hook, fullName); - const timeout = this._testRunner._hookTimeout; - const { promise, terminate } = runUserCallback(hook.body, timeout, hookArgs); - this._runningHookTerminations.set(worker, terminate); - let error = await promise; - this._runningHookTerminations.delete(worker); - - if (error) { - if (testRun && testRun._result !== TestResult.Terminated) { - // Prefer terminated result over any hook failures. - testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed; - } - let message; - if (error === TimeoutError) { - message = `Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; - error = null; - } else if (error === TerminatedError) { - // Do not report termination details - it's just noise. - message = ''; - error = null; - } else { - if (error.stack) - await this._testRunner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); - message = `FAILED while running "${hook.name}" in suite "${fullName}": `; - } - await this._didFailHook(worker, testRun, hook, fullName, message, error); - if (testRun) - testRun._error = error; - return false; - } - - await this._didCompleteHook(worker, testRun, hook, fullName); - return true; - } - - async runHook(environment, hookName, hookArgs, worker = null, testRun = null) { - const hookBody = environment[hookName]; - if (!hookBody) - return true; - const envName = environment.name ? environment.name() : environment.constructor.name; - return await this._runHookInternal(worker, testRun, {name: hookName, body: hookBody.bind(environment)}, envName, hookArgs); - } - - async maybeRunGlobalSetup(environment) { - const globalState = this._environmentToGlobalState.get(environment); - if (!globalState.globalSetupPromise) - globalState.globalSetupPromise = this.runHook(environment, 'globalSetup', []); - if (!await globalState.globalSetupPromise) { - await this._testRunner._terminate(TestResult.Crashed, 'Global setup failed!', false, null); - return false; - } - return true; - } - - async maybeRunGlobalTeardown(environment) { - const globalState = this._environmentToGlobalState.get(environment); - if (!globalState.globalTeardownPromise) { - if (!globalState.pendingTestRuns.size || (this._testRunner._terminating && globalState.globalSetupPromise)) - globalState.globalTeardownPromise = this.runHook(environment, 'globalTeardown', []); - } - if (!globalState.globalTeardownPromise) - return true; - if (!await globalState.globalTeardownPromise) { - await this._testRunner._terminate(TestResult.Crashed, 'Global teardown failed!', false, null); - return false; - } - return true; - } - - async _willStartHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" started for "${testRun ? testRun.test().fullName() : ''}"`); - } - - async _didFailHook(worker, testRun, hook, fullName, message, error) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" FAILED for "${testRun ? testRun.test().fullName() : ''}"`); - if (message) - this._testRunner._result.addError(message, error, worker); - this._testRunner._result.setResult(TestResult.Crashed, message); - } - - async _didCompleteHook(worker, testRun, hook, fullName) { - debug('testrunner:hook')(`${workerName(worker)} "${fullName}.${hook.name}" OK for "${testRun ? testRun.test().fullName() : ''}"`); - } -} - -function workerName(worker) { - return worker ? `` : `<_global_>`; -} - -class TestRunner { - constructor() { - this._sourceMapSupport = new SourceMapSupport(); - this._nextWorkerId = 1; - this._workers = []; - this._terminating = false; - this._result = null; - this._hookRunner = null; - } - - async _runDelegateCallback(callback, args) { - let { promise, terminate } = runUserCallback(callback, this._hookTimeout, args); - // Note: we do not terminate the delegate to keep reporting even when terminating. - const e = await promise; - if (e) { - debug('testrunner')(`Error while running delegate method: ${e}`); - const { message, error } = this._toError('INTERNAL ERROR', e); - this._terminate(TestResult.Crashed, message, false, error); - } - } - - _toError(message, error) { - if (!(error instanceof Error)) { - message += ': ' + error; - error = new Error(); - error.stack = ''; - } - return { message, error }; - } - - async run(testRuns, options = {}) { - const { - parallel = 1, - breakOnFailure = false, - hookTimeout = 10 * 1000, - totalTimeout = 0, - onStarted = async (testRuns) => {}, - onFinished = async (result) => {}, - onTestRunStarted = async (testRun) => {}, - onTestRunFinished = async (testRun) => {}, - } = options; - this._breakOnFailure = breakOnFailure; - this._hookTimeout = hookTimeout === 0 ? 100000000 : hookTimeout; - this._delegate = { - onStarted, - onFinished, - onTestRunStarted, - onTestRunFinished - }; - - this._result = new Result(); - this._result.runs = testRuns; - - const terminationPromises = []; - const handleSIGINT = () => this._terminate(TestResult.Terminated, 'SIGINT received', false, null); - const handleSIGHUP = () => this._terminate(TestResult.Terminated, 'SIGHUP received', false, null); - const handleSIGTERM = () => this._terminate(TestResult.Terminated, 'SIGTERM received', true, null); - const handleRejection = e => { - const { message, error } = this._toError('UNHANDLED PROMISE REJECTION', e); - terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error)); - }; - const handleException = e => { - const { message, error } = this._toError('UNHANDLED ERROR', e); - terminationPromises.push(this._terminate(TestResult.Crashed, message, false, error)); - }; - process.on('SIGINT', handleSIGINT); - process.on('SIGHUP', handleSIGHUP); - process.on('SIGTERM', handleSIGTERM); - process.on('unhandledRejection', handleRejection); - process.on('uncaughtException', handleException); - - let timeoutId; - if (totalTimeout) { - timeoutId = setTimeout(() => { - terminationPromises.push(this._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */)); - }, totalTimeout); - } - await this._runDelegateCallback(this._delegate.onStarted, [testRuns]); - - this._hookRunner = new HookRunner(this, testRuns); - - const workerCount = Math.min(parallel, testRuns.length); - const workerPromises = []; - for (let i = 0; i < workerCount; ++i) { - const initialTestRunIndex = i * Math.floor(testRuns.length / workerCount); - workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); - } - await Promise.all(workerPromises); - await Promise.all(terminationPromises); - - if (testRuns.some(run => run.isFailure())) - this._result.setResult(TestResult.Failed, ''); - - await this._runDelegateCallback(this._delegate.onFinished, [this._result]); - clearTimeout(timeoutId); - - process.removeListener('SIGINT', handleSIGINT); - process.removeListener('SIGHUP', handleSIGHUP); - process.removeListener('SIGTERM', handleSIGTERM); - process.removeListener('unhandledRejection', handleRejection); - process.removeListener('uncaughtException', handleException); - return this._result; - } - - async _runWorker(testRunIndex, testRuns, parallelIndex) { - let worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); - this._workers[parallelIndex] = worker; - while (!this._terminating) { - let skipped = 0; - while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) { - testRunIndex = (testRunIndex + 1) % testRuns.length; - skipped++; - } - const testRun = testRuns[testRunIndex]; - if (testRun._result !== null) { - // All tests have been run. - break; - } - - // Mark as running so that other workers do not run it again. - testRun._result = 'running'; - await worker.run(testRun); - if (testRun.isFailure()) { - // Something went wrong during test run, let's use a fresh worker. - await worker.shutdown(); - if (this._breakOnFailure) { - const message = `Terminating because a test has failed and |testRunner.breakOnFailure| is enabled`; - await this._terminate(TestResult.Terminated, message, false /* force */, null /* error */); - return; - } - worker = new TestWorker(this, this._hookRunner, this._nextWorkerId++, parallelIndex); - this._workers[parallelIndex] = worker; - } - } - await worker.shutdown(); - } - - async _terminate(result, message, force, error) { - debug('testrunner')(`TERMINATED result = ${result}, message = ${message}`); - this._terminating = true; - for (const worker of this._workers) - worker.terminate(force /* terminateHooks */); - if (this._hookRunner) - this._hookRunner.terminateAll(); - this._result.setResult(result, message); - if (this._result.message === 'SIGINT received' && message === 'SIGTERM received') - this._result.message = message; - if (error) { - if (error.stack) - await this._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); - this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null); - } - } - - async terminate() { - if (!this._result) - return; - await this._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', true /* force */, null /* error */); - } -} - -module.exports = { TestRunner, TestRun, TestResult, Result }; diff --git a/utils/testrunner/diffstyle.css b/utils/testrunner/diffstyle.css deleted file mode 100644 index c58f0e90a6..0000000000 --- a/utils/testrunner/diffstyle.css +++ /dev/null @@ -1,13 +0,0 @@ -body { - font-family: monospace; - white-space: pre; -} - -ins { - background-color: #9cffa0; - text-decoration: none; -} - -del { - background-color: #ff9e9e; -} diff --git a/utils/testrunner/examples/fail.js b/utils/testrunner/examples/fail.js deleted file mode 100644 index 6e2cd75ac4..0000000000 --- a/utils/testrunner/examples/fail.js +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter, Matchers} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); -const {expect} = new Matchers(); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('toBe', async (state) => { - expect(2 + 2).toBe(5); - }); - it('toBeFalsy', async (state) => { - expect(true).toBeFalsy(); - }); - it('toBeTruthy', async (state) => { - expect(false).toBeTruthy(); - }); - it('toBeGreaterThan', async (state) => { - expect(2).toBeGreaterThan(3); - }); - it('toBeNull', async (state) => { - expect(2).toBeNull(); - }); - it('toContain', async (state) => { - expect('asdf').toContain('e'); - }); - it('not.toContain', async (state) => { - expect('asdf').not.toContain('a'); - }); - it('toEqual', async (state) => { - expect([1,2,3]).toEqual([1,2,3,4]); - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/hookfail.js b/utils/testrunner/examples/hookfail.js deleted file mode 100644 index 038f58c47f..0000000000 --- a/utils/testrunner/examples/hookfail.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter, Matchers} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); -const {expect} = new Matchers(); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - beforeAll(() => { - expect(false).toBeTruthy(); - }); - it('test', async () => { - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/hooktimeout.js b/utils/testrunner/examples/hooktimeout.js deleted file mode 100644 index 525f2b7c8a..0000000000 --- a/utils/testrunner/examples/hooktimeout.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter, Matchers} = require('..'); - -const runner = new TestRunner({ timeout: 100 }); -const reporter = new Reporter(runner); -const {expect} = new Matchers(); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - beforeAll(async () => { - await new Promise(() => {}); - }); - it('something', async (state) => { - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/timeout.js b/utils/testrunner/examples/timeout.js deleted file mode 100644 index 0d451e971e..0000000000 --- a/utils/testrunner/examples/timeout.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner({ timeout: 100 }); -const reporter = new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('timeout', async (state) => { - await new Promise(() => {}); - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/unhandledpromiserejection.js b/utils/testrunner/examples/unhandledpromiserejection.js deleted file mode 100644 index 270bfcc422..0000000000 --- a/utils/testrunner/examples/unhandledpromiserejection.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('failure', async (state) => { - Promise.reject(new Error('fail!')); - }); - it('slow', async () => { - await new Promise(x => setTimeout(x, 1000)); - }); -}); - -runner.run(); diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js deleted file mode 100644 index 1cf2e5f40b..0000000000 --- a/utils/testrunner/index.js +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); -const { TestRunner, Result, TestResult } = require('./TestRunner'); -const { TestCollector, FocusedFilter, Repeater } = require('./TestCollector'); -const Reporter = require('./Reporter'); -const { Matchers } = require('./Matchers'); - -class DefaultTestRunner { - constructor(options = {}) { - const { - // Our options. - crashIfTestsAreFocusedOnCI = true, - exit = true, - reporter = true, - // Collector options. - timeout, - // Runner options. - parallel = 1, - breakOnFailure, - totalTimeout, - hookTimeout = timeout, - // Reporting options. - showSlowTests, - showMarkedAsFailingTests, - verbose, - summary, - lineBreak, - goldenPath, - outputPath, - } = options; - - this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; - this._exit = exit; - this._parallel = parallel; - this._breakOnFailure = breakOnFailure; - this._totalTimeout = totalTimeout; - this._hookTimeout = hookTimeout; - this._needReporter = reporter; - this._showSlowTests = showSlowTests; - this._showMarkedAsFailingTests = showMarkedAsFailingTests; - this._verbose = verbose; - this._summary = summary; - this._lineBreak = lineBreak; - - this._filter = new FocusedFilter(); - this._repeater = new Repeater(); - this._collector = new TestCollector({ timeout }); - - this._api = { - ...this._collector.api(), - expect: new Matchers({ goldenPath, outputPath }).expect, - }; - this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s)); - this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); - this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); - this._collector.addTestAttribute('only', t => this._filter.focusTest(t)); - this._collector.addTestAttribute('skip', t => t.setSkipped(true)); - this._collector.addTestAttribute('todo', t => t.setSkipped(true)); - this._collector.addTestAttribute('slow', t => t.setTimeout(t.timeout() * 3)); - this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); - this._api.fdescribe = this._api.describe.only; - this._api.xdescribe = this._api.describe.skip; - this._api.fit = this._api.it.only; - this._api.xit = this._api.it.skip; - } - - collector() { - return this._collector; - } - - api() { - return this._api; - } - - focusMatchingNameTests(fullNameRegex) { - const focusedTests = []; - for (const test of this._collector.tests()) { - if (fullNameRegex.test(test.fullName())) { - this._filter.focusTest(test); - focusedTests.push(test); - } - } - return focusedTests; - } - - focusMatchingFileName(filenameRegex) { - const focusedFilePaths = []; - for (const filePath of this._collector.filePaths()) { - if (filenameRegex.test(path.basename(filePath))) { - this._filter.focusFilePath(filePath); - focusedFilePaths.push(filePath); - } - } - return focusedFilePaths; - } - - repeatAll(repeatCount) { - this._repeater.repeat(this._collector.rootSuite(), repeatCount); - } - - async run() { - let reporter = null; - - if (this._needReporter) { - const reporterDelegate = { - focusedSuites: () => this._filter.focusedSuites(this._collector.suites()), - focusedTests: () => this._filter.focusedTests(this._collector.tests()), - focusedFilePaths: () => this._filter.focusedFilePaths(this._collector.filePaths()), - hasFocusedTestsOrSuitesOrFiles: () => this._filter.hasFocusedTestsOrSuitesOrFiles(), - parallel: () => this._parallel, - testCount: () => this._collector.tests().length, - }; - const reporterOptions = { - showSlowTests: this._showSlowTests, - showMarkedAsFailingTests: this._showMarkedAsFailingTests, - verbose: this._verbose, - summary: this._summary, - lineBreak: this._lineBreak, - }; - reporter = new Reporter(reporterDelegate, reporterOptions); - } - - if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this._filter.hasFocusedTestsOrSuitesOrFiles()) { - if (reporter) - await reporter.onStarted([]); - const result = new Result(); - result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); - if (reporter) - await reporter.onFinished(result); - if (this._exit) - process.exit(result.exitCode); - return result; - } - - const testRuns = this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); - const testRunner = new TestRunner(); - const result = await testRunner.run(testRuns, { - parallel: this._parallel, - breakOnFailure: this._breakOnFailure, - totalTimeout: this._totalTimeout, - hookTimeout: this._hookTimeout, - onStarted: (...args) => reporter && reporter.onStarted(...args), - onFinished: (...args) => reporter && reporter.onFinished(...args), - onTestRunStarted: (...args) => reporter && reporter.onTestRunStarted(...args), - onTestRunFinished: (...args) => reporter && reporter.onTestRunFinished(...args), - }); - if (this._exit) - process.exit(result.exitCode); - return result; - } -} - -module.exports = DefaultTestRunner; diff --git a/utils/testrunner/test/test.js b/utils/testrunner/test/test.js deleted file mode 100644 index 304b847967..0000000000 --- a/utils/testrunner/test/test.js +++ /dev/null @@ -1,4 +0,0 @@ -const TestRunner = require('..'); -const testRunner = new TestRunner(); -require('./testrunner.spec.js').addTests(testRunner.api()); -testRunner.run(); diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js deleted file mode 100644 index 3655c292ea..0000000000 --- a/utils/testrunner/test/testrunner.spec.js +++ /dev/null @@ -1,1052 +0,0 @@ -const { TestRunner } = require('../TestRunner'); -const { TestCollector, FocusedFilter, Repeater } = require('../TestCollector'); -const { TestExpectation, Environment } = require('../Test'); - -class Runner { - constructor(options = {}) { - this._options = options; - this._filter = new FocusedFilter(); - this._repeater = new Repeater(); - this._collector = new TestCollector(options); - this._collector.addSuiteAttribute('only', s => this._filter.focusSuite(s)); - this._collector.addTestAttribute('only', t => this._filter.focusTest(t)); - this._collector.addSuiteAttribute('skip', s => s.setSkipped(true)); - this._collector.addTestAttribute('skip', t => t.setSkipped(true)); - this._collector.addTestAttribute('fail', t => t.setExpectation(t.Expectations.Fail)); - this._collector.addSuiteModifier('repeat', (s, count) => this._repeater.repeat(s, count)); - this._collector.addTestModifier('repeat', (t, count) => this._repeater.repeat(t, count)); - - const api = this._collector.api(); - for (const [key, value] of Object.entries(api)) - this[key] = value; - this.fdescribe = api.describe.only; - this.xdescribe = api.describe.skip; - this.fit = api.it.only; - this.xit = api.it.skip; - this.Expectations = { ...TestExpectation }; - } - - createTestRuns() { - return this._repeater.createTestRuns(this._filter.filter(this._collector.tests())); - } - - run() { - this._testRunner = new TestRunner(); - return this._testRunner.run(this.createTestRuns(), this._options); - } - - tests() { - return this._collector.tests(); - } - - parallel() { - return this._options.parallel || 1; - } - - focusedTests() { - return this._filter.focusedTests(this._collector.tests()); - } - - suites() { - return this._collector.suites(); - } - - focusedSuites() { - return this._filter.focusedSuites(this._collector.suites()); - } - - terminate() { - this._testRunner.terminate(); - } -} - -module.exports.addTests = function({describe, fdescribe, xdescribe, it, xit, fit, expect}) { - describe('TestRunner.it', () => { - it('should declare a test', async() => { - const t = new Runner(); - t.it('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name()).toBe('uno'); - expect(test.fullName()).toBe('uno'); - expect(test.skipped()).toBe(false); - expect(test.location().filePath()).toEqual(__filename); - expect(test.location().fileName()).toEqual('testrunner.spec.js'); - expect(test.location().lineNumber()).toBeTruthy(); - expect(test.location().columnNumber()).toBeTruthy(); - }); - it('should run a test', async() => { - const t = new Runner(); - t.it('uno', () => {}); - const result = await t.run(); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('ok'); - }); - }); - - describe('TestRunner.xit', () => { - it('should declare a skipped test', async() => { - const t = new Runner(); - t.xit('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name()).toBe('uno'); - expect(test.fullName()).toBe('uno'); - expect(test.skipped()).toBe(true); - }); - it('should not run a skipped test', async() => { - const t = new Runner(); - t.xit('uno', () => {}); - const result = await t.run(); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('skipped'); - }); - }); - - describe('TestRunner.fit', () => { - it('should declare a focused test', async() => { - const t = new Runner(); - t.fit('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name()).toBe('uno'); - expect(test.fullName()).toBe('uno'); - expect(test.skipped()).toBe(false); - expect(t.focusedTests()[0]).toBe(test); - }); - it('should run a focused test', async() => { - const t = new Runner(); - t.fit('uno', () => {}); - const result = await t.run(); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('ok'); - }); - it('should run a failed focused test', async() => { - const t = new Runner(); - let run = false; - t.it.only.fail('uno', () => { - run = true; throw new Error('failure'); - }); - expect(t.focusedTests().length).toBe(1); - expect(t.tests()[0].expectation()).toBe(t.Expectations.Fail); - const result = await t.run(); - expect(run).toBe(true); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('failed'); - }); - }); - - describe('TestRunner.describe', () => { - it('should declare a suite', async() => { - const t = new Runner(); - t.describe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name()).toBe('uno'); - expect(test.fullName()).toBe('suite uno'); - expect(test.skipped()).toBe(false); - expect(test.suite().name()).toBe('suite'); - expect(test.suite().fullName()).toBe('suite'); - expect(test.suite().skipped()).toBe(false); - }); - }); - - describe('TestRunner.xdescribe', () => { - it('should declare a skipped suite', async() => { - const t = new Runner(); - t.xdescribe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.skipped()).toBe(false); - expect(test.suite().skipped()).toBe(true); - }); - it('focused tests inside a skipped suite are not run', async() => { - const t = new Runner(); - let run = false; - t.xdescribe('suite', () => { - t.fit('uno', () => { run = true; }); - }); - const result = await t.run(); - expect(run).toBe(false); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('skipped'); - }); - }); - - describe('TestRunner.fdescribe', () => { - it('should declare a focused suite', async() => { - const t = new Runner(); - t.fdescribe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.skipped()).toBe(false); - expect(t.focusedSuites()[0]).toBe(test.suite()); - expect(test.suite().skipped()).toBe(false); - }); - it('skipped tests inside a focused suite should not be run', async() => { - const t = new Runner(); - t.fdescribe('suite', () => { - t.xit('uno', () => {}); - }); - const result = await t.run(); - expect(result.runs.length).toBe(1); - expect(result.runs[0].test()).toBe(t.tests()[0]); - expect(result.runs[0].result()).toBe('skipped'); - }); - it('should run all "run" tests inside a focused suite', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2,3'); - }); - it('should run only "focus" tests inside a focused suite', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.fit('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2'); - }); - it('should run both "run" tests in focused suite and non-descendant focus tests', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.fit('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2,3,4'); - }); - }); - - describe('TestRunner attributes', () => { - it('should work', async() => { - const t = new Runner({timeout: 123}); - const log = []; - - t._collector.addTestModifier('foo', (t, ...args) => { - log.push('foo'); - - expect(t.skipped()).toBe(false); - expect(t.Expectations.Ok).toBeTruthy(); - expect(t.Expectations.Fail).toBeTruthy(); - expect(t.expectation()).toBe(t.Expectations.Ok); - expect(t.timeout()).toBe(123); - - expect(args.length).toBe(2); - expect(args[0]).toBe('uno'); - expect(args[1]).toBe('dos'); - - t.setExpectation(t.Expectations.Fail); - t.setTimeout(234); - }); - - t._collector.addTestAttribute('bar', t => { - log.push('bar'); - t.setSkipped(true); - expect(t.skipped()).toBe(true); - expect(t.expectation()).toBe(t.Expectations.Fail); - expect(t.timeout()).toBe(234); - }); - - t.it.foo('uno', 'dos').bar('test', () => { }); - expect(log).toEqual(['foo', 'bar']); - }); - }); - - describe('TestRunner hooks', () => { - it('should run all hooks in proper order', async() => { - const log = []; - const t = new Runner(); - const e = { - name() { return 'env'; }, - beforeAll() { log.push('env:beforeAll'); }, - afterAll() { log.push('env:afterAll'); }, - beforeEach() { log.push('env:beforeEach'); }, - afterEach() { log.push('env:afterEach'); }, - }; - const e2 = { - name() { return 'env2'; }, - beforeAll() { log.push('env2:beforeAll'); }, - afterAll() { log.push('env2:afterAll'); }, - }; - t.beforeAll(() => log.push('root:beforeAll')); - t.beforeEach(() => log.push('root:beforeEach')); - t.it('uno', () => log.push('test #1')); - t.describe('suite1', () => { - t.beforeAll(() => log.push('suite:beforeAll')); - t.beforeEach(() => log.push('suite:beforeEach')); - t.it('dos', () => log.push('test #2')); - t.it('tres', () => log.push('test #3')); - t.afterEach(() => log.push('suite:afterEach')); - t.afterAll(() => log.push('suite:afterAll')); - }); - t.it('cuatro', () => log.push('test #4')); - t.tests()[t.tests().length - 1].addEnvironment(e); - t.tests()[t.tests().length - 1].addEnvironment(e2); - t.describe('no hooks suite', () => { - t.describe('suite2', () => { - t.beforeAll(() => log.push('suite2:beforeAll')); - t.afterAll(() => log.push('suite2:afterAll')); - t.describe('no hooks suite 2', () => { - t.it('cinco', () => log.push('test #5')); - }); - }); - }); - t.suites()[t.suites().length - 1].addEnvironment(e); - t.suites()[t.suites().length - 1].addEnvironment(e2); - t.afterEach(() => log.push('root:afterEach')); - t.afterAll(() => log.push('root:afterAll')); - await t.run(); - expect(log).toEqual([ - 'root:beforeAll', - 'root:beforeEach', - 'test #1', - 'root:afterEach', - - 'suite:beforeAll', - - 'root:beforeEach', - 'suite:beforeEach', - 'test #2', - 'suite:afterEach', - 'root:afterEach', - - 'root:beforeEach', - 'suite:beforeEach', - 'test #3', - 'suite:afterEach', - 'root:afterEach', - - 'suite:afterAll', - - 'env:beforeAll', - 'env2:beforeAll', - - 'root:beforeEach', - 'env:beforeEach', - 'test #4', - 'env:afterEach', - 'root:afterEach', - - 'suite2:beforeAll', - 'root:beforeEach', - 'env:beforeEach', - 'test #5', - 'env:afterEach', - 'root:afterEach', - 'suite2:afterAll', - - 'env2:afterAll', - 'env:afterAll', - - 'root:afterAll', - ]); - }); - it('should remove environment', async() => { - const log = []; - const t = new Runner(); - const e = { - name() { return 'env'; }, - beforeAll() { log.push('env:beforeAll'); }, - afterAll() { log.push('env:afterAll'); }, - beforeEach() { log.push('env:beforeEach'); }, - afterEach() { log.push('env:afterEach'); }, - }; - const e2 = { - name() { return 'env2'; }, - beforeAll() { log.push('env2:beforeAll'); }, - afterAll() { log.push('env2:afterAll'); }, - beforeEach() { log.push('env2:beforeEach'); }, - afterEach() { log.push('env2:afterEach'); }, - }; - t.it('uno', () => log.push('test #1')); - t.tests()[0].addEnvironment(e).addEnvironment(e2).removeEnvironment(e); - await t.run(); - expect(log).toEqual([ - 'env2:beforeAll', - 'env2:beforeEach', - 'test #1', - 'env2:afterEach', - 'env2:afterAll', - ]); - }); - it('should have the same state object in hooks and test', async() => { - const states = []; - const t = new Runner(); - t.beforeEach(state => states.push(state)); - t.afterEach(state => states.push(state)); - t.beforeAll(state => states.push(state)); - t.afterAll(state => states.push(state)); - t.it('uno', state => states.push(state)); - await t.run(); - expect(states.length).toBe(5); - for (let i = 1; i < states.length; ++i) - expect(states[i]).toBe(states[0]); - }); - it('should unwind hooks properly when terminated', async() => { - const log = []; - const t = new Runner({timeout: 10000}); - t.beforeAll(() => log.push('beforeAll')); - t.beforeEach(() => log.push('beforeEach')); - t.afterEach(() => log.push('afterEach')); - t.afterAll(() => log.push('afterAll')); - t.it('uno', () => { - log.push('terminating...'); - t.terminate(); - }); - await t.run(); - - expect(log).toEqual([ - 'beforeAll', - 'beforeEach', - 'terminating...', - 'afterEach', - 'afterAll', - ]); - }); - it('should report as terminated even when hook crashes', async() => { - const t = new Runner({timeout: 10000}); - t.afterEach(() => { throw new Error('crash!'); }); - t.it('uno', () => { t.terminate(); }); - const result = await t.run(); - expect(result.runs[0].result()).toBe('terminated'); - }); - it('should report as terminated when terminated during hook', async() => { - const t = new Runner({timeout: 10000}); - t.afterEach(() => { t.terminate(); }); - t.it('uno', () => { }); - const result = await t.run(); - expect(result.runs[0].result()).toBe('terminated'); - }); - it('should unwind hooks properly when crashed', async() => { - const log = []; - const t = new Runner({timeout: 10000}); - t.beforeAll(() => log.push('root beforeAll')); - t.beforeEach(() => log.push('root beforeEach')); - t.describe('suite', () => { - t.beforeAll(() => log.push('suite beforeAll')); - t.beforeEach(() => log.push('suite beforeEach')); - t.it('uno', () => log.push('uno')); - t.afterEach(() => { - log.push('CRASH >> suite afterEach'); - throw new Error('crash!'); - }); - t.afterAll(() => log.push('suite afterAll')); - }); - t.afterEach(() => log.push('root afterEach')); - t.afterAll(() => log.push('root afterAll')); - await t.run(); - - expect(log).toEqual([ - 'root beforeAll', - 'suite beforeAll', - 'root beforeEach', - 'suite beforeEach', - 'uno', - 'CRASH >> suite afterEach', - 'root afterEach', - 'suite afterAll', - 'root afterAll' - ]); - }); - }); - - describe('globalSetup & globalTeardwon', () => { - it('should run globalSetup and globalTeardown in proper order', async() => { - const t = new Runner({timeout: 10000}); - const tracer = new TestTracer(t); - tracer.traceAllHooks(); - tracer.addTest('', 'test1'); - await t.run(); - - expect(tracer.trace()).toEqual([ - 'globalSetup', - 'beforeAll', - 'beforeEach', - 'test1', - 'afterEach', - 'afterAll', - 'globalTeardown', - ]); - }); - it('should run globalSetup and globalTeardown in proper order if parallel', async() => { - const t = new Runner({timeout: 10000, parallel: 2}); - const tracer = new TestTracer(t); - tracer.traceAllHooks('', async (hookName) => { - // slowdown globalsetup to see the rest hooks awaiting this one - if (hookName === 'globalSetup') - await new Promise(x => setTimeout(x, 50)); - }); - tracer.addTest('', 'test1'); - tracer.addTest('', 'test2'); - await t.run(); - - expect(tracer.trace()).toEqual([ - '<_global_> globalSetup', - ' beforeAll', - ' beforeAll', - ' beforeEach', - ' beforeEach', - ' test1', - ' test2', - ' afterEach', - ' afterEach', - ' afterAll', - ' afterAll', - '<_global_> globalTeardown', - ]); - }); - it('should support globalSetup/globalTeardown in nested suites', async() => { - const t = new Runner({timeout: 10000, parallel: 2}); - const tracer = new TestTracer(t); - tracer.traceAllHooks(''); - t.describe('suite', () => { - tracer.traceAllHooks(' '); - tracer.addTest(' ', 'test1'); - tracer.addTest(' ', 'test2'); - tracer.addTest(' ', 'test3'); - }); - await t.run(); - - expect(tracer.trace()).toEqual([ - '<_global_> globalSetup', - ' beforeAll', - ' beforeAll', - ' <_global_> globalSetup', - ' beforeAll', - ' beforeAll', - ' beforeEach', - ' beforeEach', - ' beforeEach', - ' beforeEach', - ' test1', - ' test2', - ' afterEach', - ' afterEach', - ' afterEach', - ' afterEach', - ' afterAll', - ' beforeEach', - ' afterAll', - ' beforeEach', - ' test3', - ' afterEach', - ' afterEach', - ' afterAll', - ' <_global_> globalTeardown', - ' afterAll', - '<_global_> globalTeardown', - ]); - }); - it('should report as crashed when global hook crashes', async() => { - const t = new Runner({timeout: 10000}); - t.globalSetup(() => { throw new Error('crash!'); }); - t.it('uno', () => { }); - const result = await t.run(); - expect(result.result).toBe('crashed'); - }); - it('should terminate and unwind hooks if globalSetup fails', async() => { - const t = new Runner({timeout: 10000}); - const tracer = new TestTracer(t); - tracer.traceAllHooks(); - t.describe('suite', () => { - tracer.traceAllHooks(' ', (hookName) => { - if (hookName === 'globalSetup') { - tracer.log(' !! CRASH !!'); - throw new Error('crash'); - } - }); - tracer.addTest(' ', 'test1'); - }); - await t.run(); - expect(tracer.trace()).toEqual([ - 'globalSetup', - 'beforeAll', - ' globalSetup', - ' !! CRASH !!', - ' afterAll', - ' globalTeardown', - 'afterAll', - 'globalTeardown', - ]); - }); - it('should not run globalSetup / globalTeardown if all tests are skipped', async() => { - const t = new Runner({timeout: 10000}); - const tracer = new TestTracer(t); - tracer.traceAllHooks(); - t.describe('suite', () => { - tracer.addSkippedTest(' ', 'test1'); - }); - await t.run(); - expect(tracer.trace()).toEqual([ - ]); - }); - it('should properly run globalTeardown if some tests are not run', async() => { - const t = new Runner({timeout: 10000}); - const tracer = new TestTracer(t); - tracer.traceAllHooks(); - t.describe('suite', () => { - tracer.addSkippedTest(' ', 'test1'); - tracer.addFailingTest(' ', 'test2'); - tracer.addTest(' ', 'test3'); - }); - await t.run(); - expect(tracer.trace()).toEqual([ - 'globalSetup', - 'beforeAll', - 'beforeEach', - ' test3', - 'afterEach', - 'afterAll', - 'globalTeardown', - ]); - }); - }); - - describe('TestRunner.run', () => { - it('should run a test', async() => { - const t = new Runner(); - let ran = false; - t.it('uno', () => ran = true); - await t.run(); - expect(ran).toBe(true); - }); - it('should handle repeat', async() => { - const t = new Runner(); - let suite = 0; - let test = 0; - let beforeAll = 0; - let beforeEach = 0; - t.describe.repeat(2)('suite', () => { - suite++; - t.beforeAll(() => beforeAll++); - t.beforeEach(() => beforeEach++); - t.it.repeat(3)('uno', () => test++); - }); - await t.run(); - expect(suite).toBe(1); - expect(beforeAll).toBe(1); - expect(beforeEach).toBe(6); - expect(test).toBe(6); - }); - it('should repeat without breaking test order', async() => { - const t = new Runner(); - const log = []; - t.describe.repeat(2)('suite', () => { - t.it('uno', () => log.push(1)); - t.it.repeat(2)('dos', () => log.push(2)); - }); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,2,2,1,2,2,3'); - }); - it('should run tests if some fail', async() => { - const t = new Runner(); - const log = []; - t.it('uno', () => log.push(1)); - t.it('dos', () => { throw new Error('bad'); }); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should run tests if some timeout', async() => { - const t = new Runner({timeout: 1}); - const log = []; - t.it('uno', () => log.push(1)); - t.it('dos', async() => new Promise(() => {})); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should break on first failure if configured so', async() => { - const log = []; - const t = new Runner({breakOnFailure: true}); - t.it('test#1', () => log.push('test#1')); - t.it('test#2', () => log.push('test#2')); - t.it('test#3', () => { throw new Error('crash'); }); - t.it('test#4', () => log.push('test#4')); - await t.run(); - expect(log).toEqual([ - 'test#1', - 'test#2', - ]); - }); - it('should pass a state and a test as a test parameters', async() => { - const log = []; - const t = new Runner(); - t.beforeEach(state => state.FOO = 42); - t.it('uno', (state, testRun) => { - log.push('state.FOO=' + state.FOO); - log.push('test=' + testRun.test().name()); - }); - await t.run(); - expect(log.join()).toBe('state.FOO=42,test=uno'); - }); - it('should run async test', async() => { - const t = new Runner(); - let ran = false; - t.it('uno', async() => { - await new Promise(x => setTimeout(x, 10)); - ran = true; - }); - await t.run(); - expect(ran).toBe(true); - }); - it('should run async tests in order of their declaration', async() => { - const log = []; - const t = new Runner(); - t.it('uno', async() => { - await new Promise(x => setTimeout(x, 30)); - log.push(1); - }); - t.it('dos', async() => { - await new Promise(x => setTimeout(x, 20)); - log.push(2); - }); - t.it('tres', async() => { - await new Promise(x => setTimeout(x, 10)); - log.push(3); - }); - await t.run(); - expect(log.join()).toBe('1,2,3'); - }); - it('should run multiple tests', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.it('dos', () => log.push(2)); - await t.run(); - expect(log.join()).toBe('1,2'); - }); - it('should NOT run a skipped test', async() => { - const t = new Runner(); - let ran = false; - t.xit('uno', () => ran = true); - await t.run(); - expect(ran).toBe(false); - }); - it('should run ONLY non-skipped tests', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.xit('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should run ONLY focused tests', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.xit('dos', () => log.push(2)); - t.fit('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('3'); - }); - it('should run tests in order of their declaration', async() => { - const log = []; - const t = new Runner(); - t.it('uno', () => log.push(1)); - t.describe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('1,2,3,4'); - }); - it('should respect total timeout', async() => { - const t = new Runner({timeout: 10000, totalTimeout: 1}); - t.it('uno', async () => { await new Promise(() => {}); }); - const result = await t.run(); - expect(result.runs[0].result()).toBe('terminated'); - expect(result.message).toContain('Total timeout'); - }); - }); - - describe('TestRunner.run result', () => { - it('should return OK if all tests pass', async() => { - const t = new Runner(); - t.it('uno', () => {}); - const result = await t.run(); - expect(result.result).toBe('ok'); - }); - it('should return FAIL if at least one test fails', async() => { - const t = new Runner(); - t.it('uno', () => { throw new Error('woof'); }); - const result = await t.run(); - expect(result.result).toBe('failed'); - }); - it('should return FAIL if at least one test times out', async() => { - const t = new Runner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - const result = await t.run(); - expect(result.result).toBe('failed'); - }); - it('should return TERMINATED if it was terminated', async() => { - const t = new Runner({timeout: 1000000}); - t.it('uno', async() => new Promise(() => {})); - const [result] = await Promise.all([ - t.run(), - t.terminate(), - ]); - expect(result.result).toBe('terminated'); - }); - it('should return CRASHED if it crashed', async() => { - const t = new Runner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - t.afterAll(() => { throw new Error('woof');}); - const result = await t.run(); - expect(result.result).toBe('crashed'); - }); - }); - - describe('TestRunner parallel', () => { - it('should run tests in parallel', async() => { - const log = []; - const t = new Runner({parallel: 2}); - t.it('uno', async state => { - log.push(`Worker #${state.parallelIndex} Starting: UNO`); - await Promise.resolve(); - log.push(`Worker #${state.parallelIndex} Ending: UNO`); - }); - t.it('dos', async state => { - log.push(`Worker #${state.parallelIndex} Starting: DOS`); - await Promise.resolve(); - log.push(`Worker #${state.parallelIndex} Ending: DOS`); - }); - await t.run(); - expect(log).toEqual([ - 'Worker #0 Starting: UNO', - 'Worker #1 Starting: DOS', - 'Worker #0 Ending: UNO', - 'Worker #1 Ending: DOS', - ]); - }); - }); - - describe('TestRunner.hasFocusedTestsOrSuitesOrFiles', () => { - it('should work', () => { - const t = new Runner(); - t.it('uno', () => {}); - expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(false); - }); - it('should work #2', () => { - const t = new Runner(); - t.fit('uno', () => {}); - expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(true); - }); - it('should work #3', () => { - const t = new Runner(); - t.describe('suite #1', () => { - t.fdescribe('suite #2', () => { - t.describe('suite #3', () => { - t.it('uno', () => {}); - }); - }); - }); - expect(t._filter.hasFocusedTestsOrSuitesOrFiles()).toBe(true); - }); - }); - - describe('TestRunner result', () => { - it('should work for both throwing and timeouting tests', async() => { - const t = new Runner({timeout: 1}); - t.it('uno', () => { throw new Error('boo');}); - t.it('dos', () => new Promise(() => {})); - const result = await t.run(); - expect(result.runs[0].result()).toBe('failed'); - expect(result.runs[1].result()).toBe('timedout'); - }); - it('should report crashed tests', async() => { - const t = new Runner(); - t.beforeEach(() => { throw new Error('woof');}); - t.it('uno', () => {}); - const result = await t.run(); - expect(result.runs[0].result()).toBe('crashed'); - }); - it('skipped should work for both throwing and timeouting tests', async() => { - const t = new Runner({timeout: 1}); - t.xit('uno', () => { throw new Error('boo');}); - const result = await t.run(); - expect(result.runs[0].result()).toBe('skipped'); - }); - it('should return OK', async() => { - const t = new Runner(); - t.it('uno', () => {}); - const result = await t.run(); - expect(result.runs[0].result()).toBe('ok'); - }); - it('should return TIMEDOUT', async() => { - const t = new Runner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - const result = await t.run(); - expect(result.runs[0].result()).toBe('timedout'); - }); - it('should return SKIPPED', async() => { - const t = new Runner(); - t.xit('uno', () => {}); - const result = await t.run(); - expect(result.runs[0].result()).toBe('skipped'); - }); - it('should return FAILED', async() => { - const t = new Runner(); - t.it('uno', async() => Promise.reject('woof')); - const result = await t.run(); - expect(result.runs[0].result()).toBe('failed'); - }); - it('should return TERMINATED', async() => { - const t = new Runner(); - t.it('uno', async() => t.terminate()); - const result = await t.run(); - expect(result.runs[0].result()).toBe('terminated'); - }); - it('should return CRASHED', async() => { - const t = new Runner(); - t.it('uno', () => {}); - t.afterEach(() => {throw new Error('foo');}); - const result = await t.run(); - expect(result.runs[0].result()).toBe('crashed'); - }); - }); - - describe('TestRunner delegate', () => { - it('should call delegate methods in proper order', async() => { - const log = []; - const t = new Runner({ - onStarted: () => log.push('E:started'), - onTestRunStarted: () => log.push('E:teststarted'), - onTestRunFinished: () => log.push('E:testfinished'), - onFinished: () => log.push('E:finished'), - }); - t.beforeAll(() => log.push('beforeAll')); - t.beforeEach(() => log.push('beforeEach')); - t.it('test#1', () => log.push('test#1')); - t.afterEach(() => log.push('afterEach')); - t.afterAll(() => log.push('afterAll')); - await t.run(); - expect(log).toEqual([ - 'E:started', - 'beforeAll', - 'E:teststarted', - 'beforeEach', - 'test#1', - 'afterEach', - 'E:testfinished', - 'afterAll', - 'E:finished', - ]); - }); - it('should call onFinished with result', async() => { - let onFinished; - const finishedPromise = new Promise(f => onFinished = f); - const [result] = await Promise.all([ - finishedPromise, - new TestRunner().run([], { onFinished }), - ]); - expect(result.result).toBe('ok'); - }); - it('should crash when onStarted throws', async() => { - const t = new Runner({ - onStarted: () => { throw 42; }, - }); - const result = await t.run(); - expect(result.ok()).toBe(false); - expect(result.message).toBe('INTERNAL ERROR: 42'); - }); - it('should crash when onFinished throws', async() => { - const t = new Runner({ - onFinished: () => { throw new Error('42'); }, - }); - const result = await t.run(); - expect(result.ok()).toBe(false); - expect(result.message).toBe('INTERNAL ERROR'); - expect(result.result).toBe('crashed'); - }); - }); -}; - -class TestTracer { - constructor(testRunner) { - this._testRunner = testRunner; - this._trace = []; - } - - addSkippedTest(prefix, testName, callback) { - this._testRunner.it.skip(testName, async(...args) => { - if (callback) - await callback(...args); - this._trace.push(prefix + this._workerPrefix(args[0]) + testName); - }); - } - - addFailingTest(prefix, testName, callback) { - this._testRunner.it.fail(testName, async(...args) => { - if (callback) - await callback(...args); - this._trace.push(prefix + this._workerPrefix(args[0]) + testName); - }); - } - - addTest(prefix, testName, callback) { - this._testRunner.it(testName, async(...args) => { - if (callback) - await callback(...args); - this._trace.push(prefix + this._workerPrefix(args[0]) + testName); - }); - } - - traceHooks(hookNames, prefix = '', callback) { - for (const hookName of hookNames) { - this._testRunner[hookName].call(this._testRunner, async (state) => { - this._trace.push(prefix + this._workerPrefix(state) + hookName); - if (callback) - await callback(hookName); - }); - } - } - - _workerPrefix(state) { - if (this._testRunner.parallel() === 1) - return ''; - return state && (typeof state.parallelIndex !== 'undefined') ? ` ` : `<_global_> `; - - } - - traceAllHooks(prefix = '', callback) { - this.traceHooks(['globalSetup', 'globalTeardown', 'beforeAll', 'afterAll', 'beforeEach', 'afterEach'], prefix, callback); - } - - log(text) { - this._trace.push(text); - } - - trace() { - return this._trace; - } -} -