diff --git a/test-runner/src/cli.js b/test-runner/src/cli.js index 0df874a0d6..44f379da46 100644 --- a/test-runner/src/cli.js +++ b/test-runner/src/cli.js @@ -14,128 +14,4 @@ * limitations under the License. */ -const fs = require('fs'); -const path = require('path'); -const program = require('commander'); -const { installTransform } = require('./transform'); -const { Runner } = require('./runner'); -const { TestCollector } = require('./testCollector'); - -let beforeFunction; -let afterFunction; -let matrix = {}; - -global['before'] = (fn => beforeFunction = fn); -global['after'] = (fn => afterFunction = fn); -global['matrix'] = (m => matrix = m); - -program - .version('Version ' + /** @type {any} */ (require)('../package.json').version) - .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) - .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') - .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString()) - .option('--reporter ', 'Specify reporter to use', '') - .option('--trial-run', 'Only collect the matching tests and report them as passing') - .option('--quiet', 'Suppress stdio', false) - .option('--debug', 'Run tests in-process for debugging', false) - .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) - .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000') - .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') - .action(async (command) => { - // Collect files] - const testDir = path.resolve(process.cwd(), command.args[0]); - const files = collectFiles(testDir, '', command.args.slice(1)); - - const revertBabelRequire = installTransform(); - let hasSetup = false; - try { - hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile(); - } catch (e) { - } - try { - hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile(); - } catch (e) { - } - - if (hasSetup) - require(path.join(testDir, 'setup')); - revertBabelRequire(); - - const testCollector = new TestCollector(files, matrix, { - forbidOnly: command.forbidOnly || undefined, - grep: command.grep, - timeout: command.timeout, - }); - const rootSuite = testCollector.suite; - if (command.forbidOnly && testCollector.hasOnly()) { - console.error('====================================='); - console.error(' --forbid-only found a focused test.'); - console.error('====================================='); - process.exit(1); - } - - const total = rootSuite.total(); - if (!total) { - console.error('================='); - console.error(' no tests found.'); - console.error('================='); - process.exit(1); - } - - // Trial run does not need many workers, use one. - const jobs = (command.trialRun || command.debug) ? 1 : command.jobs; - const runner = new Runner(rootSuite, total, { - debug: command.debug, - quiet: command.quiet, - grep: command.grep, - jobs, - outputDir: command.output, - reporter: command.reporter, - retries: command.retries, - snapshotDir: path.join(testDir, '__snapshots__'), - testDir, - timeout: command.timeout, - trialRun: command.trialRun, - updateSnapshots: command.updateSnapshots - }); - try { - if (beforeFunction) - await beforeFunction(); - await runner.run(); - await runner.stop(); - } finally { - if (afterFunction) - await afterFunction(); - } - process.exit(runner.stats.failures ? 1 : 0); - }); - -program.parse(process.argv); - -function collectFiles(testDir, dir, filters) { - const fullDir = path.join(testDir, dir); - if (fs.statSync(fullDir).isFile()) - return [fullDir]; - const files = []; - for (const name of fs.readdirSync(fullDir)) { - if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) { - files.push(...collectFiles(testDir, path.join(dir, name), filters)); - continue; - } - if (!name.endsWith('spec.ts')) - continue; - const relativeName = path.join(dir, name); - const fullName = path.join(testDir, relativeName); - if (!filters.length) { - files.push(fullName); - continue; - } - for (const filter of filters) { - if (relativeName.includes(filter)) { - files.push(fullName); - break; - } - } - } - return files; -} +require('./lib/cli'); diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts new file mode 100644 index 0000000000..23f9a3440e --- /dev/null +++ b/test-runner/src/cli.ts @@ -0,0 +1,141 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import program from 'commander'; +import { installTransform } from './transform'; +import { Runner } from './runner'; +import { TestCollector } from './testCollector'; + +let beforeFunction; +let afterFunction; +let matrix = {}; + +global['before'] = (fn => beforeFunction = fn); +global['after'] = (fn => afterFunction = fn); +global['matrix'] = (m => matrix = m); + +program + .version('Version ' + /** @type {any} */ (require)('../package.json').version) + .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) + .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') + .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString()) + .option('--reporter ', 'Specify reporter to use', '') + .option('--trial-run', 'Only collect the matching tests and report them as passing') + .option('--quiet', 'Suppress stdio', false) + .option('--debug', 'Run tests in-process for debugging', false) + .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) + .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000') + .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') + .action(async (command) => { + // Collect files] + const testDir = path.resolve(process.cwd(), command.args[0]); + const files = collectFiles(testDir, '', command.args.slice(1)); + + const revertBabelRequire = installTransform(); + let hasSetup = false; + try { + hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile(); + } catch (e) { + } + try { + hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile(); + } catch (e) { + } + + if (hasSetup) + require(path.join(testDir, 'setup')); + revertBabelRequire(); + + const testCollector = new TestCollector(files, matrix, { + forbidOnly: command.forbidOnly || undefined, + grep: command.grep, + timeout: command.timeout, + }); + const rootSuite = testCollector.suite; + if (command.forbidOnly && testCollector.hasOnly()) { + console.error('====================================='); + console.error(' --forbid-only found a focused test.'); + console.error('====================================='); + process.exit(1); + } + + const total = rootSuite.total(); + if (!total) { + console.error('================='); + console.error(' no tests found.'); + console.error('================='); + process.exit(1); + } + + // Trial run does not need many workers, use one. + const jobs = (command.trialRun || command.debug) ? 1 : command.jobs; + const runner = new Runner(rootSuite, total, { + debug: command.debug, + quiet: command.quiet, + grep: command.grep, + jobs, + outputDir: command.output, + reporter: command.reporter, + retries: command.retries, + snapshotDir: path.join(testDir, '__snapshots__'), + testDir, + timeout: command.timeout, + trialRun: command.trialRun, + updateSnapshots: command.updateSnapshots + }); + try { + if (beforeFunction) + await beforeFunction(); + await runner.run(); + await runner.stop(); + } finally { + if (afterFunction) + await afterFunction(); + } + process.exit(runner.stats.failures ? 1 : 0); + }); + +program.parse(process.argv); + +function collectFiles(testDir, dir, filters) { + const fullDir = path.join(testDir, dir); + if (fs.statSync(fullDir).isFile()) + return [fullDir]; + const files = []; + for (const name of fs.readdirSync(fullDir)) { + if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) { + files.push(...collectFiles(testDir, path.join(dir, name), filters)); + continue; + } + if (!name.endsWith('spec.ts')) + continue; + const relativeName = path.join(dir, name); + const fullName = path.join(testDir, relativeName); + if (!filters.length) { + files.push(fullName); + continue; + } + for (const filter of filters) { + if (relativeName.includes(filter)) { + files.push(fullName); + break; + } + } + } + return files; +} diff --git a/test-runner/src/dotReporter.js b/test-runner/src/dotReporter.js index 5489fbc610..80a7b7eeb3 100644 --- a/test-runner/src/dotReporter.js +++ b/test-runner/src/dotReporter.js @@ -43,7 +43,7 @@ class DotReporter extends Base { }); runner.on(constants.EVENT_TEST_FAIL, test => { - if (test.duration >= test.timeout()) + if (test.duration >= test.timeout) process.stdout.write(colors.red('T')); else process.stdout.write(colors.red('F')); diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts index a9d9edfa1a..ddde08ae6d 100644 --- a/test-runner/src/fixtures.ts +++ b/test-runner/src/fixtures.ts @@ -15,6 +15,7 @@ */ import debug from 'debug'; +import { Test } from './test'; declare global { interface WorkerState { @@ -38,17 +39,15 @@ export function setParameters(params: any) { registerWorkerFixture(name as keyof WorkerState, async ({}, test) => await test(parameters[name] as never)); } -type TestInfo = { - file: string; - title: string; - timeout: number; +type TestConfig = { outputDir: string; testDir: string; }; type TestResult = { success: boolean; - info: TestInfo; + test: Test; + config: TestConfig; error?: Error; }; @@ -168,10 +167,10 @@ export class FixturePool { ]); } - wrapTestCallback(callback: any, timeout: number, info: TestInfo) { + wrapTestCallback(callback: any, timeout: number, test: Test, config: TestConfig) { if (!callback) return callback; - const testResult: TestResult = { success: true, info }; + const testResult: TestResult = { success: true, test, config }; return async() => { try { await this.resolveParametersAndRun(callback, timeout); diff --git a/test-runner/src/fixturesUI.js b/test-runner/src/fixturesUI.ts similarity index 62% rename from test-runner/src/fixturesUI.js rename to test-runner/src/fixturesUI.ts index a38887270b..7789b89cdc 100644 --- a/test-runner/src/fixturesUI.js +++ b/test-runner/src/fixturesUI.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -const { Test, Suite } = require('mocha'); -const { installTransform } = require('./transform'); -const commonSuite = require('mocha/lib/interfaces/common'); +import Mocha from 'mocha'; +import { Test, Suite } from './test'; +import { installTransform } from './transform'; Error.stackTraceLimit = 15; -let revertBabelRequire; +let revertBabelRequire: () => void; function specBuilder(modifiers, specCallback) { function builder(specs, last) { @@ -52,67 +52,65 @@ function specBuilder(modifiers, specCallback) { return builder({}, null); } -function fixturesUI(wrappers, suite) { - const suites = [suite]; - - suite.on(Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file, mocha) { - const common = commonSuite(suites, context, mocha); +export function fixturesUI(options, mochaSuite: any) { + const suites = [mochaSuite.__nomocha as Suite]; + mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file) { const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => { const suite = suites[0]; - - if (suite.isPending()) - fn = null; - const wrapper = fn ? wrappers.testWrapper(fn, title, file, specs.slow && specs.slow[0]) : undefined; - if (wrapper) { - wrapper.toString = () => fn.toString(); - wrapper.__original = fn; - } - const test = new Test(title, wrapper); + const test = new Test(title, fn); test.file = file; - suite.addTest(test); - const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0]; + test.slow = specs.slow && specs.slow[0]; + test.timeout = options.timeout; + + const only = specs.only && specs.only[0]; if (only) - test.__only = true; + test.only = true; if (!only && specs.skip && specs.skip[0]) test.pending = true; if (!only && specs.fail && specs.fail[0]) test.pending = true; + + test.pending = test.pending || suite.isPending(); + if (test.pending) + fn = null; + const wrapper = fn ? options.testWrapper(test, fn) : undefined; + if (wrapper) + wrapper.toString = () => fn.toString(); + test._materialize(wrapper); + suite.addTest(test); return test; }); const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => { - const suite = common.suite.create({ - title: title, - file: file, - fn: fn - }); - const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0]; + const child = new Suite(title, suites[0]); + suites[0].addSuite(child); + child.file = file; + const only = specs.only && specs.only[0]; if (only) - suite.__only = true; + child.only = true; if (!only && specs.skip && specs.skip[0]) - suite.pending = true; + child.pending = true; if (!only && specs.fail && specs.fail[0]) - suite.pending = true; - return suite; + child.pending = true; + suites.unshift(child); + fn(); + suites.shift(); }); - context.beforeEach = fn => wrappers.hookWrapper(common.beforeEach.bind(common), fn); - context.afterEach = fn => wrappers.hookWrapper(common.afterEach.bind(common), fn); - context.run = mocha.options.delay && common.runWithSuite(suite); + context.beforeEach = fn => options.hookWrapper(mochaSuite.beforeEach.bind(mochaSuite), fn); + context.afterEach = fn => options.hookWrapper(mochaSuite.afterEach.bind(mochaSuite), fn); context.describe = describe; - context.fdescribe = describe.only(true); + (context as any).fdescribe = describe.only(true); context.xdescribe = describe.skip(true); context.it = it; - context.fit = it.only(true); + (context as any).fit = it.only(true); context.xit = it.skip(true); revertBabelRequire = installTransform(); }); - suite.on(Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) { + mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) { revertBabelRequire(); }); }; - -module.exports = { fixturesUI }; diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index 7eae1a2764..fa54310d8d 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import './builtin.fixtures'; import './expect'; -export {registerFixture, registerWorkerFixture, registerParameter, parameters} from './fixtures'; \ No newline at end of file +export {registerFixture, registerWorkerFixture, registerParameter, parameters} from './fixtures'; diff --git a/test-runner/src/runner.js b/test-runner/src/runner.js index 1d6dcacddb..62bfc4bf1d 100644 --- a/test-runner/src/runner.js +++ b/test-runner/src/runner.js @@ -50,19 +50,19 @@ class Runner extends EventEmitter { this._testsByConfiguredFile = new Map(); suite.eachTest(test => { - const configuredFile = `${test.file}::[${test.__configurationString}]`; + const configuredFile = `${test.file}::[${test._configurationString}]`; if (!this._testsByConfiguredFile.has(configuredFile)) { this._testsByConfiguredFile.set(configuredFile, { file: test.file, configuredFile, ordinals: [], - configurationObject: test.__configurationObject, - configurationString: test.__configurationString + configurationObject: test._configurationObject, + configurationString: test._configurationString }); } const { ordinals } = this._testsByConfiguredFile.get(configuredFile); - ordinals.push(test.__ordinal); - this._testById.set(`${test.__ordinal}@${configuredFile}`, test); + ordinals.push(test._ordinal); + this._testById.set(`${test._ordinal}@${configuredFile}`, test); }); if (process.stdout.isTTY) { diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts new file mode 100644 index 0000000000..c69b187bdb --- /dev/null +++ b/test-runner/src/test.ts @@ -0,0 +1,212 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Mocha from 'mocha'; +import { fixturesUI } from './fixturesUI'; +import { EventEmitter } from 'events'; + +export type Configuration = { name: string, value: string }[]; + +export class Test { + suite: Suite; + title: string; + file: string; + only = false; + pending = false; + duration = 0; + timeout = 0; + fn: Function; + + _ordinal: number; + _configurationObject: Configuration; + _configurationString: string; + _overriddenFn: Function; + _impl: any; + + constructor(title: string, fn: Function) { + this.title = title; + this.fn = fn; + } + + _materialize(overriddenFn: Function) { + this._impl = new Mocha.Test(this.title, overriddenFn); + this._impl.pending = this.pending; + } + + clone(): Test { + const test = new Test(this.title, this.fn); + test.suite = this.suite; + test.only = this.only; + test.file = this.file; + test.pending = this.pending; + test.timeout = this.timeout; + test._overriddenFn = this._overriddenFn; + test._materialize(this._overriddenFn); + return test; + } + + titlePath(): string[] { + return [...this.suite.titlePath(), this.title]; + } + + fullTitle(): string { + return this.titlePath().join(' '); + } + + slow(): number { + return 10000; + } +} + +export class Suite { + title: string; + parent?: Suite; + suites: Suite[] = []; + tests: Test[] = []; + only = false; + pending = false; + file: string; + + _impl: any; + + constructor(title: string, parent?: Suite) { + this.title = title; + this.parent = parent; + this._impl = new Mocha.Suite(title, new Mocha.Context()); + this._impl.__nomocha = this; + } + + titlePath(): string[] { + if (!this.parent) + return [this.title]; + return [...this.parent.titlePath(), this.title]; + } + + total(): number { + let count = 0; + this.eachTest(fn => ++count); + return count; + } + + isPending(): boolean { + return this.pending || (this.parent && this.parent.isPending()); + } + + addTest(test: Test) { + test.suite = this; + this.tests.push(test); + this._impl.addTest(test._impl); + } + + addSuite(suite: Suite) { + suite.parent = this; + this.suites.push(suite); + this._impl.addSuite(suite._impl); + } + + eachTest(fn: (test: Test) => void) { + for (const suite of this.suites) + suite.eachTest(fn); + for (const test of this.tests) + fn(test); + } + + clone(): Suite { + const suite = new Suite(this.title); + suite.only = this.only; + suite.file = this.file; + suite.pending = this.pending; + suite._impl = this._impl.clone(); + return suite; + } +} + +class NullReporter { + stats = { + suites: 0, + tests: 0, + passes: 0, + pending: 0, + failures: 0 + }; + runner = null; + failures = []; + epilogue: () => {}; +} + +type NoMockaOptions = { + forbidOnly?: boolean; + timeout: number; + testWrapper: (test: Test, fn: Function) => Function; + hookWrapper: (hook: any, fn: Function) => Function; +}; + +class PatchedMocha extends Mocha { + suite: any; + static pendingSuite: Suite; + + constructor(suite, options) { + PatchedMocha.pendingSuite = suite; + super(options); + } + + grep(...args) { + this.suite = new Mocha.Suite('', new Mocha.Context()); + this.suite.__nomocha = PatchedMocha.pendingSuite; + PatchedMocha.pendingSuite._impl = this.suite; + return super.grep(...args); + } +} + +export class Runner extends EventEmitter { + private _mochaRunner: any; + + constructor(mochaRunner: any) { + super(); + const constants = Mocha.Runner.constants; + this._mochaRunner = mochaRunner; + this._mochaRunner.on(constants.EVENT_TEST_BEGIN, test => this.emit('test', test)); + this._mochaRunner.on(constants.EVENT_TEST_PENDING, test => this.emit('pending', test)); + this._mochaRunner.on(constants.EVENT_TEST_PASS, test => this.emit('pass', test)); + this._mochaRunner.on(constants.EVENT_TEST_FAIL, (test, err) => this.emit('fail', test, err)); + this._mochaRunner.on(constants.EVENT_RUN_END, () => this.emit('done')); + } + + duration(): number { + return this._mochaRunner.stats.duration || 0; + } +} + +export class NoMocha { + suite: Suite; + private _mocha: Mocha; + + constructor(file: string, options: NoMockaOptions) { + this.suite = new Suite(''); + this._mocha = new PatchedMocha(this.suite, { + forbidOnly: options.forbidOnly, + reporter: NullReporter, + timeout: options.timeout, + ui: fixturesUI.bind(null, options) + }); + this._mocha.addFile(file); + (this._mocha as any).loadFiles(); + } + + run(cb: () => void): Runner { + return new Runner(this._mocha.run(cb)); + } +} diff --git a/test-runner/src/testCollector.js b/test-runner/src/testCollector.ts similarity index 75% rename from test-runner/src/testCollector.js rename to test-runner/src/testCollector.ts index ee79ae4084..796f38ceef 100644 --- a/test-runner/src/testCollector.js +++ b/test-runner/src/testCollector.ts @@ -14,22 +14,26 @@ * limitations under the License. */ -const path = require('path'); -const Mocha = require('mocha'); -const { fixturesForCallback, registerWorkerFixture } = require('./fixtures'); -const { fixturesUI } = require('./fixturesUI'); +import path from 'path'; +import Mocha from 'mocha'; +import { fixturesForCallback, registerWorkerFixture } from './fixtures'; +import { Configuration, NoMocha, Test, Suite } from './test'; -class NullReporter {} +export class TestCollector { + suite: Suite; -class TestCollector { - constructor(files, matrix, options) { + private _matrix: { [key: string]: string; }; + private _options: any; + private _grep: RegExp; + private _hasOnly: boolean; + + constructor(files: string[], matrix: { [key: string] : string }, options) { this._matrix = matrix; for (const name of Object.keys(matrix)) //@ts-ignore registerWorkerFixture(name, async ({}, test) => test()); this._options = options; - this.suite = new Mocha.Suite('', new Mocha.Context(), true); - this._total = 0; + this.suite = new Suite(''); if (options.grep) { const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/); this._grep = new RegExp(match[1] || match[0], match[2]); @@ -46,28 +50,22 @@ class TestCollector { } _addFile(file) { - const mocha = new Mocha({ + const noMocha = new NoMocha(file, { forbidOnly: this._options.forbidOnly, - reporter: NullReporter, timeout: this._options.timeout, - ui: fixturesUI.bind(null, { - testWrapper: (fn) => done => done(), - hookWrapper: (hook, fn) => {}, - ignoreOnly: false, - }), + testWrapper: (test, fn) => () => {}, + hookWrapper: (hook, fn) => () => {}, }); - mocha.addFile(file); - mocha.loadFiles(); const workerGeneratorConfigurations = new Map(); let ordinal = 0; - mocha.suite.eachTest(test => { + noMocha.suite.eachTest((test: Test) => { // All tests are identified with their ordinals. - test.__ordinal = ordinal++; + test._ordinal = ordinal++; // Get all the fixtures that the test needs. - const fixtures = fixturesForCallback(test.fn.__original); + const fixtures = fixturesForCallback(test.fn); // For generator fixtures, collect all variants of the fixture values // to build different workers for them. @@ -104,15 +102,15 @@ class TestCollector { // Clone the suite as many times as there are worker hashes. // Only include the tests that requested these generations. for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) { - const clone = this._cloneSuite(mocha.suite, configurationObject, configurationString, tests); + const clone = this._cloneSuite(noMocha.suite, configurationObject, configurationString, tests); this.suite.addSuite(clone); clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : ''); } } - _cloneSuite(suite, configurationObject, configurationString, tests) { + _cloneSuite(suite: Suite, configurationObject: Configuration, configurationString: string, tests: Set) { const copy = suite.clone(); - copy.__only = suite.__only; + copy.only = suite.only; for (const child of suite.suites) copy.addSuite(this._cloneSuite(child, configurationObject, configurationString, tests)); for (const test of suite.tests) { @@ -121,18 +119,18 @@ class TestCollector { if (this._grep && !this._grep.test(test.fullTitle())) continue; const testCopy = test.clone(); - testCopy.__only = test.__only; - testCopy.__ordinal = test.__ordinal; - testCopy.__configurationObject = configurationObject; - testCopy.__configurationString = configurationString; + testCopy.only = test.only; + testCopy._ordinal = test._ordinal; + testCopy._configurationObject = configurationObject; + testCopy._configurationString = configurationString; copy.addTest(testCopy); } return copy; } _filterOnly(suite) { - const onlySuites = suite.suites.filter(child => this._filterOnly(child) || child.__only); - const onlyTests = suite.tests.filter(test => test.__only); + const onlySuites = suite.suites.filter(child => this._filterOnly(child) || child.only); + const onlyTests = suite.tests.filter(test => test.only); if (onlySuites.length || onlyTests.length) { suite.suites = onlySuites; suite.tests = onlyTests; diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 73bc0217ea..56fc6e5f01 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -15,11 +15,10 @@ */ import path from 'path'; -import Mocha from 'mocha'; import { FixturePool, registerWorkerFixture, rerunRegistrations, setParameters } from './fixtures'; -import { fixturesUI } from './fixturesUI'; import { EventEmitter } from 'events'; import { setCurrentTestFile } from './expect'; +import { NoMocha, Runner, Test } from './test'; export const fixturePool = new FixturePool(); @@ -30,10 +29,7 @@ export type TestRunnerEntry = { configurationObject: any; }; -class NullReporter {} - export class TestRunner extends EventEmitter { - mocha: any; private _currentOrdinal = -1; private _failedWithError = false; private _file: any; @@ -47,22 +43,13 @@ export class TestRunner extends EventEmitter { private _configurationObject: any; private _parsedGeneratorConfiguration: any = {}; private _relativeTestFile: string; - private _runner: Mocha.Runner; + private _runner: Runner; private _outDir: string; private _timeout: number; private _testDir: string; constructor(entry: TestRunnerEntry, options, workerId) { super(); - this.mocha = new Mocha({ - reporter: NullReporter, - timeout: 0, - ui: fixturesUI.bind(null, { - testWrapper: (fn, title, file, isSlow) => this._testWrapper(fn, title, file, isSlow), - hookWrapper: (hook, fn) => this._hookWrapper(hook, fn), - ignoreOnly: true - }), - }); this._file = entry.file; this._ordinals = new Set(entry.ordinals); this._remaining = new Set(entry.ordinals); @@ -79,25 +66,27 @@ export class TestRunner extends EventEmitter { } this._parsedGeneratorConfiguration['parallelIndex'] = workerId; this._relativeTestFile = path.relative(options.testDir, this._file); - this.mocha.addFile(this._file); } async stop() { this._trialRun = true; - const constants = Mocha.Runner.constants; - return new Promise(f => this._runner.once(constants.EVENT_RUN_END, f)); + return new Promise(f => this._runner.once('done', f)); } async run() { let callback; const result = new Promise(f => callback = f); setParameters(this._parsedGeneratorConfiguration); - this.mocha.loadFiles(); - rerunRegistrations(this._file, 'test'); - this._runner = this.mocha.run(callback); - const constants = Mocha.Runner.constants; - this._runner.on(constants.EVENT_TEST_BEGIN, test => { + const noMocha = new NoMocha(this._file, { + timeout: 0, + testWrapper: (test, fn) => this._testWrapper(test, fn), + hookWrapper: (hook, fn) => this._hookWrapper(hook, fn), + }); + rerunRegistrations(this._file, 'test'); + this._runner = noMocha.run(callback); + + this._runner.on('test', test => { setCurrentTestFile(this._relativeTestFile); if (this._failedWithError) return; @@ -108,7 +97,7 @@ export class TestRunner extends EventEmitter { this.emit('test', { test: this._serializeTest(test, ordinal) }); }); - this._runner.on(constants.EVENT_TEST_PENDING, test => { + this._runner.on('pending', test => { if (this._failedWithError) return; const ordinal = ++this._currentOrdinal; @@ -119,7 +108,7 @@ export class TestRunner extends EventEmitter { this.emit('pending', { test: this._serializeTest(test, ordinal) }); }); - this._runner.on(constants.EVENT_TEST_PASS, test => { + this._runner.on('pass', test => { if (this._failedWithError) return; @@ -130,7 +119,7 @@ export class TestRunner extends EventEmitter { this.emit('pass', { test: this._serializeTest(test, ordinal) }); }); - this._runner.on(constants.EVENT_TEST_FAIL, (test, error) => { + this._runner.on('fail', (test, error) => { if (this._failedWithError) return; ++this._failures; @@ -141,12 +130,12 @@ export class TestRunner extends EventEmitter { }); }); - this._runner.once(constants.EVENT_RUN_END, async () => { + this._runner.once('done', async () => { this.emit('done', { - stats: this._serializeStats(this._runner.stats), + stats: this._serializeStats(), error: this._failedWithError, remaining: [...this._remaining], - total: this._runner.stats.tests + total: this._passes + this._failures + this._pending }); }); await result; @@ -166,14 +155,11 @@ export class TestRunner extends EventEmitter { return true; } - _testWrapper(fn, title, file, isSlow) { - const timeout = isSlow ? this._timeout * 3 : this._timeout; - const wrapped = fixturePool.wrapTestCallback(fn, timeout, { + _testWrapper(test: Test, fn: Function) { + const timeout = test.slow ? this._timeout * 3 : this._timeout; + const wrapped = fixturePool.wrapTestCallback(fn, timeout, test, { outputDir: this._outDir, testDir: this._testDir, - title, - file, - timeout }); return wrapped ? (done, ...args) => { if (!this._shouldRunTest()) { @@ -199,12 +185,12 @@ export class TestRunner extends EventEmitter { }; } - _serializeStats(stats) { + _serializeStats() { return { passes: this._passes, failures: this._failures, pending: this._pending, - duration: stats.duration || 0, + duration: this._runner.duration(), } } } diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index d87bc2e867..f76a03f150 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -184,13 +184,13 @@ registerFixture('context', async ({browser}, test) => { await context.close(); }); -registerFixture('page', async ({context}, test) => { +registerFixture('page', async ({context}, runTest) => { const page = await context.newPage(); - const { success, info } = await test(page); + const { success, test, config } = await runTest(page); if (!success) { - const relativePath = path.relative(info.testDir, info.file).replace(/\.spec\.[jt]s/, ''); - const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); - const assetPath = path.join(info.outputDir, relativePath, sanitizedTitle) + '-failed.png'; + const relativePath = path.relative(config.testDir, test.file).replace(/\.spec\.[jt]s/, ''); + const sanitizedTitle = test.title.replace(/[^\w\d]+/g, '_'); + const assetPath = path.join(config.outputDir, relativePath, sanitizedTitle) + '-failed.png'; await page.screenshot({ path: assetPath }); } });