diff --git a/test-runner/src/fixturesUI.ts b/test-runner/src/fixturesUI.ts index 7789b89cdc..19aa6ea939 100644 --- a/test-runner/src/fixturesUI.ts +++ b/test-runner/src/fixturesUI.ts @@ -14,14 +14,11 @@ * limitations under the License. */ -import Mocha from 'mocha'; import { Test, Suite } from './test'; import { installTransform } from './transform'; Error.stackTraceLimit = 15; -let revertBabelRequire: () => void; - function specBuilder(modifiers, specCallback) { function builder(specs, last) { const callable = (...args) => { @@ -52,65 +49,54 @@ function specBuilder(modifiers, specCallback) { return builder({}, null); } -export function fixturesUI(options, mochaSuite: any) { - const suites = [mochaSuite.__nomocha as Suite]; +export function fixturesUI(suite: Suite, file: string, timeout: number): () => void { + const suites = [suite]; - mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file) { - const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => { - const suite = suites[0]; - const test = new Test(title, fn); - test.file = file; - test.slow = specs.slow && specs.slow[0]; - test.timeout = options.timeout; + const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => { + const suite = suites[0]; + const test = new Test(title, fn); + test.file = file; + test.slow = specs.slow && specs.slow[0]; + test.timeout = timeout; - const only = specs.only && specs.only[0]; - if (only) - test.only = true; - if (!only && specs.skip && specs.skip[0]) - test.pending = true; - if (!only && specs.fail && specs.fail[0]) - test.pending = true; - - test.pending = test.pending || suite.isPending(); - if (test.pending) - fn = null; - const wrapper = fn ? options.testWrapper(test, fn) : undefined; - if (wrapper) - wrapper.toString = () => fn.toString(); - test._materialize(wrapper); - suite.addTest(test); - return test; - }); - - const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => { - const child = new Suite(title, suites[0]); - suites[0].addSuite(child); - child.file = file; - const only = specs.only && specs.only[0]; - if (only) - child.only = true; - if (!only && specs.skip && specs.skip[0]) - child.pending = true; - if (!only && specs.fail && specs.fail[0]) - child.pending = true; - suites.unshift(child); - fn(); - suites.shift(); - }); - - context.beforeEach = fn => options.hookWrapper(mochaSuite.beforeEach.bind(mochaSuite), fn); - context.afterEach = fn => options.hookWrapper(mochaSuite.afterEach.bind(mochaSuite), fn); - context.describe = describe; - (context as any).fdescribe = describe.only(true); - context.xdescribe = describe.skip(true); - context.it = it; - (context as any).fit = it.only(true); - context.xit = it.skip(true); - - revertBabelRequire = installTransform(); + const only = specs.only && specs.only[0]; + if (only) + test.only = true; + if (!only && specs.skip && specs.skip[0]) + test.pending = true; + if (!only && specs.fail && specs.fail[0]) + test.pending = true; + test.pending = test.pending || suite.isPending(); + suite.addTest(test); + return test; }); - mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) { - revertBabelRequire(); + const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => { + const child = new Suite(title, suites[0]); + suites[0].addSuite(child); + child.file = file; + const only = specs.only && specs.only[0]; + if (only) + child.only = true; + if (!only && specs.skip && specs.skip[0]) + child.pending = true; + if (!only && specs.fail && specs.fail[0]) + child.pending = true; + suites.unshift(child); + fn(); + suites.shift(); }); -}; + + (global as any).beforeEach = fn => suite._addHook('beforeEach', fn); + (global as any).afterEach = fn => suite._addHook('afterEach', fn); + (global as any).beforeAll = fn => suite._addHook('beforeAll', fn); + (global as any).afterAll = fn => suite._addHook('afterAll', fn); + (global as any).describe = describe; + (global as any).fdescribe = describe.only(true); + (global as any).xdescribe = describe.skip(true); + (global as any).it = it; + (global as any).fit = it.only(true); + (global as any).xit = it.skip(true); + + return installTransform(); +} diff --git a/test-runner/src/runner.js b/test-runner/src/runner.js index 62bfc4bf1d..925fae8bdd 100644 --- a/test-runner/src/runner.js +++ b/test-runner/src/runner.js @@ -18,15 +18,10 @@ const child_process = require('child_process'); const crypto = require('crypto'); const path = require('path'); const { EventEmitter } = require('events'); -const Mocha = require('mocha'); const builtinReporters = require('mocha/lib/reporters'); const DotRunner = require('./dotReporter'); const { lookupRegistrations } = require('./fixtures'); -const constants = Mocha.Runner.constants; -// Mocha runner does not remove uncaughtException listeners. -process.setMaxListeners(0); - class Runner extends EventEmitter { constructor(suite, total, options) { super(); @@ -81,12 +76,12 @@ class Runner extends EventEmitter { } async run() { - this.emit(constants.EVENT_RUN_BEGIN, {}); + this.emit('start', {}); this._queue = this._filesSortedByWorkerHash(); // Loop in case job schedules more jobs while (this._queue.length) await this._dispatchQueue(); - this.emit(constants.EVENT_RUN_END, {}); + this.emit('end', {}); } async _dispatchQueue() { @@ -109,16 +104,11 @@ class Runner extends EventEmitter { let doneCallback; const result = new Promise(f => doneCallback = f); worker.once('done', params => { - this.stats.duration += params.stats.duration; - this.stats.failures += params.stats.failures; - this.stats.passes += params.stats.passes; - this.stats.pending += params.stats.pending; - this.stats.tests += params.stats.passes + params.stats.pending + params.stats.failures; // When worker encounters error, we will restart it. - if (params.error) { + if (params.error || params.fatalError) { this._restartWorker(worker); // If there are remaining tests, we will queue them. - if (params.remaining.length) + if (params.remaining.length && !params.fatalError) this._queue.unshift({ ...entry, ordinals: params.remaining }); } else { this._workerAvailable(worker); @@ -150,17 +140,28 @@ class Runner extends EventEmitter { _createWorker() { const worker = this._options.debug ? new InProcessWorker(this) : new OopWorker(this); - worker.on('test', params => this.emit(constants.EVENT_TEST_BEGIN, this._updateTest(params.test))); - worker.on('pending', params => this.emit(constants.EVENT_TEST_PENDING, this._updateTest(params.test))); - worker.on('pass', params => this.emit(constants.EVENT_TEST_PASS, this._updateTest(params.test))); + worker.on('test', params => { + ++this.stats.tests; + this.emit('test', this._updateTest(params.test)); + }); + worker.on('pending', params => { + ++this.stats.tests; + ++this.stats.pending; + this.emit('pending', this._updateTest(params.test)); + }); + worker.on('pass', params => { + ++this.stats.passes; + this.emit('pass', this._updateTest(params.test)); + }); worker.on('fail', params => { + ++this.stats.failures; const out = worker.takeOut(); if (out.length) params.error.stack += '\n\x1b[33mstdout: ' + out.join('\n') + '\x1b[0m'; const err = worker.takeErr(); if (err.length) params.error.stack += '\n\x1b[33mstderr: ' + err.join('\n') + '\x1b[0m'; - this.emit(constants.EVENT_TEST_FAIL, this._updateTest(params.test), params.error); + this.emit('fail', this._updateTest(params.test), params.error); }); worker.on('exit', () => { this._workers.delete(worker); diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index c69b187bdb..7cc972268f 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -import Mocha from 'mocha'; -import { fixturesUI } from './fixturesUI'; -import { EventEmitter } from 'events'; - export type Configuration = { name: string, value: string }[]; export class Test { @@ -34,18 +30,13 @@ export class Test { _configurationObject: Configuration; _configurationString: string; _overriddenFn: Function; - _impl: any; + _startTime: number; constructor(title: string, fn: Function) { this.title = title; this.fn = fn; } - _materialize(overriddenFn: Function) { - this._impl = new Mocha.Test(this.title, overriddenFn); - this._impl.pending = this.pending; - } - clone(): Test { const test = new Test(this.title, this.fn); test.suite = this.suite; @@ -54,7 +45,6 @@ export class Test { test.pending = this.pending; test.timeout = this.timeout; test._overriddenFn = this._overriddenFn; - test._materialize(this._overriddenFn); return test; } @@ -80,13 +70,12 @@ export class Suite { pending = false; file: string; - _impl: any; + _hooks: { type: string, fn: Function } [] = []; + _entries: (Suite | Test)[] = []; constructor(title: string, parent?: Suite) { this.title = title; this.parent = parent; - this._impl = new Mocha.Suite(title, new Mocha.Context()); - this._impl.__nomocha = this; } titlePath(): string[] { @@ -97,7 +86,9 @@ export class Suite { total(): number { let count = 0; - this.eachTest(fn => ++count); + this.eachTest(fn => { + ++count; + }); return count; } @@ -108,20 +99,25 @@ export class Suite { addTest(test: Test) { test.suite = this; this.tests.push(test); - this._impl.addTest(test._impl); + this._entries.push(test); } addSuite(suite: Suite) { suite.parent = this; this.suites.push(suite); - this._impl.addSuite(suite._impl); + this._entries.push(suite); } - eachTest(fn: (test: Test) => void) { - for (const suite of this.suites) - suite.eachTest(fn); - for (const test of this.tests) - fn(test); + eachTest(fn: (test: Test) => boolean | void): boolean { + for (const suite of this.suites) { + if (suite.eachTest(fn)) + return true; + } + for (const test of this.tests) { + if (fn(test)) + return true; + } + return false; } clone(): Suite { @@ -129,84 +125,29 @@ export class Suite { suite.only = this.only; suite.file = this.file; suite.pending = this.pending; - suite._impl = this._impl.clone(); return suite; } -} -class NullReporter { - stats = { - suites: 0, - tests: 0, - passes: 0, - pending: 0, - failures: 0 - }; - runner = null; - failures = []; - epilogue: () => {}; -} - -type NoMockaOptions = { - forbidOnly?: boolean; - timeout: number; - testWrapper: (test: Test, fn: Function) => Function; - hookWrapper: (hook: any, fn: Function) => Function; -}; - -class PatchedMocha extends Mocha { - suite: any; - static pendingSuite: Suite; - - constructor(suite, options) { - PatchedMocha.pendingSuite = suite; - super(options); - } - - grep(...args) { - this.suite = new Mocha.Suite('', new Mocha.Context()); - this.suite.__nomocha = PatchedMocha.pendingSuite; - PatchedMocha.pendingSuite._impl = this.suite; - return super.grep(...args); - } -} - -export class Runner extends EventEmitter { - private _mochaRunner: any; - - constructor(mochaRunner: any) { - super(); - const constants = Mocha.Runner.constants; - this._mochaRunner = mochaRunner; - this._mochaRunner.on(constants.EVENT_TEST_BEGIN, test => this.emit('test', test)); - this._mochaRunner.on(constants.EVENT_TEST_PENDING, test => this.emit('pending', test)); - this._mochaRunner.on(constants.EVENT_TEST_PASS, test => this.emit('pass', test)); - this._mochaRunner.on(constants.EVENT_TEST_FAIL, (test, err) => this.emit('fail', test, err)); - this._mochaRunner.on(constants.EVENT_RUN_END, () => this.emit('done')); - } - - duration(): number { - return this._mochaRunner.stats.duration || 0; - } -} - -export class NoMocha { - suite: Suite; - private _mocha: Mocha; - - constructor(file: string, options: NoMockaOptions) { - this.suite = new Suite(''); - this._mocha = new PatchedMocha(this.suite, { - forbidOnly: options.forbidOnly, - reporter: NullReporter, - timeout: options.timeout, - ui: fixturesUI.bind(null, options) + _renumber() { + let ordinal = 0; + this.eachTest((test: Test) => { + // All tests are identified with their ordinals. + test._ordinal = ordinal++; }); - this._mocha.addFile(file); - (this._mocha as any).loadFiles(); } - run(cb: () => void): Runner { - return new Runner(this._mocha.run(cb)); + _addHook(type: string, fn: any) { + this._hooks.push({ type, fn }); + } + + _hasTestsToRun(): boolean { + let found = false; + this.eachTest(test => { + if (!test.pending) { + found = true; + return true; + } + }); + return found; } } diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts index 796f38ceef..e5583745ef 100644 --- a/test-runner/src/testCollector.ts +++ b/test-runner/src/testCollector.ts @@ -15,9 +15,9 @@ */ import path from 'path'; -import Mocha from 'mocha'; import { fixturesForCallback, registerWorkerFixture } from './fixtures'; -import { Configuration, NoMocha, Test, Suite } from './test'; +import { Configuration, Test, Suite } from './test'; +import { fixturesUI } from './fixturesUI'; export class TestCollector { suite: Suite; @@ -50,20 +50,15 @@ export class TestCollector { } _addFile(file) { - const noMocha = new NoMocha(file, { - forbidOnly: this._options.forbidOnly, - timeout: this._options.timeout, - testWrapper: (test, fn) => () => {}, - hookWrapper: (hook, fn) => () => {}, - }); + const suite = new Suite(''); + const revertBabelRequire = fixturesUI(suite, file, this._options.timeout); + require(file); + revertBabelRequire(); + suite._renumber(); const workerGeneratorConfigurations = new Map(); - let ordinal = 0; - noMocha.suite.eachTest((test: Test) => { - // All tests are identified with their ordinals. - test._ordinal = ordinal++; - + suite.eachTest((test: Test) => { // Get all the fixtures that the test needs. const fixtures = fixturesForCallback(test.fn); @@ -102,7 +97,7 @@ export class TestCollector { // Clone the suite as many times as there are worker hashes. // Only include the tests that requested these generations. for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) { - const clone = this._cloneSuite(noMocha.suite, configurationObject, configurationString, tests); + const clone = this._cloneSuite(suite, configurationObject, configurationString, tests); this.suite.addSuite(clone); clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : ''); } @@ -111,19 +106,22 @@ export class TestCollector { _cloneSuite(suite: Suite, configurationObject: Configuration, configurationString: string, tests: Set) { const copy = suite.clone(); copy.only = suite.only; - for (const child of suite.suites) - copy.addSuite(this._cloneSuite(child, configurationObject, configurationString, tests)); - for (const test of suite.tests) { - if (!tests.has(test)) - continue; - if (this._grep && !this._grep.test(test.fullTitle())) - continue; - const testCopy = test.clone(); - testCopy.only = test.only; - testCopy._ordinal = test._ordinal; - testCopy._configurationObject = configurationObject; - testCopy._configurationString = configurationString; - copy.addTest(testCopy); + for (const entry of suite._entries) { + if (entry instanceof Suite) { + copy.addSuite(this._cloneSuite(entry, configurationObject, configurationString, tests)); + } else { + const test = entry; + if (!tests.has(test)) + continue; + if (this._grep && !this._grep.test(test.fullTitle())) + continue; + const testCopy = test.clone(); + testCopy.only = test.only; + testCopy._ordinal = test._ordinal; + testCopy._configurationObject = configurationObject; + testCopy._configurationString = configurationString; + copy.addTest(testCopy); + } } return copy; } diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 56fc6e5f01..8244dfbce3 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -18,7 +18,8 @@ import path from 'path'; import { FixturePool, registerWorkerFixture, rerunRegistrations, setParameters } from './fixtures'; import { EventEmitter } from 'events'; import { setCurrentTestFile } from './expect'; -import { NoMocha, Runner, Test } from './test'; +import { Test, Suite } from './test'; +import { fixturesUI } from './fixturesUI'; export const fixturePool = new FixturePool(); @@ -31,19 +32,15 @@ export type TestRunnerEntry = { export class TestRunner extends EventEmitter { private _currentOrdinal = -1; - private _failedWithError = false; + private _failedWithError: Error | undefined; + private _fatalError: Error | undefined; private _file: any; private _ordinals: Set; private _remaining: Set; private _trialRun: any; - private _passes = 0; - private _failures = 0; - private _pending = 0; private _configuredFile: any; private _configurationObject: any; private _parsedGeneratorConfiguration: any = {}; - private _relativeTestFile: string; - private _runner: Runner; private _outDir: string; private _timeout: number; private _testDir: string; @@ -65,134 +62,114 @@ export class TestRunner extends EventEmitter { registerWorkerFixture(name, async ({}, test) => await test(value)); } this._parsedGeneratorConfiguration['parallelIndex'] = workerId; - this._relativeTestFile = path.relative(options.testDir, this._file); + setCurrentTestFile(path.relative(options.testDir, this._file)); } - async stop() { + stop() { this._trialRun = true; - return new Promise(f => this._runner.once('done', f)); } async run() { - let callback; - const result = new Promise(f => callback = f); setParameters(this._parsedGeneratorConfiguration); - const noMocha = new NoMocha(this._file, { - timeout: 0, - testWrapper: (test, fn) => this._testWrapper(test, fn), - hookWrapper: (hook, fn) => this._hookWrapper(hook, fn), - }); + const suite = new Suite(''); + const revertBabelRequire = fixturesUI(suite, this._file, this._timeout); + require(this._file); + revertBabelRequire(); + suite._renumber(); + rerunRegistrations(this._file, 'test'); - this._runner = noMocha.run(callback); + await this._runSuite(suite); + this._reportDone(); + } - this._runner.on('test', test => { - setCurrentTestFile(this._relativeTestFile); - if (this._failedWithError) - return; - const ordinal = ++this._currentOrdinal; - if (this._ordinals.size && !this._ordinals.has(ordinal)) - return; - this._remaining.delete(ordinal); - this.emit('test', { test: this._serializeTest(test, ordinal) }); - }); + private async _runSuite(suite: Suite) { + try { + await this._runHooks(suite, 'beforeAll', 'before'); + } catch (e) { + this._fatalError = e; + this._reportDone(); + } + for (const entry of suite._entries) { + if (entry instanceof Suite) { + await this._runSuite(entry); + } else { + await this._runTest(entry); + } + } + try { + await this._runHooks(suite, 'afterAll', 'after'); + } catch (e) { + this._fatalError = e; + this._reportDone(); + } + } - this._runner.on('pending', test => { - if (this._failedWithError) - return; - const ordinal = ++this._currentOrdinal; - if (this._ordinals.size && !this._ordinals.has(ordinal)) - return; - this._remaining.delete(ordinal); - ++this._pending; - this.emit('pending', { test: this._serializeTest(test, ordinal) }); - }); + private async _runTest(test: Test) { + if (this._failedWithError) + return false; + const ordinal = ++this._currentOrdinal; + if (this._ordinals.size && !this._ordinals.has(ordinal)) + return; + this._remaining.delete(ordinal); + if (test.pending) { + this.emit('pending', { test: this._serializeTest(test) }); + return; + } - this._runner.on('pass', test => { - if (this._failedWithError) - return; - - const ordinal = this._currentOrdinal; - if (this._ordinals.size && !this._ordinals.has(ordinal)) - return; - ++this._passes; - this.emit('pass', { test: this._serializeTest(test, ordinal) }); - }); - - this._runner.on('fail', (test, error) => { - if (this._failedWithError) - return; - ++this._failures; + this.emit('test', { test: this._serializeTest(test) }); + try { + await this._runHooks(test.suite, 'beforeEach', 'before'); + test._startTime = Date.now(); + if (!this._trialRun) + await this._testWrapper(test)(); + this.emit('pass', { test: this._serializeTest(test) }); + await this._runHooks(test.suite, 'afterEach', 'after'); + } catch (error) { this._failedWithError = error; this.emit('fail', { - test: this._serializeTest(test, this._currentOrdinal), + test: this._serializeTest(test), error: serializeError(error), }); - }); - - this._runner.once('done', async () => { - this.emit('done', { - stats: this._serializeStats(), - error: this._failedWithError, - remaining: [...this._remaining], - total: this._passes + this._failures + this._pending - }); - }); - await result; - } - - _shouldRunTest(hook = false) { - if (this._trialRun || this._failedWithError) - return false; - if (hook) { - // Hook starts before we bump the test ordinal. - if (!this._ordinals.has(this._currentOrdinal + 1)) - return false; - } else { - if (!this._ordinals.has(this._currentOrdinal)) - return false; } - return true; } - _testWrapper(test: Test, fn: Function) { + private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after') { + if (!suite._hasTestsToRun()) + return; + const all = []; + for (let s = suite; s; s = s.parent) { + const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn); + all.push(...funcs.reverse()); + } + if (dir === 'before') + all.reverse(); + for (const hook of all) + await fixturePool.resolveParametersAndRun(hook, 0); + } + + private _reportDone() { + this.emit('done', { + error: this._failedWithError, + fatalError: this._fatalError, + remaining: [...this._remaining], + }); + } + + private _testWrapper(test: Test) { const timeout = test.slow ? this._timeout * 3 : this._timeout; - const wrapped = fixturePool.wrapTestCallback(fn, timeout, test, { + return fixturePool.wrapTestCallback(test.fn, timeout, test, { outputDir: this._outDir, testDir: this._testDir, }); - return wrapped ? (done, ...args) => { - if (!this._shouldRunTest()) { - done(); - return; - } - wrapped(...args).then(done).catch(done); - } : undefined; } - _hookWrapper(hook, fn) { - if (!this._shouldRunTest(true)) - return; - return hook(async () => { - return await fixturePool.resolveParametersAndRun(fn, 0); - }); - } - - _serializeTest(test, ordinal) { + private _serializeTest(test) { return { - id: `${ordinal}@${this._configuredFile}`, - duration: test.duration, + id: `${test._ordinal}@${this._configuredFile}`, + duration: Date.now() - test._startTime, }; } - - _serializeStats() { - return { - passes: this._passes, - failures: this._failures, - pending: this._pending, - duration: this._runner.duration(), - } - } } function trimCycles(obj) { diff --git a/test/focus.spec.ts b/test/focus.spec.ts index 7c2df6b15a..921b66aaca 100644 --- a/test/focus.spec.ts +++ b/test/focus.spec.ts @@ -15,14 +15,14 @@ */ import { options } from './playwright.fixtures'; -it.skip(options.FIREFOX)('should work', async function({page, server}) { +it.skip(options.FIREFOX)('should work', async function({page}) { await page.setContent(`
`); expect(await page.evaluate(() => document.activeElement.nodeName)).toBe('BODY'); await page.focus('#d1'); expect(await page.evaluate(() => document.activeElement.id)).toBe('d1'); }); -it('should emit focus event', async function({page, server}) { +it('should emit focus event', async function({page}) { await page.setContent(`
`); let focused = false; await page.exposeFunction('focusEvent', () => focused = true); @@ -31,7 +31,7 @@ it('should emit focus event', async function({page, server}) { expect(focused).toBe(true); }); -it('should emit blur event', async function({page, server}) { +it('should emit blur event', async function({page}) { await page.setContent(`
DIV1
DIV2
`); await page.focus('#d1'); let focused = false;