test: translate tests into ts, extract mocha (#3565)

test-runner/src/cli.js

@@ -14,128 +14,4 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
+require('./lib/cli');
-const path = require('path');
-const program = require('commander');
-const { installTransform } = require('./transform');
-const { Runner } = require('./runner');
-const { TestCollector } = require('./testCollector');
-
-let beforeFunction;
-let afterFunction;
-let matrix = {};
-
-global['before'] = (fn => beforeFunction = fn);
-global['after'] = (fn => afterFunction = fn);
-global['matrix'] = (m => matrix = m);
-
-program
-  .version('Version ' + /** @type {any} */ (require)('../package.json').version)
-  .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
-  .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
-  .option('--reporter <reporter>', 'Specify reporter to use', '')
-  .option('--trial-run', 'Only collect the matching tests and report them as passing')
-  .option('--quiet', 'Suppress stdio', false)
-  .option('--debug', 'Run tests in-process for debugging', false)
-  .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
-  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
-  .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
-  .action(async (command) => {
-    // Collect files]
-    const testDir = path.resolve(process.cwd(), command.args[0]);
-    const files = collectFiles(testDir, '', command.args.slice(1));
-
-    const revertBabelRequire = installTransform();
-    let hasSetup = false;
-    try {
-      hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile();
-    } catch (e) {
-    }
-    try {
-      hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile();
-    } catch (e) {
-    }
-
-    if (hasSetup)
-      require(path.join(testDir, 'setup'));
-    revertBabelRequire();
-
-    const testCollector = new TestCollector(files, matrix, {
-      forbidOnly: command.forbidOnly || undefined,
-      grep: command.grep,
-      timeout: command.timeout,
-    });
-    const rootSuite = testCollector.suite;
-    if (command.forbidOnly && testCollector.hasOnly()) {
-      console.error('=====================================');
-      console.error(' --forbid-only found a focused test.');
-      console.error('=====================================');
-      process.exit(1);
-    }
-
-    const total = rootSuite.total();
-    if (!total) {
-      console.error('=================');
-      console.error(' no tests found.');
-      console.error('=================');
-      process.exit(1);
-    }
-
-    // Trial run does not need many workers, use one.
-    const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
-    const runner = new Runner(rootSuite, total, {
-      debug: command.debug,
-      quiet: command.quiet,
-      grep: command.grep,
-      jobs,
-      outputDir: command.output,
-      reporter: command.reporter,
-      retries: command.retries,
-      snapshotDir: path.join(testDir, '__snapshots__'),
-      testDir,
-      timeout: command.timeout,
-      trialRun: command.trialRun,
-      updateSnapshots: command.updateSnapshots
-    });
-    try {
-      if (beforeFunction)
-        await beforeFunction();
-      await runner.run();
-      await runner.stop();
-    } finally {
-      if (afterFunction)
-        await afterFunction();
-    }
-    process.exit(runner.stats.failures ? 1 : 0);
-  });
-
-program.parse(process.argv);
-
-function collectFiles(testDir, dir, filters) {
-  const fullDir = path.join(testDir, dir);
-  if (fs.statSync(fullDir).isFile())
-    return [fullDir];
-  const files = [];
-  for (const name of fs.readdirSync(fullDir)) {
-    if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) {
-      files.push(...collectFiles(testDir, path.join(dir, name), filters));
-      continue;
-    }
-    if (!name.endsWith('spec.ts'))
-      continue;
-    const relativeName = path.join(dir, name);
-    const fullName = path.join(testDir, relativeName);
-    if (!filters.length) {
-      files.push(fullName);
-      continue;
-    }
-    for (const filter of filters) {
-      if (relativeName.includes(filter)) {
-        files.push(fullName);
-        break;
-      }
-    }
-  }
-  return files;
-}

test-runner/src/cli.ts

@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import program from 'commander';
+import { installTransform } from './transform';
+import { Runner } from './runner';
+import { TestCollector } from './testCollector';
+
+let beforeFunction;
+let afterFunction;
+let matrix = {};
+
+global['before'] = (fn => beforeFunction = fn);
+global['after'] = (fn => afterFunction = fn);
+global['matrix'] = (m => matrix = m);
+
+program
+  .version('Version ' + /** @type {any} */ (require)('../package.json').version)
+  .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
+  .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
+  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
+  .option('--reporter <reporter>', 'Specify reporter to use', '')
+  .option('--trial-run', 'Only collect the matching tests and report them as passing')
+  .option('--quiet', 'Suppress stdio', false)
+  .option('--debug', 'Run tests in-process for debugging', false)
+  .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
+  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
+  .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
+  .action(async (command) => {
+    // Collect files]
+    const testDir = path.resolve(process.cwd(), command.args[0]);
+    const files = collectFiles(testDir, '', command.args.slice(1));
+
+    const revertBabelRequire = installTransform();
+    let hasSetup = false;
+    try {
+      hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile();
+    } catch (e) {
+    }
+    try {
+      hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile();
+    } catch (e) {
+    }
+
+    if (hasSetup)
+      require(path.join(testDir, 'setup'));
+    revertBabelRequire();
+
+    const testCollector = new TestCollector(files, matrix, {
+      forbidOnly: command.forbidOnly || undefined,
+      grep: command.grep,
+      timeout: command.timeout,
+    });
+    const rootSuite = testCollector.suite;
+    if (command.forbidOnly && testCollector.hasOnly()) {
+      console.error('=====================================');
+      console.error(' --forbid-only found a focused test.');
+      console.error('=====================================');
+      process.exit(1);
+    }
+
+    const total = rootSuite.total();
+    if (!total) {
+      console.error('=================');
+      console.error(' no tests found.');
+      console.error('=================');
+      process.exit(1);
+    }
+
+    // Trial run does not need many workers, use one.
+    const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
+    const runner = new Runner(rootSuite, total, {
+      debug: command.debug,
+      quiet: command.quiet,
+      grep: command.grep,
+      jobs,
+      outputDir: command.output,
+      reporter: command.reporter,
+      retries: command.retries,
+      snapshotDir: path.join(testDir, '__snapshots__'),
+      testDir,
+      timeout: command.timeout,
+      trialRun: command.trialRun,
+      updateSnapshots: command.updateSnapshots
+    });
+    try {
+      if (beforeFunction)
+        await beforeFunction();
+      await runner.run();
+      await runner.stop();
+    } finally {
+      if (afterFunction)
+        await afterFunction();
+    }
+    process.exit(runner.stats.failures ? 1 : 0);
+  });
+
+program.parse(process.argv);
+
+function collectFiles(testDir, dir, filters) {
+  const fullDir = path.join(testDir, dir);
+  if (fs.statSync(fullDir).isFile())
+    return [fullDir];
+  const files = [];
+  for (const name of fs.readdirSync(fullDir)) {
+    if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) {
+      files.push(...collectFiles(testDir, path.join(dir, name), filters));
+      continue;
+    }
+    if (!name.endsWith('spec.ts'))
+      continue;
+    const relativeName = path.join(dir, name);
+    const fullName = path.join(testDir, relativeName);
+    if (!filters.length) {
+      files.push(fullName);
+      continue;
+    }
+    for (const filter of filters) {
+      if (relativeName.includes(filter)) {
+        files.push(fullName);
+        break;
+      }
+    }
+  }
+  return files;
+}

test-runner/src/dotReporter.js

@@ -43,7 +43,7 @@ class DotReporter extends Base {
     });
 
     runner.on(constants.EVENT_TEST_FAIL, test => {
-      if (test.duration >= test.timeout())
+      if (test.duration >= test.timeout)
         process.stdout.write(colors.red('T'));
       else
         process.stdout.write(colors.red('F'));

test-runner/src/fixtures.ts

@@ -15,6 +15,7 @@
  */
 
 import debug from 'debug';
+import { Test } from './test';
 
 declare global {
   interface WorkerState {
@@ -38,17 +39,15 @@ export function setParameters(params: any) {
     registerWorkerFixture(name as keyof WorkerState, async ({}, test) => await test(parameters[name] as never));
 }
 
-type TestInfo = {
+type TestConfig = {
-  file: string;
-  title: string;
-  timeout: number;
   outputDir: string;
   testDir: string;
 };
 
 type TestResult = {
   success: boolean;
-  info: TestInfo;
+  test: Test;
+  config: TestConfig;
   error?: Error;
 };
 
@@ -168,10 +167,10 @@ export class FixturePool {
     ]);
   }
 
-  wrapTestCallback(callback: any, timeout: number, info: TestInfo) {
+  wrapTestCallback(callback: any, timeout: number, test: Test, config: TestConfig) {
     if (!callback)
       return callback;
-    const testResult: TestResult = { success: true, info };
+    const testResult: TestResult = { success: true, test, config };
     return async() => {
       try {
         await this.resolveParametersAndRun(callback, timeout);

test-runner/src/fixturesUI.ts

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-const { Test, Suite } = require('mocha');
+import Mocha from 'mocha';
-const { installTransform } = require('./transform');
+import { Test, Suite } from './test';
-const commonSuite = require('mocha/lib/interfaces/common');
+import { installTransform } from './transform';
 
 Error.stackTraceLimit = 15;
 
-let revertBabelRequire;
+let revertBabelRequire: () => void;
 
 function specBuilder(modifiers, specCallback) {
   function builder(specs, last) {
@@ -52,67 +52,65 @@ function specBuilder(modifiers, specCallback) {
   return builder({}, null);
 }
 
-function fixturesUI(wrappers, suite) {
+export function fixturesUI(options, mochaSuite: any) {
-  const suites = [suite];
+  const suites = [mochaSuite.__nomocha as Suite];
-
-  suite.on(Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file, mocha) {
-    const common = commonSuite(suites, context, mocha);
 
+  mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file) {
     const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => {
       const suite = suites[0];
-
+      const test = new Test(title, fn);
-      if (suite.isPending())
-        fn = null;
-      const wrapper = fn ? wrappers.testWrapper(fn, title, file, specs.slow && specs.slow[0]) : undefined;
-      if (wrapper) {
-        wrapper.toString = () => fn.toString();
-        wrapper.__original = fn;
-      }
-      const test = new Test(title, wrapper);
       test.file = file;
-      suite.addTest(test);
+      test.slow = specs.slow && specs.slow[0];
-      const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0];
+      test.timeout = options.timeout;
+
+      const only = specs.only && specs.only[0];
       if (only)
-        test.__only = true;
+        test.only = true;
       if (!only && specs.skip && specs.skip[0])
         test.pending = true;
       if (!only && specs.fail && specs.fail[0])
         test.pending = true;
+
+      test.pending = test.pending || suite.isPending();
+      if (test.pending)
+        fn = null;
+      const wrapper = fn ? options.testWrapper(test, fn) : undefined;
+      if (wrapper)
+        wrapper.toString = () => fn.toString();
+      test._materialize(wrapper);
+      suite.addTest(test);
       return test;
     });
 
     const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => {
-      const suite = common.suite.create({
+      const child = new Suite(title, suites[0]);
-        title: title,
+      suites[0].addSuite(child);
-        file: file,
+      child.file = file;
-        fn: fn
+      const only = specs.only && specs.only[0];
-      });
-      const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0];
       if (only)
-        suite.__only = true;
+        child.only = true;
       if (!only && specs.skip && specs.skip[0])
-        suite.pending = true;
+        child.pending = true;
       if (!only && specs.fail && specs.fail[0])
-        suite.pending = true;
+        child.pending = true;
-      return suite;
+      suites.unshift(child);
+      fn();
+      suites.shift();
     });
 
-    context.beforeEach = fn => wrappers.hookWrapper(common.beforeEach.bind(common), fn);
+    context.beforeEach = fn => options.hookWrapper(mochaSuite.beforeEach.bind(mochaSuite), fn);
-    context.afterEach = fn => wrappers.hookWrapper(common.afterEach.bind(common), fn);
+    context.afterEach = fn => options.hookWrapper(mochaSuite.afterEach.bind(mochaSuite), fn);
-    context.run = mocha.options.delay && common.runWithSuite(suite);
     context.describe = describe;
-    context.fdescribe = describe.only(true);
+    (context as any).fdescribe = describe.only(true);
     context.xdescribe = describe.skip(true);
     context.it = it;
-    context.fit = it.only(true);
+    (context as any).fit = it.only(true);
     context.xit = it.skip(true);
 
     revertBabelRequire = installTransform();
   });
 
-  suite.on(Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) {
+  mochaSuite.on(Mocha.Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) {
     revertBabelRequire();
   });
 };
-
-module.exports = { fixturesUI };

test-runner/src/index.ts

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import './builtin.fixtures';
 import './expect';
-export {registerFixture, registerWorkerFixture, registerParameter, parameters} from './fixtures';
+export {registerFixture, registerWorkerFixture, registerParameter, parameters} from './fixtures';

test-runner/src/runner.js

@@ -50,19 +50,19 @@ class Runner extends EventEmitter {
     this._testsByConfiguredFile = new Map();
 
     suite.eachTest(test => {
-      const configuredFile = `${test.file}::[${test.__configurationString}]`;
+      const configuredFile = `${test.file}::[${test._configurationString}]`;
       if (!this._testsByConfiguredFile.has(configuredFile)) {
         this._testsByConfiguredFile.set(configuredFile, {
           file: test.file,
           configuredFile,
           ordinals: [],
-          configurationObject: test.__configurationObject,
+          configurationObject: test._configurationObject,
-          configurationString: test.__configurationString
+          configurationString: test._configurationString
         });
       }
       const { ordinals } = this._testsByConfiguredFile.get(configuredFile);
-      ordinals.push(test.__ordinal);
+      ordinals.push(test._ordinal);
-      this._testById.set(`${test.__ordinal}@${configuredFile}`, test);
+      this._testById.set(`${test._ordinal}@${configuredFile}`, test);
     });
 
     if (process.stdout.isTTY) {

test-runner/src/test.ts

@@ -0,0 +1,212 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Mocha from 'mocha';
+import { fixturesUI } from './fixturesUI';
+import { EventEmitter } from 'events';
+
+export type Configuration = { name: string, value: string }[];
+
+export class Test {
+  suite: Suite;
+  title: string;
+  file: string;
+  only = false;
+  pending = false;
+  duration = 0;
+  timeout = 0;
+  fn: Function;
+
+  _ordinal: number;
+  _configurationObject: Configuration;
+  _configurationString: string;
+  _overriddenFn: Function;
+  _impl: any;
+
+  constructor(title: string, fn: Function) {
+    this.title = title;
+    this.fn = fn;
+  }
+
+  _materialize(overriddenFn: Function) {
+    this._impl = new Mocha.Test(this.title, overriddenFn);
+    this._impl.pending = this.pending;
+  }
+
+  clone(): Test {
+    const test = new Test(this.title, this.fn);
+    test.suite = this.suite;
+    test.only = this.only;
+    test.file = this.file;
+    test.pending = this.pending;
+    test.timeout = this.timeout;
+    test._overriddenFn = this._overriddenFn;
+    test._materialize(this._overriddenFn);
+    return test;
+  }
+
+  titlePath(): string[] {
+    return [...this.suite.titlePath(), this.title];
+  }
+
+  fullTitle(): string {
+    return this.titlePath().join(' ');
+  }
+
+  slow(): number {
+    return 10000;
+  }
+}
+
+export class Suite {
+  title: string;
+  parent?: Suite;
+  suites: Suite[] = [];
+  tests: Test[] = [];
+  only = false;
+  pending = false;
+  file: string;
+
+  _impl: any;
+
+  constructor(title: string, parent?: Suite) {
+    this.title = title;
+    this.parent = parent;
+    this._impl = new Mocha.Suite(title, new Mocha.Context());
+    this._impl.__nomocha = this;
+  }
+
+  titlePath(): string[] {
+    if (!this.parent)
+      return [this.title];
+    return [...this.parent.titlePath(), this.title];
+  }
+
+  total(): number {
+    let count = 0;
+    this.eachTest(fn => ++count);
+    return count;
+  }
+
+  isPending(): boolean {
+    return this.pending || (this.parent && this.parent.isPending());
+  }
+
+  addTest(test: Test) {
+    test.suite = this;
+    this.tests.push(test);
+    this._impl.addTest(test._impl);
+  }
+
+  addSuite(suite: Suite) {
+    suite.parent = this;
+    this.suites.push(suite);
+    this._impl.addSuite(suite._impl);
+  }
+
+  eachTest(fn: (test: Test) => void) {
+    for (const suite of this.suites)
+      suite.eachTest(fn);
+    for (const test of this.tests)
+      fn(test);
+  }
+
+  clone(): Suite {
+    const suite = new Suite(this.title);
+    suite.only = this.only;
+    suite.file = this.file;
+    suite.pending = this.pending;
+    suite._impl = this._impl.clone();
+    return suite;
+  }
+}
+
+class NullReporter {
+  stats = {
+    suites: 0,
+    tests: 0,
+    passes: 0,
+    pending: 0,
+    failures: 0
+  };
+  runner = null;
+  failures = [];
+  epilogue: () => {};
+}
+
+type NoMockaOptions = {
+  forbidOnly?: boolean;
+  timeout: number;
+  testWrapper: (test: Test, fn: Function) => Function;
+  hookWrapper: (hook: any, fn: Function) => Function;
+};
+
+class PatchedMocha extends Mocha {
+  suite: any;
+  static pendingSuite: Suite;
+
+  constructor(suite, options) {
+    PatchedMocha.pendingSuite = suite;
+    super(options);
+  }
+
+  grep(...args) {
+    this.suite = new Mocha.Suite('', new Mocha.Context());
+    this.suite.__nomocha = PatchedMocha.pendingSuite;
+    PatchedMocha.pendingSuite._impl = this.suite;
+    return super.grep(...args);
+  }
+}
+
+export class Runner extends EventEmitter {
+  private _mochaRunner: any;
+
+  constructor(mochaRunner: any) {
+    super();
+    const constants = Mocha.Runner.constants;
+    this._mochaRunner = mochaRunner;
+    this._mochaRunner.on(constants.EVENT_TEST_BEGIN, test => this.emit('test', test));
+    this._mochaRunner.on(constants.EVENT_TEST_PENDING, test => this.emit('pending', test));
+    this._mochaRunner.on(constants.EVENT_TEST_PASS, test => this.emit('pass', test));
+    this._mochaRunner.on(constants.EVENT_TEST_FAIL, (test, err) => this.emit('fail', test, err));
+    this._mochaRunner.on(constants.EVENT_RUN_END, () => this.emit('done'));
+  }
+
+  duration(): number {
+    return this._mochaRunner.stats.duration || 0;
+  }
+}
+
+export class NoMocha {
+  suite: Suite;
+  private _mocha: Mocha;
+
+  constructor(file: string, options: NoMockaOptions) {
+    this.suite = new Suite('');
+    this._mocha = new PatchedMocha(this.suite, {
+      forbidOnly: options.forbidOnly,
+      reporter: NullReporter,
+      timeout: options.timeout,
+      ui: fixturesUI.bind(null, options)
+    });
+    this._mocha.addFile(file);
+    (this._mocha as any).loadFiles();
+  }
+
+  run(cb: () => void): Runner {
+    return new Runner(this._mocha.run(cb));
+  }
+}

test-runner/src/testCollector.ts

@@ -14,22 +14,26 @@
  * limitations under the License.
  */
 
-const path = require('path');
+import path from 'path';
-const Mocha = require('mocha');
+import Mocha from 'mocha';
-const { fixturesForCallback, registerWorkerFixture } = require('./fixtures');
+import { fixturesForCallback, registerWorkerFixture } from './fixtures';
-const { fixturesUI } = require('./fixturesUI');
+import { Configuration, NoMocha, Test, Suite } from './test';
 
-class NullReporter {}
+export class TestCollector {
+  suite: Suite;
 
-class TestCollector {
+  private _matrix: { [key: string]: string; };
-  constructor(files, matrix, options) {
+  private _options: any;
+  private _grep: RegExp;
+  private _hasOnly: boolean;
+
+  constructor(files: string[], matrix: { [key: string] : string }, options) {
     this._matrix = matrix;
     for (const name of Object.keys(matrix))
       //@ts-ignore
       registerWorkerFixture(name, async ({}, test) => test());
     this._options = options;
-    this.suite = new Mocha.Suite('', new Mocha.Context(), true);
+    this.suite = new Suite('');
-    this._total = 0;
     if (options.grep) {
       const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/);
       this._grep = new RegExp(match[1] || match[0], match[2]);
@@ -46,28 +50,22 @@ class TestCollector {
   }
 
   _addFile(file) {
-    const mocha = new Mocha({
+    const noMocha = new NoMocha(file, {
       forbidOnly: this._options.forbidOnly,
-      reporter: NullReporter,
       timeout: this._options.timeout,
-      ui: fixturesUI.bind(null, {
+      testWrapper: (test, fn) => () => {},
-        testWrapper: (fn) => done => done(),
+      hookWrapper: (hook, fn) => () => {},
-        hookWrapper: (hook, fn) => {},
-        ignoreOnly: false,
-      }),
     });
-    mocha.addFile(file);
-    mocha.loadFiles();
 
     const workerGeneratorConfigurations = new Map();
 
     let ordinal = 0;
-    mocha.suite.eachTest(test => {
+    noMocha.suite.eachTest((test: Test) => {
       // All tests are identified with their ordinals.
-      test.__ordinal = ordinal++;
+      test._ordinal = ordinal++;
 
       // Get all the fixtures that the test needs.
-      const fixtures = fixturesForCallback(test.fn.__original);
+      const fixtures = fixturesForCallback(test.fn);
 
       // For generator fixtures, collect all variants of the fixture values
       // to build different workers for them.
@@ -104,15 +102,15 @@ class TestCollector {
     // Clone the suite as many times as there are worker hashes.
     // Only include the tests that requested these generations.
     for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
-      const clone = this._cloneSuite(mocha.suite, configurationObject, configurationString, tests);
+      const clone = this._cloneSuite(noMocha.suite, configurationObject, configurationString, tests);
       this.suite.addSuite(clone);
       clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '');
     }
   }
 
-  _cloneSuite(suite, configurationObject, configurationString, tests) {
+  _cloneSuite(suite: Suite, configurationObject: Configuration, configurationString: string, tests: Set<Test>) {
     const copy = suite.clone();
-    copy.__only = suite.__only;
+    copy.only = suite.only;
     for (const child of suite.suites)
       copy.addSuite(this._cloneSuite(child, configurationObject, configurationString, tests));
     for (const test of suite.tests) {
@@ -121,18 +119,18 @@ class TestCollector {
       if (this._grep && !this._grep.test(test.fullTitle()))
         continue;
       const testCopy = test.clone();
-      testCopy.__only = test.__only;
+      testCopy.only = test.only;
-      testCopy.__ordinal = test.__ordinal;
+      testCopy._ordinal = test._ordinal;
-      testCopy.__configurationObject = configurationObject;
+      testCopy._configurationObject = configurationObject;
-      testCopy.__configurationString = configurationString;
+      testCopy._configurationString = configurationString;
       copy.addTest(testCopy);
     }
     return copy;
   }
 
   _filterOnly(suite) {
-    const onlySuites = suite.suites.filter(child => this._filterOnly(child) || child.__only);
+    const onlySuites = suite.suites.filter(child => this._filterOnly(child) || child.only);
-    const onlyTests = suite.tests.filter(test => test.__only);
+    const onlyTests = suite.tests.filter(test => test.only);
     if (onlySuites.length || onlyTests.length) {
       suite.suites = onlySuites;
       suite.tests = onlyTests;

test-runner/src/testRunner.ts

@@ -15,11 +15,10 @@
  */
 
 import path from 'path';
-import Mocha from 'mocha';
 import { FixturePool, registerWorkerFixture, rerunRegistrations, setParameters } from './fixtures';
-import { fixturesUI } from './fixturesUI';
 import { EventEmitter } from 'events';
 import { setCurrentTestFile } from './expect';
+import { NoMocha, Runner, Test } from './test';
 
 export const fixturePool = new FixturePool();
 
@@ -30,10 +29,7 @@ export type TestRunnerEntry = {
   configurationObject: any;
 };
 
-class NullReporter {}
-
 export class TestRunner extends EventEmitter {
-  mocha: any;
   private _currentOrdinal = -1;
   private _failedWithError = false;
   private _file: any;
@@ -47,22 +43,13 @@ export class TestRunner extends EventEmitter {
   private _configurationObject: any;
   private _parsedGeneratorConfiguration: any = {};
   private _relativeTestFile: string;
-  private _runner: Mocha.Runner;
+  private _runner: Runner;
   private _outDir: string;
   private _timeout: number;
   private _testDir: string;
 
   constructor(entry: TestRunnerEntry, options, workerId) {
     super();
-    this.mocha = new Mocha({
-      reporter: NullReporter,
-      timeout: 0,
-      ui: fixturesUI.bind(null, {
-        testWrapper: (fn, title, file, isSlow) => this._testWrapper(fn, title, file, isSlow),
-        hookWrapper: (hook, fn) => this._hookWrapper(hook, fn),
-        ignoreOnly: true
-      }),
-    });
     this._file = entry.file;
     this._ordinals = new Set(entry.ordinals);
     this._remaining = new Set(entry.ordinals);
@@ -79,25 +66,27 @@ export class TestRunner extends EventEmitter {
     }
     this._parsedGeneratorConfiguration['parallelIndex'] = workerId;
     this._relativeTestFile = path.relative(options.testDir, this._file);
-    this.mocha.addFile(this._file);
   }
 
   async stop() {
     this._trialRun = true;
-    const constants = Mocha.Runner.constants;
+    return new Promise(f => this._runner.once('done', f));
-    return new Promise(f => this._runner.once(constants.EVENT_RUN_END, f));
   }
 
   async run() {
     let callback;
     const result = new Promise(f => callback = f);
     setParameters(this._parsedGeneratorConfiguration);
-    this.mocha.loadFiles();
-    rerunRegistrations(this._file, 'test');
-    this._runner = this.mocha.run(callback);
 
-    const constants = Mocha.Runner.constants;
+    const noMocha = new NoMocha(this._file, {
-    this._runner.on(constants.EVENT_TEST_BEGIN, test => {
+      timeout: 0,
+      testWrapper: (test, fn) => this._testWrapper(test, fn),
+      hookWrapper: (hook, fn) => this._hookWrapper(hook, fn),
+    });
+    rerunRegistrations(this._file, 'test');
+    this._runner = noMocha.run(callback);
+
+    this._runner.on('test', test => {
       setCurrentTestFile(this._relativeTestFile);
       if (this._failedWithError)
         return;
@@ -108,7 +97,7 @@ export class TestRunner extends EventEmitter {
       this.emit('test', { test: this._serializeTest(test, ordinal) });
     });
 
-    this._runner.on(constants.EVENT_TEST_PENDING, test => {
+    this._runner.on('pending', test => {
       if (this._failedWithError)
         return;
       const ordinal = ++this._currentOrdinal;
@@ -119,7 +108,7 @@ export class TestRunner extends EventEmitter {
       this.emit('pending', { test: this._serializeTest(test, ordinal) });
     });
 
-    this._runner.on(constants.EVENT_TEST_PASS, test => {
+    this._runner.on('pass', test => {
       if (this._failedWithError)
         return;
 
@@ -130,7 +119,7 @@ export class TestRunner extends EventEmitter {
       this.emit('pass', { test: this._serializeTest(test, ordinal) });
     });
 
-    this._runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
+    this._runner.on('fail', (test, error) => {
       if (this._failedWithError)
         return;
       ++this._failures;
@@ -141,12 +130,12 @@ export class TestRunner extends EventEmitter {
       });
     });
 
-    this._runner.once(constants.EVENT_RUN_END, async () => {
+    this._runner.once('done', async () => {
       this.emit('done', {
-        stats: this._serializeStats(this._runner.stats),
+        stats: this._serializeStats(),
         error: this._failedWithError,
         remaining: [...this._remaining],
-        total: this._runner.stats.tests
+        total: this._passes + this._failures + this._pending
       });
     });
     await result;
@@ -166,14 +155,11 @@ export class TestRunner extends EventEmitter {
     return true;
   }
 
-  _testWrapper(fn, title, file, isSlow) {
+  _testWrapper(test: Test, fn: Function) {
-    const timeout = isSlow ? this._timeout * 3 : this._timeout;
+    const timeout = test.slow ? this._timeout * 3 : this._timeout;
-    const wrapped = fixturePool.wrapTestCallback(fn, timeout, {
+    const wrapped = fixturePool.wrapTestCallback(fn, timeout, test, {
       outputDir: this._outDir,
       testDir: this._testDir,
-      title,
-      file,
-      timeout
     });
     return wrapped ? (done, ...args) => {
       if (!this._shouldRunTest()) {
@@ -199,12 +185,12 @@ export class TestRunner extends EventEmitter {
     };
   }
   
-  _serializeStats(stats) {
+  _serializeStats() {
     return {
       passes: this._passes,
       failures: this._failures,
       pending: this._pending,
-      duration: stats.duration || 0,
+      duration: this._runner.duration(),
     }
   }  
 }

test/playwright.fixtures.ts

@@ -184,13 +184,13 @@ registerFixture('context', async ({browser}, test) => {
   await context.close();
 });
 
-registerFixture('page', async ({context}, test) => {
+registerFixture('page', async ({context}, runTest) => {
   const page = await context.newPage();
-  const { success, info } = await test(page);
+  const { success, test, config } = await runTest(page);
   if (!success) {
-    const relativePath = path.relative(info.testDir, info.file).replace(/\.spec\.[jt]s/, '');
+    const relativePath = path.relative(config.testDir, test.file).replace(/\.spec\.[jt]s/, '');
-    const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
+    const sanitizedTitle = test.title.replace(/[^\w\d]+/g, '_');
-    const assetPath = path.join(info.outputDir, relativePath, sanitizedTitle) + '-failed.png';
+    const assetPath = path.join(config.outputDir, relativePath, sanitizedTitle) + '-failed.png';
     await page.screenshot({ path: assetPath });
   }
 });