test: implement in-process debug mode (#3486)

test/runner/index.js

@@ -31,6 +31,7 @@ program
   .option('--reporter <reporter>', 'Specify reporter to use', '')
   .option('--trial-run', 'Only collect the matching tests and report them as passing')
   .option('--dumpio', 'Dump stdout and stderr from workers', false)
+  .option('--debug', 'Run tests in-process for debugging', false)
   .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000)
   .action(async (command) => {
     // Collect files
@@ -50,11 +51,8 @@ program
       if (command.grep)
         mocha.grep(command.grep);
       mocha.addFile(file);
-      let runner;
-      await new Promise(f => {
-        runner = mocha.run(f);
-      });
-      total += runner.grepTotal(mocha.suite);
+      mocha.loadFiles();
+      total += grepTotal(mocha.suite, mocha.options.grep);
 
       rootSuite.addSuite(mocha.suite);
       mocha.suite.title = path.basename(file);
@@ -75,8 +73,9 @@ program
     }
 
     // Trial run does not need many workers, use one.
-    const jobs = command.trialRun ? 1 : command.jobs;
+    const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
     const runner = new Runner(rootSuite, {
+      debug: command.debug,
       dumpio: command.dumpio,
       grep: command.grep,
       jobs,
@@ -101,7 +100,7 @@ function collectFiles(dir, filters) {
       files.push(...collectFiles(path.join(dir, name), filters));
       continue;
     }
-    if (!name.includes('spec'))
+    if (!name.endsWith('spec.ts'))
       continue;
     if (!filters.length) {
       files.push(path.join(dir, name));
@@ -116,3 +115,12 @@ function collectFiles(dir, filters) {
   }
   return files;
 }
+
+function grepTotal(suite, grep) {
+  let total = 0;
+  suite.eachTest(test => {
+    if (grep.test(test.fullTitle()))
+      total++;
+  });
+  return total;
+}

test/runner/runner.js

@@ -20,7 +20,7 @@ const { EventEmitter } = require('events');
 const Mocha = require('mocha');
 const builtinReporters = require('mocha/lib/reporters');
 const DotRunner = require('./dotReporter');
-const { computeWorkerHash } = require('./fixtures');
+const { computeWorkerHash, FixturePool } = require('./fixtures');
 
 const constants = Mocha.Runner.constants;
 // Mocha runner does not remove uncaughtException listeners.
@@ -132,7 +132,7 @@ class Runner extends EventEmitter {
   }
 
   _createWorker() {
-    const worker = new Worker(this);
+    const worker = this._options.debug ? new InProcessWorker(this) : new OopWorker(this);
     worker.on('test', params => this.emit(constants.EVENT_TEST_BEGIN, this._updateTest(params.test)));
     worker.on('pending', params => this.emit(constants.EVENT_TEST_PENDING, this._updateTest(params.test)));
     worker.on('pass', params => this.emit(constants.EVENT_TEST_PASS, this._updateTest(params.test)));
@@ -154,8 +154,8 @@ class Runner extends EventEmitter {
     worker.init().then(() => this._workerAvailable(worker));
   }
 
-  _restartWorker(worker) {
-    worker.stop();
+  async _restartWorker(worker) {
+    await worker.stop();
     this._createWorker();
   }
 
@@ -175,7 +175,7 @@ class Runner extends EventEmitter {
 
 let lastWorkerId = 0;
 
-class Worker extends EventEmitter {
+class OopWorker extends EventEmitter {
   constructor(runner) {
     super();
     this.runner = runner;
@@ -240,4 +240,37 @@ class Worker extends EventEmitter {
   }
 }
 
+class InProcessWorker extends EventEmitter {
+  constructor(runner) {
+    super();
+    this.runner = runner;
+    this.fixturePool = require('./fixturesUI').fixturePool;
+  }
+
+  async init() {
+  }
+
+  async run(file) {
+    delete require.cache[file];
+    const { TestRunner } = require('./testRunner');
+    const testRunner = new TestRunner(file, this.runner._options);
+    for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
+      testRunner.on(event, this.emit.bind(this, event)); 
+    testRunner.run();
+  }
+
+  async stop() {
+    await this.fixturePool.teardownScope('worker');
+    this.emit('exit');
+  }
+
+  takeOut() {
+    return [];
+  }
+
+  takeErr() {
+    return [];
+  }
+}
+
 module.exports = { Runner };

test/runner/testRunner.js

@@ -0,0 +1,125 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const Mocha = require('mocha');
+const { fixturesUI } = require('./fixturesUI');
+const { EventEmitter } = require('events');
+
+global.expect = require('expect');
+global.testOptions = require('./testOptions');
+const GoldenUtils = require('./GoldenUtils');
+
+(function extendExpects() {
+  function toMatchImage(received, path) {
+    const {pass, message} =  GoldenUtils.compare(received, path);
+    return {pass, message: () => message};
+  };
+  global.expect.extend({ toMatchImage });
+})();
+
+class NullReporter {}
+
+class TestRunner extends EventEmitter {
+  constructor(file, options) {
+    super();
+    this.mocha = new Mocha({
+      ui: fixturesUI.bind(null, options.trialRun),
+      timeout: options.timeout,
+      reporter: NullReporter
+    });
+    if (options.grep)
+      this.mocha.grep(options.grep);
+    this.mocha.addFile(file);
+    this.mocha.suite.filterOnly();
+    this._lastOrdinal = -1;
+    this._failedWithError = false;
+  }
+
+  async run() {
+    let callback;
+    const result = new Promise(f => callback = f);
+    const runner = this.mocha.run(callback);
+
+    const constants = Mocha.Runner.constants;
+    runner.on(constants.EVENT_TEST_BEGIN, test => {
+      this.emit('test', { test: serializeTest(test, ++this._lastOrdinal) });
+    });
+
+    runner.on(constants.EVENT_TEST_PENDING, test => {
+      this.emit('pending', { test: serializeTest(test, ++this._lastOrdinal) });
+    });
+
+    runner.on(constants.EVENT_TEST_PASS, test => {
+      this.emit('pass', { test: serializeTest(test, this._lastOrdinal) });
+    });
+
+    runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
+      this._failedWithError = error;
+      this.emit('fail', {
+        test: serializeTest(test, this._lastOrdinal),
+        error: serializeError(error),
+      });
+    });
+
+    runner.once(constants.EVENT_RUN_END, async () => {
+      this.emit('done', { stats: serializeStats(runner.stats), error: this._failedWithError });
+    });
+    await result;
+  }
+}
+
+function serializeTest(test, origin) {
+  return {
+    id: `${test.file}::${origin}`,
+    duration: test.duration,
+  };
+}
+
+function serializeStats(stats) {
+  return {
+    tests: stats.tests,
+    passes: stats.passes,
+    duration: stats.duration,
+    failures: stats.failures,
+    pending: stats.pending,
+  }
+}
+
+function trimCycles(obj) {
+  const cache = new Set();
+  return JSON.parse(
+    JSON.stringify(obj, function(key, value) {
+      if (typeof value === 'object' && value !== null) {
+        if (cache.has(value))
+          return '' + value;
+        cache.add(value);
+      }
+      return value;
+    })
+  );
+}
+
+function serializeError(error) {
+  if (error instanceof Error) {
+    return {
+      message: error.message,
+      stack: error.stack
+    }
+  }
+  return trimCycles(error);
+}
+
+module.exports = { TestRunner };

test/runner/worker.js

@@ -15,20 +15,14 @@
  */
 
 const debug = require('debug');
-const Mocha = require('mocha');
-const { fixturesUI, fixturePool } = require('./fixturesUI');
+const { fixturePool } = require('./fixturesUI');
 const { gracefullyCloseAll } = require('../../lib/server/processLauncher');
-const GoldenUtils = require('./GoldenUtils');
-
-global.expect = require('expect');
-global.testOptions = require('./testOptions');
-
-const constants = Mocha.Runner.constants;
-
-extendExpects();
+const { TestRunner } = require('./testRunner');
 
 let closed = false;
 
+sendMessageToParent('ready');
+
 process.stdout.write = chunk => {
   sendMessageToParent('stdout', chunk);
 };
@@ -41,22 +35,29 @@ debug.log = data => {
   sendMessageToParent('debug', data);
 };
 
+process.on('disconnect', gracefullyCloseAndExit);
+process.on('SIGINT',() => {});
+process.on('SIGTERM',() => {});
+
 process.on('message', async message => {
   if (message.method === 'init')
     process.env.JEST_WORKER_ID = message.params.workerId;
   if (message.method === 'stop') {
     await fixturePool.teardownScope('worker');
     await gracefullyCloseAndExit();
-  } if (message.method === 'run')
-    await runSingleTest(message.params.file, message.params.options);
+  } if (message.method === 'run') {
+    const testRunner = new TestRunner(message.params.file, message.params.options);
+    for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
+      testRunner.on(event, sendMessageToParent.bind(null, event));
+    await testRunner.run();
+    // Mocha runner adds these; if we don't remove them, we'll get a leak.
+    process.removeAllListeners('uncaughtException');
+  }
 });
 
-process.on('disconnect', gracefullyCloseAndExit);
-process.on('SIGINT',() => {});
-process.on('SIGTERM',() => {});
-sendMessageToParent('ready');
-
 async function gracefullyCloseAndExit() {
+  if (closed)
+    return;
   closed = true;
   // Force exit after 30 seconds.
   setTimeout(() => process.exit(0), 30000);
@@ -65,52 +66,6 @@ async function gracefullyCloseAndExit() {
   process.exit(0);
 }
 
-class NullReporter {}
-
-let failedWithError = false;
-
-async function runSingleTest(file, options) {
-  let lastOrdinal = -1;
-  const mocha = new Mocha({
-    ui: fixturesUI.bind(null, options.trialRun),
-    timeout: options.timeout,
-    reporter: NullReporter
-  });
-  if (options.grep)
-    mocha.grep(options.grep);
-  mocha.addFile(file);
-  mocha.suite.filterOnly();
-
-  const runner = mocha.run(() => {
-    // Runner adds these; if we don't remove them, we'll get a leak.
-    process.removeAllListeners('uncaughtException');
-  });
-
-  runner.on(constants.EVENT_TEST_BEGIN, test => {
-    sendMessageToParent('test', { test: serializeTest(test, ++lastOrdinal) });
-  });
-
-  runner.on(constants.EVENT_TEST_PENDING, test => {
-    sendMessageToParent('pending', { test: serializeTest(test, ++lastOrdinal) });
-  });
-
-  runner.on(constants.EVENT_TEST_PASS, test => {
-    sendMessageToParent('pass', { test: serializeTest(test, lastOrdinal) });
-  });
-
-  runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
-    failedWithError = error;
-    sendMessageToParent('fail', {
-      test: serializeTest(test, lastOrdinal),
-      error: serializeError(error),
-    });
-  });
-
-  runner.once(constants.EVENT_RUN_END, async () => {
-    sendMessageToParent('done', { stats: serializeStats(runner.stats), error: failedWithError });
-  });
-}
-
 function sendMessageToParent(method, params = {}) {
   if (closed)
     return;
@@ -120,52 +75,3 @@ function sendMessageToParent(method, params = {}) {
     // Can throw when closing.
   }
 }
-
-function serializeTest(test, origin) {
-  return {
-    id: `${test.file}::${origin}`,
-    duration: test.duration,
-  };
-}
-
-function serializeStats(stats) {
-  return {
-    tests: stats.tests,
-    passes: stats.passes,
-    duration: stats.duration,
-    failures: stats.failures,
-    pending: stats.pending,
-  }
-}
-
-function trimCycles(obj) {
-  const cache = new Set();
-  return JSON.parse(
-    JSON.stringify(obj, function(key, value) {
-      if (typeof value === 'object' && value !== null) {
-        if (cache.has(value))
-          return '' + value;
-        cache.add(value);
-      }
-      return value;
-    })
-  );
-}
-
-function serializeError(error) {
-  if (error instanceof Error) {
-    return {
-      message: error.message,
-      stack: error.stack
-    }
-  }
-  return trimCycles(error);
-}
-
-function extendExpects() {
-  function toMatchImage(received, path) {
-    const {pass, message} =  GoldenUtils.compare(received, path);
-    return {pass, message: () => message};
-  };
-  global.expect.extend({ toMatchImage });
-}