feat(test-runner): use require to resolve reporters (#7749)

src/test/cli.ts

@@ -20,13 +20,12 @@ import * as commander from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
 import type { Config } from './types';
-import { Runner } from './runner';
+import { Runner, builtInReporters, BuiltInReporter } from './runner';
 import { stopProfiling, startProfiling } from './profiler';
-import type { FilePatternFilter } from './util';
+import { FilePatternFilter } from './util';
 
 const defaultTimeout = 30000;
-const defaultReporter = process.env.CI ? 'dot' : 'list';
-const builtinReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'];
+const defaultReporter: BuiltInReporter = process.env.CI ? 'dot' : 'list';
 const tsConfig = 'playwright.config.ts';
 const jsConfig = 'playwright.config.js';
 const mjsConfig = 'playwright.config.mjs';
@@ -55,7 +54,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   command.option('--output <dir>', `Folder for output artifacts (default: "test-results")`);
   command.option('--quiet', `Suppress stdio`);
   command.option('--repeat-each <N>', `Run each test N times (default: 1)`);
-  command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtinReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
+  command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
   command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
   command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
   command.option('--project <project-name>', `Only run tests from the specified project (default: run all projects)`);
@@ -177,10 +176,19 @@ function overridesFromOptions(options: { [key: string]: any }): Config {
     quiet: options.quiet ? options.quiet : undefined,
     repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined,
     retries: options.retries ? parseInt(options.retries, 10) : undefined,
-    reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [r]) : undefined,
+    reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined,
     shard: shardPair ? { current: shardPair[0] - 1, total: shardPair[1] } : undefined,
     timeout: isDebuggerAttached ? 0 : (options.timeout ? parseInt(options.timeout, 10) : undefined),
     updateSnapshots: options.updateSnapshots ? 'all' as const : undefined,
     workers: options.workers ? parseInt(options.workers, 10) : undefined,
   };
 }
+
+function resolveReporter(id: string) {
+  if (builtInReporters.includes(id as any))
+    return id;
+  const localPath = path.resolve(process.cwd(), id);
+  if (fs.existsSync(localPath))
+    return localPath;
+  return require.resolve(id, { paths: [ process.cwd() ] });
+}

src/test/loader.ts

@@ -25,6 +25,7 @@ import * as url from 'url';
 import { ProjectImpl } from './project';
 import { Reporter } from '../../types/testReporter';
 import { LaunchConfig } from '../../types/test';
+import { BuiltInReporter, builtInReporters } from './runner';
 
 export class Loader {
   private _defaultConfig: Config;
@@ -80,7 +81,7 @@ export class Loader {
     const configUse = mergeObjects(this._defaultConfig.use, this._config.use);
     this._config = mergeObjects(mergeObjects(this._defaultConfig, this._config), { use: configUse });
 
-    if (('testDir' in this._config) && this._config.testDir !== undefined && !path.isAbsolute(this._config.testDir))
+    if (this._config.testDir !== undefined)
       this._config.testDir = path.resolve(rootDir, this._config.testDir);
     const projects: Project[] = ('projects' in this._config) && this._config.projects !== undefined ? this._config.projects : [this._config];
 
@@ -93,7 +94,7 @@ export class Loader {
     this._fullConfig.grepInvert = takeFirst(this._configOverrides.grepInvert, this._config.grepInvert, baseFullConfig.grepInvert);
     this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures);
     this._fullConfig.preserveOutput = takeFirst<PreserveOutput>(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput);
-    this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter);
+    this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter as any), resolveReporters(this._config.reporter, rootDir), baseFullConfig.reporter);
     this._fullConfig.reportSlowTests = takeFirst(this._configOverrides.reportSlowTests, this._config.reportSlowTests, baseFullConfig.reportSlowTests);
     this._fullConfig.quiet = takeFirst(this._configOverrides.quiet, this._config.quiet, baseFullConfig.quiet);
     this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
@@ -220,7 +221,7 @@ function takeFirst<T>(...args: (T | undefined)[]): T {
   return undefined as any as T;
 }
 
-function toReporters(reporters: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[] | undefined): ReporterDescription[] | undefined {
+function toReporters(reporters: BuiltInReporter | ReporterDescription[] | undefined): ReporterDescription[] | undefined {
   if (!reporters)
     return;
   if (typeof reporters === 'string')
@@ -313,10 +314,8 @@ function validateConfig(file: string, config: Config) {
         if (!Array.isArray(item) || item.length <= 0 || item.length > 2 || typeof item[0] !== 'string')
           throw errorWithFile(file, `config.reporter[${index}] must be a tuple [name, optionalArgument]`);
       });
-    } else {
-      const builtinReporters = ['dot', 'line', 'list', 'junit', 'json', 'null'];
-      if (typeof config.reporter !== 'string' || !builtinReporters.includes(config.reporter))
-        throw errorWithFile(file, `config.reporter must be one of ${builtinReporters.map(name => `"${name}"`).join(', ')}`);
+    } else if (typeof config.reporter !== 'string') {
+      throw errorWithFile(file, `config.reporter must be a string`);
     }
   }
 
@@ -437,3 +436,11 @@ const baseFullConfig: FullConfig = {
   workers: 1,
   launch: [],
 };
+
+function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined {
+  return toReporters(reporters as any)?.map(([id, arg]) => {
+    if (builtInReporters.includes(id as any))
+      return [id, arg];
+    return [require.resolve(id, { paths: [ rootDir ] }), arg];
+  });
+}

src/test/runner.ts

@@ -64,7 +64,7 @@ export class Runner {
 
   private async _createReporter(list: boolean) {
     const reporters: Reporter[] = [];
-    const defaultReporters = {
+    const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
       dot: list ? ListModeReporter : DotReporter,
       line: list ? ListModeReporter : LineReporter,
       list: list ? ListModeReporter : ListReporter,
@@ -427,3 +427,6 @@ class ListModeReporter implements Reporter {
     console.log(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);
   }
 }
+
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'] as const;
+export type BuiltInReporter = typeof builtInReporters[number];

tests/playwright-test/global-setup.spec.ts

@@ -21,7 +21,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) =>
     'playwright.config.ts': `
       import * as path from 'path';
       module.exports = {
-        globalSetup: 'globalSetup.ts',
+        globalSetup: './globalSetup.ts',
         globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
       };
     `,

tests/playwright-test/reporter.spec.ts

@@ -16,6 +16,24 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
+const smallReporterJS = `
+class Reporter {
+  onBegin(config, suite) {
+    console.log('\\n%%begin');
+  }
+  onTestBegin(test) {}
+  onStdOut() {}
+  onStdErr() {}
+  onTestEnd(test, result) {}
+  onTimeout() {}
+  onError() {}
+  onEnd() {
+    console.log('\\n%%end');
+  }
+}
+module.exports = Reporter;
+`;
+
 test('should work with custom reporter', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'reporter.ts': `
@@ -91,3 +109,48 @@ test('should work with custom reporter', async ({ runInlineTest }) => {
     '%%reporter-end-end%%',
   ]);
 });
+
+
+test('should work without a file extension', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}) => {
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%begin',
+    '%%end',
+  ]);
+});
+
+test('should load reporter from node_modules', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'node_modules/my-reporter/index.js': smallReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: 'my-reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({}) => {
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%begin',
+    '%%end',
+  ]);
+});

types/test.d.ts

@@ -146,6 +146,9 @@ export type LaunchConfig = {
   cwd?: string,
 };
 
+type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never })
+
+
 /**
  * Testing configuration.
  */
@@ -213,7 +216,7 @@ interface ConfigBase {
    * It is possible to pass multiple reporters. A common pattern is using one terminal reporter
    * like `'line'` or `'list'`, and one file reporter like `'json'` or `'junit'`.
    */
-  reporter?: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[];
+  reporter?: LiteralUnion<'list'|'dot'|'line'|'json'|'junit'|'null', string> | ReporterDescription[];
 
   /**
    * Whether to report slow tests. When `null`, slow tests are not reported.