From 051dc332a69f1b4736ff5a2336eb74a39d887be0 Mon Sep 17 00:00:00 2001 From: Joel Einbinder Date: Tue, 20 Jul 2021 15:03:01 -0500 Subject: [PATCH] feat(test-runner): use require to resolve reporters (#7749) --- src/test/cli.ts | 20 ++++--- src/test/loader.ts | 21 +++++--- src/test/runner.ts | 5 +- tests/playwright-test/global-setup.spec.ts | 2 +- tests/playwright-test/reporter.spec.ts | 63 ++++++++++++++++++++++ types/test.d.ts | 5 +- 6 files changed, 100 insertions(+), 16 deletions(-) diff --git a/src/test/cli.ts b/src/test/cli.ts index 48b08f9fe8..000fccc421 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -20,13 +20,12 @@ import * as commander from 'commander'; import * as fs from 'fs'; import * as path from 'path'; import type { Config } from './types'; -import { Runner } from './runner'; +import { Runner, builtInReporters, BuiltInReporter } from './runner'; import { stopProfiling, startProfiling } from './profiler'; -import type { FilePatternFilter } from './util'; +import { FilePatternFilter } from './util'; const defaultTimeout = 30000; -const defaultReporter = process.env.CI ? 'dot' : 'list'; -const builtinReporters = ['list', 'line', 'dot', 'json', 'junit', 'null']; +const defaultReporter: BuiltInReporter = process.env.CI ? 'dot' : 'list'; const tsConfig = 'playwright.config.ts'; const jsConfig = 'playwright.config.js'; const mjsConfig = 'playwright.config.mjs'; @@ -55,7 +54,7 @@ export function addTestCommand(program: commander.CommanderStatic) { command.option('--output ', `Folder for output artifacts (default: "test-results")`); command.option('--quiet', `Suppress stdio`); command.option('--repeat-each ', `Run each test N times (default: 1)`); - command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtinReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`); + command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`); command.option('--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`); command.option('--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`); command.option('--project ', `Only run tests from the specified project (default: run all projects)`); @@ -177,10 +176,19 @@ function overridesFromOptions(options: { [key: string]: any }): Config { quiet: options.quiet ? options.quiet : undefined, repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined, - reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [r]) : undefined, + reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined, shard: shardPair ? { current: shardPair[0] - 1, total: shardPair[1] } : undefined, timeout: isDebuggerAttached ? 0 : (options.timeout ? parseInt(options.timeout, 10) : undefined), updateSnapshots: options.updateSnapshots ? 'all' as const : undefined, workers: options.workers ? parseInt(options.workers, 10) : undefined, }; } + +function resolveReporter(id: string) { + if (builtInReporters.includes(id as any)) + return id; + const localPath = path.resolve(process.cwd(), id); + if (fs.existsSync(localPath)) + return localPath; + return require.resolve(id, { paths: [ process.cwd() ] }); +} diff --git a/src/test/loader.ts b/src/test/loader.ts index d43937071e..1102d4ae82 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -25,6 +25,7 @@ import * as url from 'url'; import { ProjectImpl } from './project'; import { Reporter } from '../../types/testReporter'; import { LaunchConfig } from '../../types/test'; +import { BuiltInReporter, builtInReporters } from './runner'; export class Loader { private _defaultConfig: Config; @@ -80,7 +81,7 @@ export class Loader { const configUse = mergeObjects(this._defaultConfig.use, this._config.use); this._config = mergeObjects(mergeObjects(this._defaultConfig, this._config), { use: configUse }); - if (('testDir' in this._config) && this._config.testDir !== undefined && !path.isAbsolute(this._config.testDir)) + if (this._config.testDir !== undefined) this._config.testDir = path.resolve(rootDir, this._config.testDir); const projects: Project[] = ('projects' in this._config) && this._config.projects !== undefined ? this._config.projects : [this._config]; @@ -93,7 +94,7 @@ export class Loader { this._fullConfig.grepInvert = takeFirst(this._configOverrides.grepInvert, this._config.grepInvert, baseFullConfig.grepInvert); this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures); this._fullConfig.preserveOutput = takeFirst(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput); - this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter); + this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter as any), resolveReporters(this._config.reporter, rootDir), baseFullConfig.reporter); this._fullConfig.reportSlowTests = takeFirst(this._configOverrides.reportSlowTests, this._config.reportSlowTests, baseFullConfig.reportSlowTests); this._fullConfig.quiet = takeFirst(this._configOverrides.quiet, this._config.quiet, baseFullConfig.quiet); this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard); @@ -220,7 +221,7 @@ function takeFirst(...args: (T | undefined)[]): T { return undefined as any as T; } -function toReporters(reporters: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[] | undefined): ReporterDescription[] | undefined { +function toReporters(reporters: BuiltInReporter | ReporterDescription[] | undefined): ReporterDescription[] | undefined { if (!reporters) return; if (typeof reporters === 'string') @@ -313,10 +314,8 @@ function validateConfig(file: string, config: Config) { if (!Array.isArray(item) || item.length <= 0 || item.length > 2 || typeof item[0] !== 'string') throw errorWithFile(file, `config.reporter[${index}] must be a tuple [name, optionalArgument]`); }); - } else { - const builtinReporters = ['dot', 'line', 'list', 'junit', 'json', 'null']; - if (typeof config.reporter !== 'string' || !builtinReporters.includes(config.reporter)) - throw errorWithFile(file, `config.reporter must be one of ${builtinReporters.map(name => `"${name}"`).join(', ')}`); + } else if (typeof config.reporter !== 'string') { + throw errorWithFile(file, `config.reporter must be a string`); } } @@ -437,3 +436,11 @@ const baseFullConfig: FullConfig = { workers: 1, launch: [], }; + +function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { + return toReporters(reporters as any)?.map(([id, arg]) => { + if (builtInReporters.includes(id as any)) + return [id, arg]; + return [require.resolve(id, { paths: [ rootDir ] }), arg]; + }); +} diff --git a/src/test/runner.ts b/src/test/runner.ts index 74b968d0cc..54baa57891 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -64,7 +64,7 @@ export class Runner { private async _createReporter(list: boolean) { const reporters: Reporter[] = []; - const defaultReporters = { + const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { dot: list ? ListModeReporter : DotReporter, line: list ? ListModeReporter : LineReporter, list: list ? ListModeReporter : ListReporter, @@ -427,3 +427,6 @@ class ListModeReporter implements Reporter { console.log(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`); } } + +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'] as const; +export type BuiltInReporter = typeof builtInReporters[number]; diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index ddbbad7ea4..2898e04c08 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -21,7 +21,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => 'playwright.config.ts': ` import * as path from 'path'; module.exports = { - globalSetup: 'globalSetup.ts', + globalSetup: './globalSetup.ts', globalTeardown: path.join(__dirname, 'globalTeardown.ts'), }; `, diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 0bb36fb423..23b23d8a89 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -16,6 +16,24 @@ import { test, expect } from './playwright-test-fixtures'; +const smallReporterJS = ` +class Reporter { + onBegin(config, suite) { + console.log('\\n%%begin'); + } + onTestBegin(test) {} + onStdOut() {} + onStdErr() {} + onTestEnd(test, result) {} + onTimeout() {} + onError() {} + onEnd() { + console.log('\\n%%end'); + } +} +module.exports = Reporter; +`; + test('should work with custom reporter', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` @@ -91,3 +109,48 @@ test('should work with custom reporter', async ({ runInlineTest }) => { '%%reporter-end-end%%', ]); }); + + +test('should work without a file extension', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}) => { + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%begin', + '%%end', + ]); +}); + +test('should load reporter from node_modules', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'node_modules/my-reporter/index.js': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: 'my-reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({}) => { + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%begin', + '%%end', + ]); +}); diff --git a/types/test.d.ts b/types/test.d.ts index 960b117d74..3c7d09cf47 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -146,6 +146,9 @@ export type LaunchConfig = { cwd?: string, }; +type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }) + + /** * Testing configuration. */ @@ -213,7 +216,7 @@ interface ConfigBase { * It is possible to pass multiple reporters. A common pattern is using one terminal reporter * like `'line'` or `'list'`, and one file reporter like `'json'` or `'junit'`. */ - reporter?: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'json'|'junit'|'null', string> | ReporterDescription[]; /** * Whether to report slow tests. When `null`, slow tests are not reported.