diff --git a/.eslintignore b/.eslintignore index 82c905afa2..954cfdbd30 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,8 +1,5 @@ test/assets/modernizr.js -third_party/* -utils/browser/playwright-web.js utils/doclint/check_public_api/test/ -utils/testrunner/examples/ lib/ *.js src/generated/* @@ -14,5 +11,7 @@ src/server/webkit/protocol.ts /electron-types.d.ts utils/generate_types/overrides.d.ts utils/generate_types/test/test.ts -test/ -test-runner/ \ No newline at end of file +/test/ +node_modules/ +browser_patches/*/checkout/ +packages/**/*.d.ts diff --git a/.eslintrc.js b/.eslintrc.js index d0a264ac13..edaa436791 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -2,7 +2,6 @@ module.exports = { parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint', 'notice'], parserOptions: { - project: ['./tsconfig.json', './test/tsconfig.json'], ecmaVersion: 9, sourceType: 'module', }, @@ -113,8 +112,5 @@ module.exports = { "mustMatch": "Copyright", "templateFile": "./utils/copyright.js", }], - - // type-aware rules - "@typescript-eslint/no-unnecessary-type-assertion": 2, } }; diff --git a/package.json b/package.json index 1ad499a1ce..f47df92e70 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "ftest": "cross-env BROWSER=firefox node test-runner/cli test/", "wtest": "cross-env BROWSER=webkit node test-runner/cli test/", "test": "node test-runner/cli test/", - "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src", + "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .", "tsc": "tsc -p .", "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", diff --git a/src/utils/browserPaths.ts b/src/utils/browserPaths.ts index c965bf3da1..151d2bdc4b 100644 --- a/src/utils/browserPaths.ts +++ b/src/utils/browserPaths.ts @@ -32,7 +32,9 @@ export type BrowserDescriptor = { export const hostPlatform = ((): BrowserPlatform => { const platform = os.platform(); if (platform === 'darwin') { - const macVersion = execSync('sw_vers -productVersion').toString('utf8').trim().split('.').slice(0, 2).join('.'); + const macVersion = execSync('sw_vers -productVersion', { + stdio: ['ignore', 'pipe', 'ignore'] + }).toString('utf8').trim().split('.').slice(0, 2).join('.'); return `mac${macVersion}` as BrowserPlatform; } if (platform === 'linux') { diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts index 27e47d3f90..893843b1c9 100644 --- a/test-runner/src/builtin.fixtures.ts +++ b/test-runner/src/builtin.fixtures.ts @@ -30,7 +30,7 @@ declare global { repeat(n: number): DescribeFunction; }; - type ItFunction = ((name: string, inner: (state: STATE) => Promise) => void) & { + type ItFunction = ((name: string, inner: (state: STATE) => Promise | void) => void) & { fail(condition: boolean): ItFunction; skip(condition: boolean): ItFunction; slow(): ItFunction; diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index 7611540d62..33740d9dc6 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -33,65 +33,65 @@ export const reporters = { }; program - .version('Version ' + /** @type {any} */ (require)('../package.json').version) - .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) - .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') - .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any) - .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') - .option('--trial-run', 'Only collect the matching tests and report them as passing') - .option('--quiet', 'Suppress stdio', false) - .option('--debug', 'Run tests in-process for debugging', false) - .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) - .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000') - .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') - .action(async (command) => { - const testDir = path.resolve(process.cwd(), command.args[0]); - const config: RunnerConfig = { - debug: command.debug, - forbidOnly: command.forbidOnly, - quiet: command.quiet, - grep: command.grep, - jobs: command.jobs, - outputDir: command.output, - snapshotDir: path.join(testDir, '__snapshots__'), - testDir, - timeout: command.timeout, - trialRun: command.trialRun, - updateSnapshots: command.updateSnapshots - }; + .version('Version ' + /** @type {any} */ (require)('../package.json').version) + .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) + .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') + .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any) + .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') + .option('--trial-run', 'Only collect the matching tests and report them as passing') + .option('--quiet', 'Suppress stdio', false) + .option('--debug', 'Run tests in-process for debugging', false) + .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) + .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000') + .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') + .action(async command => { + const testDir = path.resolve(process.cwd(), command.args[0]); + const config: RunnerConfig = { + debug: command.debug, + forbidOnly: command.forbidOnly, + quiet: command.quiet, + grep: command.grep, + jobs: command.jobs, + outputDir: command.output, + snapshotDir: path.join(testDir, '__snapshots__'), + testDir, + timeout: command.timeout, + trialRun: command.trialRun, + updateSnapshots: command.updateSnapshots + }; - const reporterList = command.reporter.split(','); - const reporterObjects: Reporter[] = reporterList.map(c => { - if (reporters[c]) - return new reporters[c](); - try { - const p = path.resolve(process.cwd(), c); - return new (require(p).default); - } catch (e) { - console.error('Invalid reporter ' + c, e); + const reporterList = command.reporter.split(','); + const reporterObjects: Reporter[] = reporterList.map(c => { + if (reporters[c]) + return new reporters[c](); + try { + const p = path.resolve(process.cwd(), c); + return new (require(p).default)(); + } catch (e) { + console.error('Invalid reporter ' + c, e); + process.exit(1); + } + }); + + const files = collectFiles(testDir, '', command.args.slice(1)); + const result = await run(config, files, new Multiplexer(reporterObjects)); + if (result === 'forbid-only') { + console.error('====================================='); + console.error(' --forbid-only found a focused test.'); + console.error('====================================='); process.exit(1); } + + if (result === 'no-tests') { + console.error('================='); + console.error(' no tests found.'); + console.error('================='); + process.exit(1); + } + + process.exit(result === 'failed' ? 1 : 0); }); - const files = collectFiles(testDir, '', command.args.slice(1)); - const result = await run(config, files, new Multiplexer(reporterObjects)); - if (result === 'forbid-only') { - console.error('====================================='); - console.error(' --forbid-only found a focused test.'); - console.error('====================================='); - process.exit(1); - } - - if (result === 'no-tests') { - console.error('================='); - console.error(' no tests found.'); - console.error('================='); - process.exit(1); - } - - process.exit(result === 'failed' ? 1 : 0); - }); - program.parse(process.argv); function collectFiles(testDir: string, dir: string, filters: string[]): string[] { diff --git a/test-runner/src/expect.ts b/test-runner/src/expect.ts index f60c33e215..5bed3db694 100644 --- a/test-runner/src/expect.ts +++ b/test-runner/src/expect.ts @@ -35,7 +35,7 @@ export function initializeImageMatcher(config: RunnerConfig) { function toMatchImage(received: Buffer, name: string, options?: { threshold?: number }) { const { pass, message } = compare(received, name, config, testFile, options); return { pass, message: () => message }; - }; + } expect.extend({ toMatchImage }); } diff --git a/test-runner/src/fixtures.ts b/test-runner/src/fixtures.ts index ea5f1c8b33..aed216aa1b 100644 --- a/test-runner/src/fixtures.ts +++ b/test-runner/src/fixtures.ts @@ -15,14 +15,21 @@ */ import debug from 'debug'; -import { Test, serializeError } from './test'; +import { RunnerConfig } from './runnerConfig'; +import { serializeError, Test, TestResult } from './test'; type Scope = 'test' | 'worker'; type FixtureRegistration = { name: string; scope: Scope; - fn: Function; + fn: Function; +}; + +export type TestInfo = { + config: RunnerConfig; + test: Test; + result: TestResult; }; const registrations = new Map(); @@ -36,8 +43,8 @@ export function setParameters(params: any) { registerWorkerFixture(name, async ({}, test) => await test(parameters[name])); } -class Fixture { - pool: FixturePool; +class Fixture { + pool: FixturePool; name: string; scope: Scope; fn: Function; @@ -50,7 +57,7 @@ class Fixture { _setup = false; _teardown = false; - constructor(pool: FixturePool, name: string, scope: Scope, fn: any) { + constructor(pool: FixturePool, name: string, scope: Scope, fn: any) { this.pool = pool; this.name = name; this.scope = scope; @@ -61,11 +68,11 @@ class Fixture { this.value = this.hasGeneratorValue ? parameters[name] : null; } - async setup(config: Config, test?: Test) { + async setup(config: RunnerConfig, info?: TestInfo) { if (this.hasGeneratorValue) return; for (const name of this.deps) { - await this.pool.setupFixture(name, config, test); + await this.pool.setupFixture(name, config, info); this.pool.instances.get(name).usages.add(this.name); } @@ -77,11 +84,12 @@ class Fixture { const setupFence = new Promise((f, r) => { setupFenceFulfill = f; setupFenceReject = r; }); const teardownFence = new Promise(f => this._teardownFenceCallback = f); debug('pw:test:hook')(`setup "${this.name}"`); + const param = info || config; this._tearDownComplete = this.fn(params, async (value: any) => { this.value = value; setupFenceFulfill(); return await teardownFence; - }, config, test).catch((e: any) => setupFenceReject(e)); + }, param).catch((e: any) => setupFenceReject(e)); await setupFence; this._setup = true; } @@ -107,13 +115,13 @@ class Fixture { } } -export class FixturePool { - instances: Map>; +export class FixturePool { + instances: Map; constructor() { this.instances = new Map(); } - async setupFixture(name: string, config: Config, test?: Test) { + async setupFixture(name: string, config: RunnerConfig, info?: TestInfo) { let fixture = this.instances.get(name); if (fixture) return fixture; @@ -123,45 +131,48 @@ export class FixturePool { const { scope, fn } = registrations.get(name); fixture = new Fixture(this, name, scope, fn); this.instances.set(name, fixture); - await fixture.setup(config, test); + await fixture.setup(config, info); return fixture; } async teardownScope(scope: string) { - for (const [name, fixture] of this.instances) { + for (const [, fixture] of this.instances) { if (fixture.scope === scope) await fixture.teardown(); } } - async resolveParametersAndRun(fn: Function, config: Config, test?: Test) { + async resolveParametersAndRun(fn: Function, config: RunnerConfig, info?: TestInfo) { const names = fixtureParameterNames(fn); for (const name of names) - await this.setupFixture(name, config, test); + await this.setupFixture(name, config, info); const params = {}; for (const n of names) params[n] = this.instances.get(n).value; return fn(params); } - wrapTestCallback(callback: any, timeout: number, config: Config, test: Test) { - if (!callback) - return callback; - return async() => { - let timer: NodeJS.Timer; - let timerPromise = new Promise(f => timer = setTimeout(f, timeout)); - try { - await Promise.race([ - this.resolveParametersAndRun(callback, config, test).then(() => clearTimeout(timer)), - timerPromise.then(() => Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`))) - ]); - } catch (e) { - test.error = serializeError(e); - throw e; - } finally { - await this.teardownScope('test'); - } - }; + async runTestWithFixtures(fn: Function, timeout: number, info: TestInfo) { + let timer: NodeJS.Timer; + const timerPromise = new Promise(f => timer = setTimeout(f, timeout)); + try { + await Promise.race([ + this.resolveParametersAndRun(fn, info.config, info).then(() => { + info.result.status = 'passed'; + clearTimeout(timer); + }), + timerPromise.then(() => { + info.result.status = 'timedOut'; + Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`)); + }) + ]); + } catch (e) { + info.result.status = 'failed'; + info.result.error = serializeError(e); + throw e; + } finally { + await this.teardownScope('test'); + } } } @@ -171,10 +182,10 @@ export function fixturesForCallback(callback: Function): string[] { for (const name of fixtureParameterNames(callback)) { if (name in names) continue; - names.add(name); - if (!registrations.has(name)) { + names.add(name); + if (!registrations.has(name)) throw new Error('Using undefined fixture ' + name); - } + const { fn } = registrations.get(name); visit(fn); } @@ -190,7 +201,7 @@ function fixtureParameterNames(fn: Function): string[] { const match = text.match(/async(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/); if (!match || !match[1].trim()) return []; - let signature = match[1]; + const signature = match[1]; return signature.split(',').map((t: string) => t.trim()); } @@ -205,15 +216,15 @@ function innerRegisterFixture(name: string, scope: Scope, fn: Function, caller: if (!registrationsByFile.has(file)) registrationsByFile.set(file, []); registrationsByFile.get(file).push(registration); -}; +} -export function registerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, config: Config, test: Test) => Promise) { +export function registerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, info: TestInfo) => Promise) { innerRegisterFixture(name, 'test', fn, registerFixture); -}; +} -export function registerWorkerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, config: Config) => Promise) { +export function registerWorkerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise, config: RunnerConfig) => Promise) { innerRegisterFixture(name, 'worker', fn, registerWorkerFixture); -}; +} export function registerParameter(name: string, fn: () => any) { registerWorkerFixture(name, async ({}: any, test: Function) => await test(parameters[name])); @@ -236,7 +247,7 @@ export function lookupRegistrations(file: string, scope: Scope) { const deps = new Set(); collectRequires(file, deps); const allDeps = [...deps].reverse(); - let result = new Map(); + const result = new Map(); for (const dep of allDeps) { const registrationList = registrationsByFile.get(dep); if (!registrationList) @@ -244,7 +255,7 @@ export function lookupRegistrations(file: string, scope: Scope) { for (const r of registrationList) { if (scope && r.scope !== scope) continue; - result.set(r.name, r); + result.set(r.name, r); } } return result; diff --git a/test-runner/src/golden.ts b/test-runner/src/golden.ts index fde652d368..ba4c70b690 100644 --- a/test-runner/src/golden.ts +++ b/test-runner/src/golden.ts @@ -70,7 +70,7 @@ function compareText(actual: Buffer, expectedBuffer: Buffer): { diff?: object; e }; } -export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number } ): { pass: boolean; message?: string; } { +export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number }): { pass: boolean; message?: string; } { let expectedPath: string; const relativeTestFile = path.relative(config.testDir, testFile); const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, ''); @@ -125,8 +125,8 @@ export function compare(actual: Buffer, name: string, config: RunnerConfig, test } fs.writeFileSync(actualPath, actual); if (result.diff) - fs.writeFileSync(diffPath, result.diff); - + fs.writeFileSync(diffPath, result.diff); + const output = [ c.red(`Image comparison failed:`), ]; diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index 6e07b73f48..c0bda73a03 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -19,11 +19,10 @@ import * as fs from 'fs'; import * as path from 'path'; import './builtin.fixtures'; import './expect'; -import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures'; +import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures'; import { Reporter } from './reporter'; import { Runner } from './runner'; import { RunnerConfig } from './runnerConfig'; -import { Test } from './test'; import { Matrix, TestCollector } from './testCollector'; import { installTransform } from './transform'; export { parameters, registerParameter } from './fixtures'; @@ -42,21 +41,21 @@ declare global { } } -let beforeFunctions: Function[] = []; -let afterFunctions: Function[] = []; +const beforeFunctions: Function[] = []; +const afterFunctions: Function[] = []; let matrix: Matrix = {}; global['before'] = (fn: Function) => beforeFunctions.push(fn); global['after'] = (fn: Function) => afterFunctions.push(fn); global['matrix'] = (m: Matrix) => matrix = m; -export function registerFixture(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise, config: RunnerConfig, test: Test) => Promise) { - registerFixtureT(name, fn); -}; +export function registerFixture(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise, info: TestInfo) => Promise) { + registerFixtureT(name, fn); +} -export function registerWorkerFixture(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise, config: RunnerConfig) => Promise) { - registerWorkerFixtureT(name, fn); -}; +export function registerWorkerFixture(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise, config: RunnerConfig) => Promise) { + registerWorkerFixtureT(name, fn); +} type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests'; @@ -100,5 +99,7 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor for (const f of afterFunctions) await f(); } - return suite.findTest(t => t.error) ? 'failed' : 'passed'; + return suite.findTest(test => { + return !!test.results.find(result => result.status === 'failed' || result.status === 'timedOut'); + }) ? 'failed' : 'passed'; } diff --git a/test-runner/src/reporter.ts b/test-runner/src/reporter.ts index 39872d8492..36106fe24c 100644 --- a/test-runner/src/reporter.ts +++ b/test-runner/src/reporter.ts @@ -15,15 +15,13 @@ */ import { RunnerConfig } from './runnerConfig'; -import { Suite, Test } from './test'; +import { Suite, Test, TestResult } from './test'; export interface Reporter { onBegin(config: RunnerConfig, suite: Suite): void; - onTest(test: Test): void; - onSkippedTest(test: Test): void; + onTestBegin(test: Test): void; onTestStdOut(test: Test, chunk: string | Buffer); onTestStdErr(test: Test, chunk: string | Buffer); - onTestPassed(test: Test): void; - onTestFailed(test: Test): void; + onTestEnd(test: Test, result: TestResult); onEnd(): void; } diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts index 2e8c67871f..12af020e6b 100644 --- a/test-runner/src/reporters/base.ts +++ b/test-runner/src/reporters/base.ts @@ -24,15 +24,15 @@ import StackUtils from 'stack-utils'; import terminalLink from 'terminal-link'; import { Reporter } from '../reporter'; import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test } from '../test'; +import { Suite, Test, TestResult } from '../test'; -const stackUtils = new StackUtils() +const stackUtils = new StackUtils(); export class BaseReporter implements Reporter { skipped: Test[] = []; - passes: Test[] = []; - failures: Test[] = []; - timeouts: Test[] = []; + passed: { test: Test, result: TestResult }[] = []; + failed: { test: Test, result: TestResult }[] = []; + timedOut: { test: Test, result: TestResult }[] = []; duration = 0; startTime: number; config: RunnerConfig; @@ -51,11 +51,7 @@ export class BaseReporter implements Reporter { this.suite = suite; } - onTest(test: Test) { - } - - onSkippedTest(test: Test) { - this.skipped.push(test); + onTestBegin(test: Test) { } onTestStdOut(test: Test, chunk: string | Buffer) { @@ -68,15 +64,13 @@ export class BaseReporter implements Reporter { process.stderr.write(chunk); } - onTestPassed(test: Test) { - this.passes.push(test); - } - - onTestFailed(test: Test) { - if (test.duration >= test.timeout) - this.timeouts.push(test); - else - this.failures.push(test); + onTestEnd(test: Test, result: TestResult) { + switch (result.status) { + case 'skipped': this.skipped.push(test); break; + case 'passed': this.passed.push({ test, result }); break; + case 'failed': this.failed.push({ test, result }); break; + case 'timedOut': this.timedOut.push({ test, result }); break; + } } onEnd() { @@ -86,56 +80,61 @@ export class BaseReporter implements Reporter { epilogue() { console.log(''); - console.log(colors.green(` ${this.passes.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); + console.log(colors.green(` ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); if (this.skipped.length) console.log(colors.yellow(` ${this.skipped.length} skipped`)); - if (this.failures.length) { - console.log(colors.red(` ${this.failures.length} failed`)); + if (this.failed.length) { + console.log(colors.red(` ${this.failed.length} failed`)); console.log(''); - this._printFailures(this.failures); + this._printFailures(this.failed); } - if (this.timeouts.length) { - console.log(colors.red(` ${this.timeouts.length} timed out`)); + if (this.timedOut.length) { + console.log(colors.red(` ${this.timedOut.length} timed out`)); console.log(''); - this._printFailures(this.timeouts); + this._printFailures(this.timedOut); } } - private _printFailures(failures: Test[]) { - failures.forEach((failure, index) => { - console.log(this.formatFailure(failure, index + 1)); + private _printFailures(failures: { test: Test, result: TestResult}[]) { + failures.forEach(({test, result}, index) => { + console.log(this.formatFailure(test, result, index + 1)); }); } - formatFailure(failure: Test, index?: number): string { + formatFailure(test: Test, failure: TestResult, index?: number): string { const tokens: string[] = []; - const relativePath = path.relative(process.cwd(), failure.file); - const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`; + const relativePath = path.relative(process.cwd(), test.file); + const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`; tokens.push(colors.bold(colors.red(header))); - const stack = failure.error.stack; - if (stack) { + if (failure.status === 'timedOut') { tokens.push(''); - const messageLocation = failure.error.stack.indexOf(failure.error.message); - const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length); - tokens.push(indent(preamble, ' ')); - const position = positionInFile(stack, failure.file); - if (position) { - const source = fs.readFileSync(failure.file, 'utf8'); + tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), ' ')); + } else { + const stack = failure.error.stack; + if (stack) { tokens.push(''); - tokens.push(indent(codeFrameColumns(source, { + const messageLocation = failure.error.stack.indexOf(failure.error.message); + const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length); + tokens.push(indent(preamble, ' ')); + const position = positionInFile(stack, test.file); + if (position) { + const source = fs.readFileSync(test.file, 'utf8'); + tokens.push(''); + tokens.push(indent(codeFrameColumns(source, { start: position, }, { highlightCode: true} - ), ' ')); + ), ' ')); + } + tokens.push(''); + tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), ' ')); + } else { + tokens.push(''); + tokens.push(indent(String(failure.error), ' ')); } - tokens.push(''); - tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), ' ')); - } else { - tokens.push(''); - tokens.push(indent(String(failure.error), ' ')); } tokens.push(''); return tokens.join('\n'); diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts index bbfb01c2e3..1e4fe32069 100644 --- a/test-runner/src/reporters/dot.ts +++ b/test-runner/src/reporters/dot.ts @@ -16,27 +16,19 @@ import colors from 'colors/safe'; import { BaseReporter } from './base'; -import { Test } from '../test'; +import { Test, TestResult } from '../test'; class DotReporter extends BaseReporter { - onSkippedTest(test: Test) { - super.onSkippedTest(test); - process.stdout.write(colors.yellow('∘')) + onTestEnd(test: Test, result: TestResult) { + super.onTestEnd(test, result); + switch (result.status) { + case 'skipped': process.stdout.write(colors.yellow('∘')); break; + case 'passed': process.stdout.write(colors.green('·')); break; + case 'failed': process.stdout.write(colors.red('F')); break; + case 'timedOut': process.stdout.write(colors.red('T')); break; + } } - - onTestPassed(test: Test) { - super.onTestPassed(test); - process.stdout.write(colors.green('·')); - } - - onTestFailed(test: Test) { - super.onTestFailed(test); - if (test.duration >= test.timeout) - process.stdout.write(colors.red('T')); - else - process.stdout.write(colors.red('F')); - } - + onEnd() { super.onEnd(); process.stdout.write('\n'); diff --git a/test-runner/src/reporters/json.ts b/test-runner/src/reporters/json.ts index 909e20a951..2382c6f228 100644 --- a/test-runner/src/reporters/json.ts +++ b/test-runner/src/reporters/json.ts @@ -15,7 +15,7 @@ */ import { BaseReporter } from './base'; -import { Suite, Test } from '../test'; +import { Suite, Test, TestResult } from '../test'; import * as fs from 'fs'; class JSONReporter extends BaseReporter { @@ -50,14 +50,19 @@ class JSONReporter extends BaseReporter { title: test.title, file: test.file, only: test.only, - skipped: test.skipped, slow: test.slow, - duration: test.duration, timeout: test.timeout, - error: test.error, - stdout: test.stdout.map(s => stdioEntry(s)), - stderr: test.stderr.map(s => stdioEntry(s)), - data: test.data + results: test.results.map(r => this._serializeTestResult(r)) + }; + } + + private _serializeTestResult(result: TestResult): any { + return { + duration: result.duration, + error: result.error, + stdout: result.stdout.map(s => stdioEntry(s)), + stderr: result.stderr.map(s => stdioEntry(s)), + data: result.data }; } } @@ -65,7 +70,7 @@ class JSONReporter extends BaseReporter { function stdioEntry(s: string | Buffer): any { if (typeof s === 'string') return { text: s }; - return { buffer: s.toString('base64') } + return { buffer: s.toString('base64') }; } export default JSONReporter; diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts index 75b990c471..ab37cdc6d5 100644 --- a/test-runner/src/reporters/list.ts +++ b/test-runner/src/reporters/list.ts @@ -17,7 +17,7 @@ import colors from 'colors/safe'; import { BaseReporter } from './base'; import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test } from '../test'; +import { Suite, Test, TestResult } from '../test'; class ListReporter extends BaseReporter { _failure = 0; @@ -27,29 +27,22 @@ class ListReporter extends BaseReporter { console.log(); } - onTest(test: Test) { - super.onTest(test); + onTestBegin(test: Test) { + super.onTestBegin(test); process.stdout.write(' ' + colors.gray(test.fullTitle() + ': ')); } - onSkippedTest(test: Test) { - super.onSkippedTest(test); - process.stdout.write(colors.green(' - ') + colors.cyan(test.fullTitle())); - process.stdout.write('\n'); - } - - onTestPassed(test: Test) { - super.onTestPassed(test); - process.stdout.write('\u001b[2K\u001b[0G'); - process.stdout.write(colors.green(' ✓ ') + colors.gray(test.fullTitle())); - process.stdout.write('\n'); - } - - onTestFailed(test: Test) { - super.onTestFailed(test); - process.stdout.write('\u001b[2K\u001b[0G'); - process.stdout.write(colors.red(` ${++this._failure}) ` + test.fullTitle())); - process.stdout.write('\n'); + onTestEnd(test: Test, result: TestResult) { + super.onTestEnd(test, result); + let text = ''; + switch (result.status) { + case 'skipped': text = colors.green(' - ') + colors.cyan(test.fullTitle()); break; + case 'passed': text = '\u001b[2K\u001b[0G' + colors.green(' ✓ ') + colors.gray(test.fullTitle()); break; + case 'failed': + // fall through + case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle()); break; + } + process.stdout.write(text + '\n'); } onEnd() { diff --git a/test-runner/src/reporters/multiplexer.ts b/test-runner/src/reporters/multiplexer.ts index 614c6f647d..6d068fb7a5 100644 --- a/test-runner/src/reporters/multiplexer.ts +++ b/test-runner/src/reporters/multiplexer.ts @@ -15,7 +15,7 @@ */ import { RunnerConfig } from '../runnerConfig'; -import { Suite, Test } from '../test'; +import { Suite, Test, TestResult } from '../test'; import { Reporter } from '../reporter'; export class Multiplexer implements Reporter { @@ -30,14 +30,9 @@ export class Multiplexer implements Reporter { reporter.onBegin(config, suite); } - onTest(test: Test) { + onTestBegin(test: Test) { for (const reporter of this._reporters) - reporter.onTest(test); - } - - onSkippedTest(test: Test) { - for (const reporter of this._reporters) - reporter.onSkippedTest(test); + reporter.onTestBegin(test); } onTestStdOut(test: Test, chunk: string | Buffer) { @@ -47,17 +42,12 @@ export class Multiplexer implements Reporter { onTestStdErr(test: Test, chunk: string | Buffer) { for (const reporter of this._reporters) - reporter.onTestStdErr(test, chunk); + reporter.onTestStdErr(test, chunk); } - onTestPassed(test: Test) { + onTestEnd(test: Test, result: TestResult) { for (const reporter of this._reporters) - reporter.onTestPassed(test); - } - - onTestFailed(test: Test) { - for (const reporter of this._reporters) - reporter.onTestFailed(test); + reporter.onTestEnd(test, result); } onEnd() { diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts index ffec65c2f4..e9de390f0b 100644 --- a/test-runner/src/reporters/pytest.ts +++ b/test-runner/src/reporters/pytest.ts @@ -17,12 +17,12 @@ import colors from 'colors/safe'; import milliseconds from 'ms'; import * as path from 'path'; -import { Test, Suite, Configuration } from '../test'; +import { Test, Suite, Configuration, TestResult } from '../test'; import { BaseReporter } from './base'; import { RunnerConfig } from '../runnerConfig'; const cursorPrevLine = '\u001B[F'; -const eraseLine = '\u001B[2K' +const eraseLine = '\u001B[2K'; type Row = { id: string; @@ -43,7 +43,6 @@ class PytestReporter extends BaseReporter { private _suiteIds = new Map(); private _lastOrdinal = 0; private _visibleRows: number; - private _failed = false; private _total: number; private _progress: string[] = []; private _throttler = new Throttler(250, () => this._repaint()); @@ -77,20 +76,13 @@ class PytestReporter extends BaseReporter { } } - onTest(test: Test) { - super.onTest(test); + onTestBegin(test: Test) { + super.onTestBegin(test); const row = this._rows.get(this._id(test)); if (!row.startTime) row.startTime = Date.now(); } - onSkippedTest(test: Test) { - super.onSkippedTest(test); - this._append(test, colors.yellow('∘')); - this._progress.push('S'); - this._throttler.schedule(); - } - onTestStdOut(test: Test, chunk: string | Buffer) { this._repaint(chunk); } @@ -99,21 +91,32 @@ class PytestReporter extends BaseReporter { this._repaint(chunk); } - onTestPassed(test: Test) { - super.onTestPassed(test); - this._append(test, colors.green('✓')); - this._progress.push('P'); - this._throttler.schedule(); - } - - onTestFailed(test: Test) { - super.onTestFailed(test); - const title = test.duration >= test.timeout ? colors.red('T') : colors.red('F'); - const row = this._append(test, title); - row.failed = true; - this._failed = true; - this._progress.push('F'); - this._repaint(this.formatFailure(test) + '\n'); + onTestEnd(test: Test, result: TestResult) { + super.onTestEnd(test, result); + switch (result.status) { + case 'skipped': { + this._append(test, colors.yellow('∘')); + this._progress.push('S'); + this._throttler.schedule(); + break; + } + case 'passed': { + this._append(test, colors.green('✓')); + this._progress.push('P'); + this._throttler.schedule(); + break; + } + case 'failed': + // fall through + case 'timedOut': { + const title = result.status === 'timedOut' ? colors.red('T') : colors.red('F'); + const row = this._append(test, title); + row.failed = true; + this._progress.push('F'); + this._repaint(this.formatFailure(test, result) + '\n'); + break; + } + } } private _append(test: Test, s: string): Row { @@ -146,14 +149,14 @@ class PytestReporter extends BaseReporter { } const status = []; - if (this.passes.length) - status.push(colors.green(`${this.passes.length} passed`)); + if (this.passed.length) + status.push(colors.green(`${this.passed.length} passed`)); if (this.skipped.length) status.push(colors.yellow(`${this.skipped.length} skipped`)); - if (this.failures.length) - status.push(colors.red(`${this.failures.length} failed`)); - if (this.timeouts.length) - status.push(colors.red(`${this.timeouts.length} timed out`)); + if (this.failed.length) + status.push(colors.red(`${this.failed.length} failed`)); + if (this.timedOut.length) + status.push(colors.red(`${this.timedOut.length} timed out`)); status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`)); for (let i = lines.length; i < this._visibleRows; ++i) @@ -214,7 +217,7 @@ const yellowBar = colors.yellow('▇'); function serializeConfiguration(configuration: Configuration): string { const tokens = []; for (const { name, value } of configuration) - tokens.push(`${name}=${value}`); + tokens.push(`${name}=${value}`); return tokens.join(', '); } diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts index 4f2d5c10d9..1fd938fe06 100644 --- a/test-runner/src/runner.ts +++ b/test-runner/src/runner.ts @@ -19,8 +19,8 @@ import crypto from 'crypto'; import path from 'path'; import { EventEmitter } from 'events'; import { lookupRegistrations, FixturePool } from './fixtures'; -import { Suite, Test } from './test'; -import { TestRunnerEntry, SerializedTest } from './testRunner'; +import { Suite, Test, TestResult } from './test'; +import { TestRunnerEntry } from './testRunner'; import { RunnerConfig } from './runnerConfig'; import { Reporter } from './reporter'; @@ -28,9 +28,8 @@ export class Runner { private _workers = new Set(); private _freeWorkers: Worker[] = []; private _workerClaimers: (() => void)[] = []; - stats: { duration: number; failures: number; passes: number; skipped: number; tests: number; }; - private _testById = new Map(); + private _testById = new Map(); private _queue: TestRunnerEntry[] = []; private _stopCallback: () => void; readonly _config: RunnerConfig; @@ -40,18 +39,11 @@ export class Runner { constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) { this._config = config; this._reporter = reporter; - this.stats = { - duration: 0, - failures: 0, - passes: 0, - skipped: 0, - tests: 0, - }; this._suite = suite; for (const suite of this._suite.suites) { suite.findTest(test => { - this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, test); + this._testById.set(`${test._ordinal}@${suite.file}::[${suite._configurationString}]`, { test, result: test._appendResult() }); }); } @@ -59,7 +51,7 @@ export class Runner { const total = suite.total(); console.log(); const jobs = Math.min(config.jobs, suite.suites.length); - console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${jobs} worker${ jobs > 1 ? 's' : ''}`); + console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}`); } } @@ -158,33 +150,29 @@ export class Runner { _createWorker() { const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this); - worker.on('test', params => { - ++this.stats.tests; - this._reporter.onTest(this._updateTest(params.test)); + worker.on('testBegin', params => { + const { test } = this._testById.get(params.id); + this._reporter.onTestBegin(test); }); - worker.on('skipped', params => { - ++this.stats.tests; - ++this.stats.skipped; - this._reporter.onSkippedTest(this._updateTest(params.test)); + worker.on('testEnd', params => { + const workerResult: TestResult = params.result; + // We were accumulating these below. + delete workerResult.stdout; + delete workerResult.stderr; + const { test, result } = this._testById.get(params.id); + Object.assign(result, workerResult); + this._reporter.onTestEnd(test, result); }); - worker.on('pass', params => { - ++this.stats.passes; - this._reporter.onTestPassed(this._updateTest(params.test)); - }); - worker.on('fail', params => { - ++this.stats.failures; - this._reporter.onTestFailed(this._updateTest(params.test)); - }); - worker.on('stdout', params => { + worker.on('testStdOut', params => { const chunk = chunkFromParams(params); - const test = this._testById.get(params.testId); - test.stdout.push(chunk); + const { test, result } = this._testById.get(params.id); + result.stdout.push(chunk); this._reporter.onTestStdOut(test, chunk); }); - worker.on('stderr', params => { + worker.on('testStdErr', params => { const chunk = chunkFromParams(params); - const test = this._testById.get(params.testId); - test.stderr.push(chunk); + const { test, result } = this._testById.get(params.id); + result.stderr.push(chunk); this._reporter.onTestStdErr(test, chunk); }); worker.on('exit', () => { @@ -201,14 +189,6 @@ export class Runner { this._createWorker(); } - _updateTest(serialized: SerializedTest): Test { - const test = this._testById.get(serialized.id); - test.duration = serialized.duration; - test.error = serialized.error; - test.data = serialized.data; - return test; - } - async stop() { const result = new Promise(f => this._stopCallback = f); for (const worker of this._workers) @@ -253,7 +233,7 @@ class OopWorker extends Worker { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }); this.process.on('exit', () => this.emit('exit')); - this.process.on('error', (e) => {}); // do not yell at a send to dead process. + this.process.on('error', e => {}); // do not yell at a send to dead process. this.process.on('message', message => { const { method, params } = message; this.emit(method, params); @@ -276,11 +256,11 @@ class OopWorker extends Worker { } class InProcessWorker extends Worker { - fixturePool: FixturePool; + fixturePool: FixturePool; constructor(runner: Runner) { super(runner); - this.fixturePool = require('./testRunner').fixturePool as FixturePool; + this.fixturePool = require('./testRunner').fixturePool as FixturePool; } async init() { @@ -292,7 +272,7 @@ class InProcessWorker extends Worker { delete require.cache[entry.file]; const { TestRunner } = require('./testRunner'); const testRunner = new TestRunner(entry, this.runner._config, 0); - for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr']) + for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done']) testRunner.on(event, this.emit.bind(this, event)); testRunner.run(); } diff --git a/test-runner/src/spec.ts b/test-runner/src/spec.ts index 35358aa067..17448e1509 100644 --- a/test-runner/src/spec.ts +++ b/test-runner/src/spec.ts @@ -64,9 +64,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { if (only) test.only = true; if (!only && specs.skip && specs.skip[0]) - test.skipped = true; + test._skipped = true; if (!only && specs.fail && specs.fail[0]) - test.skipped = true; + test._skipped = true; suite._addTest(test); return test; }); diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index 3172aae989..f9aee9ead2 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -21,15 +21,11 @@ export class Test { title: string; file: string; only = false; - skipped = false; + _skipped = false; slow = false; - duration = 0; timeout = 0; fn: Function; - error: any; - stdout: (string | Buffer)[] = []; - stderr: (string | Buffer)[] = []; - data: any = {}; + results: TestResult[] = []; _ordinal: number; _overriddenFn: Function; @@ -48,18 +44,38 @@ export class Test { return this.titlePath().join(' '); } + _appendResult(): TestResult { + const result: TestResult = { + duration: 0, + status: 'none', + stdout: [], + stderr: [], + data: {} + }; + this.results.push(result); + return result; + } + _clone(): Test { const test = new Test(this.title, this.fn); test.suite = this.suite; test.only = this.only; test.file = this.file; - test.skipped = this.skipped; test.timeout = this.timeout; test._overriddenFn = this._overriddenFn; return test; } } +export type TestResult = { + duration: number; + status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped'; + error?: any; + stdout: (string | Buffer)[]; + stderr: (string | Buffer)[]; + data: any; +} + export class Suite { title: string; parent?: Suite; @@ -152,7 +168,7 @@ export class Suite { _hasTestsToRun(): boolean { let found = false; this.findTest(test => { - if (!test.skipped) { + if (!test._skipped) { found = true; return true; } @@ -164,7 +180,7 @@ export class Suite { export function serializeConfiguration(configuration: Configuration): string { const tokens = []; for (const { name, value } of configuration) - tokens.push(`${name}=${value}`); + tokens.push(`${name}=${value}`); return tokens.join(', '); } @@ -173,7 +189,7 @@ export function serializeError(error: Error | any): any { return { message: error.message, stack: error.stack - } + }; } return trimCycles(error); } @@ -181,13 +197,13 @@ export function serializeError(error: Error | any): any { function trimCycles(obj: any): any { const cache = new Set(); return JSON.parse( - JSON.stringify(obj, function(key, value) { - if (typeof value === 'object' && value !== null) { - if (cache.has(value)) - return '' + value; - cache.add(value); - } - return value; - }) + JSON.stringify(obj, function(key, value) { + if (typeof value === 'object' && value !== null) { + if (cache.has(value)) + return '' + value; + cache.add(value); + } + return value; + }) ); } diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts index 2dcae6404a..b83b4926d6 100644 --- a/test-runner/src/testCollector.ts +++ b/test-runner/src/testCollector.ts @@ -71,7 +71,7 @@ export class TestCollector { const values = this._matrix[name]; if (!values) continue; - let state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]]; + const state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]]; generatorConfigurations.length = 0; for (const gen of state) { for (const value of values) diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index be7c82d5cc..cca9fe9a4f 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import { FixturePool, rerunRegistrations, setParameters } from './fixtures'; +import { FixturePool, rerunRegistrations, setParameters, TestInfo } from './fixtures'; import { EventEmitter } from 'events'; import { setCurrentTestFile } from './expect'; -import { Test, Suite, Configuration, serializeError } from './test'; +import { Test, Suite, Configuration, serializeError, TestResult } from './test'; import { spec } from './spec'; import { RunnerConfig } from './runnerConfig'; import * as util from 'util'; -export const fixturePool = new FixturePool(); +export const fixturePool = new FixturePool(); export type TestRunnerEntry = { file: string; @@ -40,13 +40,6 @@ function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: strin return { text: chunk }; } -export type SerializedTest = { - id: string, - error: any, - duration: number, - data: any[] -}; - export class TestRunner extends EventEmitter { private _failedTestId: string | undefined; private _fatalError: any | undefined; @@ -58,7 +51,10 @@ export class TestRunner extends EventEmitter { private _parsedGeneratorConfiguration: any = {}; private _config: RunnerConfig; private _timeout: number; - private _test: Test | null = null; + private _testId: string | null; + private _stdOutBuffer: (string | Buffer)[] = []; + private _stdErrBuffer: (string | Buffer)[] = []; + private _testResult: TestResult | null = null; constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) { super(); @@ -81,21 +77,32 @@ export class TestRunner extends EventEmitter { fatalError(error: Error | any) { this._fatalError = serializeError(error); - if (this._test) { - this._test.error = this._fatalError; - this.emit('fail', { - test: this._serializeTest(), + if (this._testResult) { + this._testResult.error = this._fatalError; + this.emit('testEnd', { + id: this._testId, + result: this._testResult }); } this._reportDone(); } stdout(chunk: string | Buffer) { - this.emit('stdout', { testId: this._testId(), ...chunkToParams(chunk) }) + this._stdOutBuffer.push(chunk); + if (!this._testId) + return; + for (const c of this._stdOutBuffer) + this.emit('testStdOut', { id: this._testId, ...chunkToParams(c) }); + this._stdOutBuffer = []; } stderr(chunk: string | Buffer) { - this.emit('stderr', { testId: this._testId(), ...chunkToParams(chunk) }) + this._stdErrBuffer.push(chunk); + if (!this._testId) + return; + for (const c of this._stdErrBuffer) + this.emit('testStdErr', { id: this._testId, ...chunkToParams(c) }); + this._stdErrBuffer = []; } async run() { @@ -120,11 +127,11 @@ export class TestRunner extends EventEmitter { this._reportDone(); } for (const entry of suite._entries) { - if (entry instanceof Suite) { + if (entry instanceof Suite) await this._runSuite(entry); - } else { + else await this._runTest(entry); - } + } try { await this._runHooks(suite, 'afterAll', 'after'); @@ -137,32 +144,52 @@ export class TestRunner extends EventEmitter { private async _runTest(test: Test) { if (this._failedTestId) return false; - this._test = test; if (this._ordinals.size && !this._ordinals.has(test._ordinal)) return; this._remaining.delete(test._ordinal); - if (test.skipped || test.suite._isSkipped()) { - this.emit('skipped', { test: this._serializeTest() }); + + const id = `${test._ordinal}@${this._configuredFile}`; + this._testId = id; + this.emit('testBegin', { id }); + + const result: TestResult = { + duration: 0, + status: 'none', + stdout: [], + stderr: [], + data: {} + }; + this._testResult = result; + + if (test._skipped || test.suite._isSkipped()) { + result.status = 'skipped'; + this.emit('testEnd', { id, result }); return; } - this.emit('test', { test: this._serializeTest() }); + const startTime = Date.now(); try { - await this._runHooks(test.suite, 'beforeEach', 'before'); - test._startTime = Date.now(); - if (!this._trialRun) - await this._testWrapper(test)(); - await this._runHooks(test.suite, 'afterEach', 'after'); - this.emit('pass', { test: this._serializeTest(true) }); + const testInfo = { config: this._config, test, result }; + await this._runHooks(test.suite, 'beforeEach', 'before', testInfo); + if (!this._trialRun) { + const timeout = test.slow ? this._timeout * 3 : this._timeout; + await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo); + } + await this._runHooks(test.suite, 'afterEach', 'after', testInfo); + result.duration = Date.now() - startTime; + this.emit('testEnd', { id, result }); } catch (error) { - test.error = serializeError(error); - this._failedTestId = this._testId(); - this.emit('fail', { test: this._serializeTest(true) }); + result.error = serializeError(error); + result.status = 'failed'; + result.duration = Date.now() - startTime; + this._failedTestId = this._testId; + this.emit('testEnd', { id, result }); } - this._test = null; + this._testResult = null; + this._testId = null; } - private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after') { + private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after', testInfo?: TestInfo) { if (!suite._hasTestsToRun()) return; const all = []; @@ -173,7 +200,7 @@ export class TestRunner extends EventEmitter { if (dir === 'before') all.reverse(); for (const hook of all) - await fixturePool.resolveParametersAndRun(hook, this._config); + await fixturePool.resolveParametersAndRun(hook, this._config, testInfo); } private _reportDone() { @@ -183,22 +210,4 @@ export class TestRunner extends EventEmitter { remaining: [...this._remaining], }); } - - private _testWrapper(test: Test) { - const timeout = test.slow ? this._timeout * 3 : this._timeout; - return fixturePool.wrapTestCallback(test.fn, timeout, { ...this._config }, test); - } - - private _testId() { - return `${this._test._ordinal}@${this._configuredFile}`; - } - - private _serializeTest(full = false): SerializedTest { - return { - id: this._testId(), - error: this._test.error, - duration: Date.now() - this._test._startTime, - data: full ? this._test.data : undefined - }; - } } diff --git a/test-runner/src/transform.ts b/test-runner/src/transform.ts index a1898b1e56..b97e16fa78 100644 --- a/test-runner/src/transform.ts +++ b/test-runner/src/transform.ts @@ -56,7 +56,7 @@ export function installTransform(): () => void { sourceMaps.set(filename, sourceMapPath); if (fs.existsSync(codePath)) return fs.readFileSync(codePath, 'utf8'); - + const result = babel.transformFileSync(filename, { presets: [ ['@babel/preset-env', { targets: {node: 'current'} }], @@ -67,7 +67,7 @@ export function installTransform(): () => void { if (result.code) { fs.mkdirSync(path.dirname(cachePath), {recursive: true}); if (result.map) - fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8'); + fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8'); fs.writeFileSync(codePath, result.code, 'utf8'); } return result.code; diff --git a/test-runner/src/worker.ts b/test-runner/src/worker.ts index b6938a9609..08ac48d798 100644 --- a/test-runner/src/worker.ts +++ b/test-runner/src/worker.ts @@ -69,7 +69,7 @@ process.on('message', async message => { } if (message.method === 'run') { testRunner = new TestRunner(message.params.entry, message.params.config, workerId); - for (const event of ['test', 'skipped', 'pass', 'fail', 'done', 'stdout', 'stderr']) + for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done']) testRunner.on(event, sendMessageToParent.bind(null, event)); await testRunner.run(); testRunner = null; diff --git a/test-runner/test/assets/test-data-visible-in-fixture.js b/test-runner/test/assets/test-data-visible-in-fixture.js index fa26526564..41d56b1c02 100644 --- a/test-runner/test/assets/test-data-visible-in-fixture.js +++ b/test-runner/test/assets/test-data-visible-in-fixture.js @@ -18,9 +18,9 @@ const { registerFixture } = require('../../'); const fs = require('fs'); const path = require('path'); -registerFixture('postProcess', async ({}, runTest, config, test) => { +registerFixture('postProcess', async ({}, runTest, info) => { await runTest(''); - test.data['myname'] = 'myvalue'; + info.result.data['myname'] = 'myvalue'; }); it('ensure fixture handles test error', async ({ postProcess }) => { diff --git a/test-runner/test/assets/test-error-visible-in-fixture.js b/test-runner/test/assets/test-error-visible-in-fixture.js index 7a63cd83c8..45228e67e0 100644 --- a/test-runner/test/assets/test-error-visible-in-fixture.js +++ b/test-runner/test/assets/test-error-visible-in-fixture.js @@ -18,9 +18,10 @@ const { registerFixture } = require('../../'); const fs = require('fs'); const path = require('path'); -registerFixture('postProcess', async ({}, runTest, config, test) => { +registerFixture('postProcess', async ({}, runTest, info) => { await runTest(''); - fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(test.error, undefined, 2)); + const { config, result } = info; + fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(result.error, undefined, 2)); }); it('ensure fixture handles test error', async ({ postProcess }) => { diff --git a/test-runner/test/assets/typescript.ts b/test-runner/test/assets/typescript.ts index b8362f12d3..e0bd3b3447 100644 --- a/test-runner/test/assets/typescript.ts +++ b/test-runner/test/assets/typescript.ts @@ -1,5 +1,20 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import '../../'; -import { abc } from "./global-foo"; +import './global-foo'; it('should find global foo', () => { expect(global['foo']).toBe(true); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts index b0cadde507..4ab44c6bca 100644 --- a/test-runner/test/exit-code.spec.ts +++ b/test-runner/test/exit-code.spec.ts @@ -23,66 +23,66 @@ import '../lib'; const removeFolderAsync = promisify(rimraf); -it('should fail', async() => { +it('should fail', async () => { const result = await runTest('one-failure.js'); expect(result.exitCode).toBe(1); expect(result.passed).toBe(0); expect(result.failed).toBe(1); }); -it('should succeed', async() => { +it('should succeed', async () => { const result = await runTest('one-success.js'); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); expect(result.failed).toBe(0); }); -it('should access error in fixture', async() => { +it('should access error in fixture', async () => { const result = await runTest('test-error-visible-in-fixture.js'); expect(result.exitCode).toBe(1); const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString()); expect(data.message).toContain('Object.is equality'); }); -it('should access data in fixture', async() => { +it('should access data in fixture', async () => { const result = await runTest('test-data-visible-in-fixture.js'); expect(result.exitCode).toBe(1); const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString()); - const test = data.suites[0].tests[0]; - expect(test.data).toEqual({ 'myname': 'myvalue' }); - expect(test.stdout).toEqual([{ text: 'console.log\n' }]); - expect(test.stderr).toEqual([{ text: 'console.error\n' }]); + const testResult = data.suites[0].tests[0].results[0]; + expect(testResult.data).toEqual({ 'myname': 'myvalue' }); + expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]); + expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]); }); -it('should handle worker fixture timeout', async() => { +it('should handle worker fixture timeout', async () => { const result = await runTest('worker-fixture-timeout.js', 1000); expect(result.exitCode).toBe(1); expect(result.output).toContain('Timeout of 1000ms'); }); -it('should handle worker fixture error', async() => { +it('should handle worker fixture error', async () => { const result = await runTest('worker-fixture-error.js'); expect(result.exitCode).toBe(1); expect(result.output).toContain('Worker failed'); }); -it('should collect stdio', async() => { +it('should collect stdio', async () => { const result = await runTest('stdio.js'); expect(result.exitCode).toBe(0); const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString()); - const test = data.suites[0].tests[0]; - const { stdout, stderr } = test; + const testResult = data.suites[0].tests[0].results[0]; + const { stdout, stderr } = testResult; expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]); expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]); }); -it('should work with typescript', async() => { +it('should work with typescript', async () => { const result = await runTest('typescript.ts'); expect(result.exitCode).toBe(0); }); async function runTest(filePath: string, timeout = 10000) { - const outputDir = path.join(__dirname, 'test-results') + const outputDir = path.join(__dirname, 'test-results'); await removeFolderAsync(outputDir).catch(e => {}); const { output, status } = spawnSync('node', [ @@ -102,7 +102,7 @@ async function runTest(filePath: string, timeout = 10000) { return { exitCode: status, output: output.toString(), - passed: parseInt(passed), - failed: parseInt(failed || '0') - } + passed: parseInt(passed, 10), + failed: parseInt(failed || '0', 10) + }; } diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index 979be85c19..178da4cb3c 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -189,10 +189,11 @@ registerFixture('context', async ({browser}, test) => { await context.close(); }); -registerFixture('page', async ({context}, runTest, config, test) => { +registerFixture('page', async ({context}, runTest, info) => { const page = await context.newPage(); await runTest(page); - if (test.error) { + const { test, config, result } = info; + if (result.status === 'failed' || result.status === 'timedOut') { const relativePath = path.relative(config.testDir, test.file).replace(/\.spec\.[jt]s/, ''); const sanitizedTitle = test.title.replace(/[^\w\d]+/g, '_'); const assetPath = path.join(config.outputDir, relativePath, sanitizedTitle) + '-failed.png';