From ab78865a8d779eaeda835fae68e9d455d5911db1 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 24 Oct 2022 18:54:53 -0400 Subject: [PATCH] chore: render steps in list reporter (#18269) --- docs/src/test-reporters-js.md | 70 ++++++---- .../playwright-test/src/reporters/base.ts | 4 +- .../playwright-test/src/reporters/list.ts | 129 ++++++++++++------ tests/playwright-test/reporter-list.spec.ts | 65 +++++++-- 4 files changed, 190 insertions(+), 78 deletions(-) diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 7019204e5a..449f6e637a 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -133,16 +133,40 @@ Here is an example output in the middle of a test run. Failures will be listed a npx playwright test --reporter=list Running 124 tests using 6 workers - ✓ should access error in env (438ms) - ✓ handle long test names (515ms) - x 1) render expected (691ms) - ✓ should timeout (932ms) - should repeat each: - ✓ should respect enclosing .gitignore (569ms) - should teardown env after timeout: - should respect excluded tests: - ✓ should handle env beforeEach error (638ms) - should respect enclosing .gitignore: + 1 ✓ should access error in env (438ms) + 2 ✓ handle long test names (515ms) + 3 x 1) render expected (691ms) + 4 ✓ should timeout (932ms) + 5 should repeat each: + 6 ✓ should respect enclosing .gitignore (569ms) + 7 should teardown env after timeout: + 8 should respect excluded tests: + 9 ✓ should handle env beforeEach error (638ms) +10 should respect enclosing .gitignore: +``` + +You can opt into the step rendering via passing the following config option: + +```js tab=js-js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + reporter: [['list', { printSteps: true }], +}; + +module.exports = config; +``` + +```js tab=js-ts +// playwright.config.ts +import type { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + reporter: [['list', { printSteps: true }], +}; +export default config; ``` ### Line reporter @@ -246,11 +270,11 @@ You can also configure `host` and `port` that are used to serve the HTML report. /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - reporter: [ ['html', { + reporter: [['html', { open: 'never', host: '0.0.0.0', port: 9223, - }] ], + }]], }; module.exports = config; @@ -261,7 +285,7 @@ module.exports = config; import type { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - reporter: [ ['html', { open: 'never' }] ], + reporter: [['html', { open: 'never' }]], }; export default config; ``` @@ -276,7 +300,7 @@ In configuration file, pass options directly: /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - reporter: [ ['html', { outputFolder: 'my-report' }] ], + reporter: [['html', { outputFolder: 'my-report' }]], }; module.exports = config; @@ -287,7 +311,7 @@ module.exports = config; import type { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - reporter: [ ['html', { outputFolder: 'my-report' }] ], + reporter: [['html', { outputFolder: 'my-report' }]], }; export default config; ``` @@ -334,7 +358,7 @@ In configuration file, pass options directly: /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - reporter: [ ['json', { outputFile: 'results.json' }] ], + reporter: [['json', { outputFile: 'results.json' }]], }; module.exports = config; @@ -345,7 +369,7 @@ module.exports = config; import type { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - reporter: [ ['json', { outputFile: 'results.json' }] ], + reporter: [['json', { outputFile: 'results.json' }]], }; export default config; ``` @@ -377,7 +401,7 @@ In configuration file, pass options directly: /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - reporter: [ ['junit', { outputFile: 'results.xml' }] ], + reporter: [['junit', { outputFile: 'results.xml' }]], }; module.exports = config; @@ -388,7 +412,7 @@ module.exports = config; import type { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - reporter: [ ['junit', { outputFile: 'results.xml' }] ], + reporter: [['junit', { outputFile: 'results.xml' }]], }; export default config; ``` @@ -421,7 +445,7 @@ const xrayOptions = { }; const config = { - reporter: [ ['junit', xrayOptions] ] + reporter: [['junit', xrayOptions]] }; module.exports = config; @@ -449,7 +473,7 @@ const xrayOptions = { }; const config: PlaywrightTestConfig = { - reporter: [ ['junit', xrayOptions] ] + reporter: [['junit', xrayOptions]] }; export default config; @@ -496,7 +520,7 @@ The following configuration sample enables embedding attachments by using the `t /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ], + reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]], }; module.exports = config; @@ -507,7 +531,7 @@ module.exports = config; import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { - reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ], + reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]], }; export default config; diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 1b8c3657dc..c71cc919e2 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -352,7 +352,7 @@ function relativeTestPath(config: FullConfig, test: TestCase): string { return relativeFilePath(config, test.location.file); } -function stepSuffix(step: TestStep | undefined) { +export function stepSuffix(step: TestStep | undefined) { const stepTitles = step ? step.titlePath() : []; return stepTitles.map(t => ' › ' + t).join(''); } @@ -364,7 +364,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS if (omitLocation) location = `${relativeTestPath(config, test)}`; else - location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`; + location = `${relativeTestPath(config, test)}:${step?.location?.line ?? test.location.line}:${step?.location?.column ?? test.location.column}`; const projectTitle = projectName ? `[${projectName}] › ` : ''; return `${projectTitle}${location} › ${titles.join(' › ')}${stepSuffix(step)}`; } diff --git a/packages/playwright-test/src/reporters/list.ts b/packages/playwright-test/src/reporters/list.ts index 8f35b1a723..64de2254c0 100644 --- a/packages/playwright-test/src/reporters/list.ts +++ b/packages/playwright-test/src/reporters/list.ts @@ -16,7 +16,7 @@ /* eslint-disable no-console */ import { colors, ms as milliseconds } from 'playwright-core/lib/utilsBundle'; -import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base'; +import { BaseReporter, formatError, formatTestTitle, stepSuffix, stripAnsiEscapes } from './base'; import type { FullConfig, FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; // Allow it in the Visual Studio Code Terminal and the new Windows Terminal @@ -28,12 +28,16 @@ class ListReporter extends BaseReporter { private _lastRow = 0; private _lastColumn = 0; private _testRows = new Map(); - private _resultIndex = new Map(); + private _stepRows = new Map(); + private _resultIndex = new Map(); + private _stepIndex = new Map(); private _needNewLine = false; private readonly _liveTerminal: string | boolean | undefined; + private _printSteps: boolean; - constructor(options: { omitFailures?: boolean } = {}) { + constructor(options: { omitFailures?: boolean, printSteps?: boolean } = {}) { super(options); + this._printSteps = options.printSteps || !!process.env.PW_TEST_DEBUG_REPORTERS_PRINT_STEPS; this._liveTerminal = process.stdout.isTTY || !!process.env.PWTEST_TTY_WIDTH; } @@ -50,12 +54,13 @@ class ListReporter extends BaseReporter { onTestBegin(test: TestCase, result: TestResult) { if (this._liveTerminal) this._maybeWriteNewLine(); - this._resultIndex.set(result, this._resultIndex.size + 1); - this._testRows.set(test, this._lastRow++); + this._resultIndex.set(result, String(this._resultIndex.size + 1)); if (this._liveTerminal) { - const prefix = this._testPrefix(result, ''); + this._testRows.set(test, this._lastRow); + const index = this._resultIndex.get(result)!; + const prefix = this._testPrefix(index, ''); const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result); - process.stdout.write(prefix + this.fitToScreen(line, prefix) + '\n'); + this._appendLine(line, prefix); } } @@ -70,19 +75,52 @@ class ListReporter extends BaseReporter { } onStepBegin(test: TestCase, result: TestResult, step: TestStep) { - if (!this._liveTerminal) - return; if (step.category !== 'test.step') return; - this._updateTestLine(test, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(result, '')); + const testIndex = this._resultIndex.get(result)!; + if (!this._printSteps) { + if (this._liveTerminal) + this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); + return; + } + + const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1; + (result as any)[lastStepOrdinalSymbol] = ordinal; + const stepIndex = `${testIndex}.${ordinal}`; + this._stepIndex.set(step, stepIndex); + + if (this._liveTerminal) + this._maybeWriteNewLine(); + if (this._liveTerminal) { + this._stepRows.set(step, this._lastRow); + const prefix = this._testPrefix(stepIndex, ''); + const line = test.title + colors.dim(stepSuffix(step)); + this._appendLine(line, prefix); + } } onStepEnd(test: TestCase, result: TestResult, step: TestStep) { - if (!this._liveTerminal) - return; if (step.category !== 'test.step') return; - this._updateTestLine(test, colors.dim(formatTestTitle(this.config, test, step.parent)) + this._retrySuffix(result), this._testPrefix(result, '')); + + const testIndex = this._resultIndex.get(result)!; + if (!this._printSteps) { + if (this._liveTerminal) + this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step.parent)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); + return; + } + + const index = this._stepIndex.get(step)!; + const title = test.title + colors.dim(stepSuffix(step)); + const prefix = this._testPrefix(index, ''); + let text = ''; + if (step.error) + text = colors.red(title); + else + text = title; + text += colors.dim(` (${milliseconds(step.duration)})`); + + this._updateOrAppendLine(this._stepRows.get(step)!, text, prefix); } private _maybeWriteNewLine() { @@ -125,68 +163,77 @@ class ListReporter extends BaseReporter { const title = formatTestTitle(this.config, test); let prefix = ''; let text = ''; + const index = this._resultIndex.get(result)!; if (result.status === 'skipped') { - prefix = this._testPrefix(result, colors.green('-')); + prefix = this._testPrefix(index, colors.green('-')); // Do not show duration for skipped. text = colors.cyan(title) + this._retrySuffix(result); } else { const statusMark = result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK; if (result.status === test.expectedStatus) { - prefix = this._testPrefix(result, colors.green(statusMark)); - text = colors.dim(title); + prefix = this._testPrefix(index, colors.green(statusMark)); + text = title; } else { - prefix = this._testPrefix(result, colors.red(statusMark)); + prefix = this._testPrefix(index, colors.red(statusMark)); text = colors.red(title); } text += this._retrySuffix(result) + colors.dim(` (${milliseconds(result.duration)})`); } + this._updateOrAppendLine(this._testRows.get(test)!, text, prefix); + } + + private _updateOrAppendLine(row: number, text: string, prefix: string) { if (this._liveTerminal) { - this._updateTestLine(test, text, prefix); + this._updateLine(row, text, prefix); } else { this._maybeWriteNewLine(); - process.stdout.write(prefix + text); - process.stdout.write('\n'); + this._appendLine(text, prefix); } } - private _updateTestLine(test: TestCase, line: string, prefix: string) { - if (process.env.PW_TEST_DEBUG_REPORTERS) - this._updateTestLineForTest(test, line, prefix); - else - this._updateTestLineForTTY(test, line, prefix); + private _appendLine(text: string, prefix: string) { + const line = prefix + this.fitToScreen(text, prefix); + if (process.env.PW_TEST_DEBUG_REPORTERS) { + process.stdout.write(this._lastRow + ' : ' + line + '\n'); + } else { + process.stdout.write(line); + process.stdout.write('\n'); + } + ++this._lastRow; } - private _updateTestLineForTTY(test: TestCase, line: string, prefix: string) { - const testRow = this._testRows.get(test)!; + private _updateLine(row: number, text: string, prefix: string) { + const line = prefix + this.fitToScreen(text, prefix); + if (process.env.PW_TEST_DEBUG_REPORTERS) + process.stdout.write(row + ' : ' + line + '\n'); + else + this._updateLineForTTY(row, line); + } + + private _updateLineForTTY(row: number, line: string) { // Go up if needed - if (testRow !== this._lastRow) - process.stdout.write(`\u001B[${this._lastRow - testRow}A`); + if (row !== this._lastRow) + process.stdout.write(`\u001B[${this._lastRow - row}A`); // Erase line, go to the start process.stdout.write('\u001B[2K\u001B[0G'); - process.stdout.write(prefix + this.fitToScreen(line, prefix)); + process.stdout.write(line); // Go down if needed. - if (testRow !== this._lastRow) - process.stdout.write(`\u001B[${this._lastRow - testRow}E`); + if (row !== this._lastRow) + process.stdout.write(`\u001B[${this._lastRow - row}E`); if (process.env.PWTEST_TTY_WIDTH) process.stdout.write('\n'); // For testing. } - private _testPrefix(result: TestResult, statusMark: string) { - const index = this._resultIndex.get(result)!; + private _testPrefix(index: string, statusMark: string) { const statusMarkLength = stripAnsiEscapes(statusMark).length; - return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + colors.dim(String(index) + ' '); + return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + colors.dim(index + ' '); } private _retrySuffix(result: TestResult) { return (result.retry ? colors.yellow(` (retry #${result.retry})`) : ''); } - private _updateTestLineForTest(test: TestCase, line: string, prefix: string) { - const testRow = this._testRows.get(test)!; - process.stdout.write(testRow + ' : ' + prefix + line + '\n'); - } - override onError(error: TestError): void { super.onError(error); this._maybeWriteNewLine(); @@ -202,4 +249,6 @@ class ListReporter extends BaseReporter { } } +const lastStepOrdinalSymbol = Symbol('lastStepOrdinal'); + export default ListReporter; diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts index 98f84b1ed2..136b343190 100644 --- a/tests/playwright-test/reporter-list.spec.ts +++ b/tests/playwright-test/reporter-list.spec.ts @@ -58,30 +58,67 @@ test('render steps', async ({ runInlineTest }) => { test('passes', async ({}) => { await test.step('outer 1.0', async () => { await test.step('inner 1.1', async () => {}); - await test.step('inner 1.1', async () => {}); + await test.step('inner 1.2', async () => {}); }); await test.step('outer 2.0', async () => { await test.step('inner 2.1', async () => {}); + await test.step('inner 2.2', async () => {}); + }); + }); + `, + }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }); + const text = stripAnsi(result.output); + const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); + lines.pop(); // Remove last item that contains [v] and time in ms. + expect(lines).toEqual([ + '0 : 1 a.test.ts:6:7 › passes', + '1 : 1.1 passes › outer 1.0', + '2 : 1.2 passes › outer 1.0 › inner 1.1', + '2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', + '3 : 1.3 passes › outer 1.0 › inner 1.2', + '3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', + '1 : 1.1 passes › outer 1.0 (Xms)', + '4 : 1.4 passes › outer 2.0', + '5 : 1.5 passes › outer 2.0 › inner 2.1', + '5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', + '6 : 1.6 passes › outer 2.0 › inner 2.2', + '6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', + '4 : 1.4 passes › outer 2.0 (Xms)', + ]); +}); + +test('render steps inlint', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => {}); + await test.step('inner 1.2', async () => {}); + }); + await test.step('outer 2.0', async () => { await test.step('inner 2.1', async () => {}); + await test.step('inner 2.2', async () => {}); }); }); `, }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' }); const text = stripAnsi(result.output); - const lines = text.split('\n').filter(l => l.startsWith('0 :')); + const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); lines.pop(); // Remove last item that contains [v] and time in ms. expect(lines).toEqual([ - '0 : 1 a.test.ts:6:7 › passes › outer 1.0', - '0 : 1 a.test.ts:6:7 › passes › outer 1.0 › inner 1.1', - '0 : 1 a.test.ts:6:7 › passes › outer 1.0', - '0 : 1 a.test.ts:6:7 › passes › outer 1.0 › inner 1.1', - '0 : 1 a.test.ts:6:7 › passes › outer 1.0', '0 : 1 a.test.ts:6:7 › passes', - '0 : 1 a.test.ts:6:7 › passes › outer 2.0', - '0 : 1 a.test.ts:6:7 › passes › outer 2.0 › inner 2.1', - '0 : 1 a.test.ts:6:7 › passes › outer 2.0', - '0 : 1 a.test.ts:6:7 › passes › outer 2.0 › inner 2.1', - '0 : 1 a.test.ts:6:7 › passes › outer 2.0', + '0 : 1 a.test.ts:7:20 › passes › outer 1.0', + '0 : 1 a.test.ts:8:22 › passes › outer 1.0 › inner 1.1', + '0 : 1 a.test.ts:7:20 › passes › outer 1.0', + '0 : 1 a.test.ts:9:22 › passes › outer 1.0 › inner 1.2', + '0 : 1 a.test.ts:7:20 › passes › outer 1.0', + '0 : 1 a.test.ts:6:7 › passes', + '0 : 1 a.test.ts:11:20 › passes › outer 2.0', + '0 : 1 a.test.ts:12:22 › passes › outer 2.0 › inner 2.1', + '0 : 1 a.test.ts:11:20 › passes › outer 2.0', + '0 : 1 a.test.ts:13:22 › passes › outer 2.0 › inner 2.2', + '0 : 1 a.test.ts:11:20 › passes › outer 2.0', '0 : 1 a.test.ts:6:7 › passes', ]); }); @@ -119,7 +156,9 @@ test('render retries', async ({ runInlineTest }) => { const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms')); expect(lines).toEqual([ + `0 : 1 a.test.ts:6:7 › flaky`, `0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:6:7 › flaky (XXms)`, + `1 : 2 a.test.ts:6:7 › flaky (retry #1)`, `1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:6:7 › flaky (retry #1) (XXms)`, ]); }); @@ -173,7 +212,7 @@ test('should truncate long test names', async ({ runInlineTest }) => { function simpleAnsiRenderer(text, ttyWidth) { let lineNumber = 0; let columnNumber = 0; - const screenLines = []; + const screenLines: string[][] = []; const ensureScreenSize = () => { if (lineNumber < 0) throw new Error('Bad terminal navigation!');