microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-01-05 19:04:43 +03:00
Code Issues Projects Releases Wiki Activity

chore: render steps in list reporter (#18269)

Browse Source
This commit is contained in:
Pavel Feldman 2022-10-24 18:54:53 -04:00 committed by GitHub
parent 3f850d27e9
commit ab78865a8d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 190 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
docs/src/test-reporters-js.md
View File

@ -133,16 +133,40 @@ Here is an example output in the middle of a test run. Failures will be listed a
npx playwright test --reporter=list
Running 124 tests using 6 workers
✓ should access error in env (438ms)
✓ handle long test names (515ms)
x 1) render expected (691ms)
✓ should timeout (932ms)
should repeat each:
✓ should respect enclosing .gitignore (569ms)
should teardown env after timeout:
should respect excluded tests:
✓ should handle env beforeEach error (638ms)
should respect enclosing .gitignore:
1 ✓ should access error in env (438ms)
2 ✓ handle long test names (515ms)
3 x 1) render expected (691ms)
4 ✓ should timeout (932ms)
5 should repeat each:
6 ✓ should respect enclosing .gitignore (569ms)
7 should teardown env after timeout:
8 should respect excluded tests:
9 ✓ should handle env beforeEach error (638ms)
10 should respect enclosing .gitignore:
```
You can opt into the step rendering via passing the following config option:
```js tab=js-js
// playwright.config.js
// @ts-check
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [['list', { printSteps: true }],
};
module.exports = config;
```
```js tab=js-ts
// playwright.config.ts
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [['list', { printSteps: true }],
};
export default config;
```
### Line reporter
@ -246,11 +270,11 @@ You can also configure `host` and `port` that are used to serve the HTML report.
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [ ['html', {
reporter: [['html', {
open: 'never',
host: '0.0.0.0',
port: 9223,
}] ],
}]],
};
module.exports = config;
@ -261,7 +285,7 @@ module.exports = config;
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [ ['html', { open: 'never' }] ],
reporter: [['html', { open: 'never' }]],
};
export default config;
```
@ -276,7 +300,7 @@ In configuration file, pass options directly:
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [ ['html', { outputFolder: 'my-report' }] ],
reporter: [['html', { outputFolder: 'my-report' }]],
};
module.exports = config;
@ -287,7 +311,7 @@ module.exports = config;
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [ ['html', { outputFolder: 'my-report' }] ],
reporter: [['html', { outputFolder: 'my-report' }]],
};
export default config;
```
@ -334,7 +358,7 @@ In configuration file, pass options directly:
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [ ['json', { outputFile: 'results.json' }] ],
reporter: [['json', { outputFile: 'results.json' }]],
};
module.exports = config;
@ -345,7 +369,7 @@ module.exports = config;
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [ ['json', { outputFile: 'results.json' }] ],
reporter: [['json', { outputFile: 'results.json' }]],
};
export default config;
```
@ -377,7 +401,7 @@ In configuration file, pass options directly:
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [ ['junit', { outputFile: 'results.xml' }] ],
reporter: [['junit', { outputFile: 'results.xml' }]],
};
module.exports = config;
@ -388,7 +412,7 @@ module.exports = config;
import type { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [ ['junit', { outputFile: 'results.xml' }] ],
reporter: [['junit', { outputFile: 'results.xml' }]],
};
export default config;
```
@ -421,7 +445,7 @@ const xrayOptions = {
};
const config = {
reporter: [ ['junit', xrayOptions] ]
reporter: [['junit', xrayOptions]]
};
module.exports = config;
@ -449,7 +473,7 @@ const xrayOptions = {
};
const config: PlaywrightTestConfig = {
reporter: [ ['junit', xrayOptions] ]
reporter: [['junit', xrayOptions]]
};
export default config;
@ -496,7 +520,7 @@ The following configuration sample enables embedding attachments by using the `t
/** @type {import('@playwright/test').PlaywrightTestConfig} */
const config = {
reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ],
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
};
module.exports = config;
@ -507,7 +531,7 @@ module.exports = config;
import { PlaywrightTestConfig } from '@playwright/test';
const config: PlaywrightTestConfig = {
reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ],
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
};
export default config;

4
packages/playwright-test/src/reporters/base.ts
View File

@ -352,7 +352,7 @@ function relativeTestPath(config: FullConfig, test: TestCase): string {
return relativeFilePath(config, test.location.file);
}
function stepSuffix(step: TestStep | undefined) {
export function stepSuffix(step: TestStep | undefined) {
const stepTitles = step ? step.titlePath() : [];
return stepTitles.map(t => ' › ' + t).join('');
}
@ -364,7 +364,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS
if (omitLocation)
location = `${relativeTestPath(config, test)}`;
else
location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`;
location = `${relativeTestPath(config, test)}:${step?.location?.line ?? test.location.line}:${step?.location?.column ?? test.location.column}`;
const projectTitle = projectName ? `[${projectName}] › ` : '';
return `${projectTitle}${location} › ${titles.join(' › ')}${stepSuffix(step)}`;
}

129
packages/playwright-test/src/reporters/list.ts
View File

@ -16,7 +16,7 @@
/* eslint-disable no-console */
import { colors, ms as milliseconds } from 'playwright-core/lib/utilsBundle';
import { BaseReporter, formatError, formatTestTitle, stripAnsiEscapes } from './base';
import { BaseReporter, formatError, formatTestTitle, stepSuffix, stripAnsiEscapes } from './base';
import type { FullConfig, FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
// Allow it in the Visual Studio Code Terminal and the new Windows Terminal
@ -28,12 +28,16 @@ class ListReporter extends BaseReporter {
private _lastRow = 0;
private _lastColumn = 0;
private _testRows = new Map<TestCase, number>();
private _resultIndex = new Map<TestResult, number>();
private _stepRows = new Map<TestStep, number>();
private _resultIndex = new Map<TestResult, string>();
private _stepIndex = new Map<TestStep, string>();
private _needNewLine = false;
private readonly _liveTerminal: string | boolean | undefined;
private _printSteps: boolean;
constructor(options: { omitFailures?: boolean } = {}) {
constructor(options: { omitFailures?: boolean, printSteps?: boolean } = {}) {
super(options);
this._printSteps = options.printSteps || !!process.env.PW_TEST_DEBUG_REPORTERS_PRINT_STEPS;
this._liveTerminal = process.stdout.isTTY || !!process.env.PWTEST_TTY_WIDTH;
}
@ -50,12 +54,13 @@ class ListReporter extends BaseReporter {
onTestBegin(test: TestCase, result: TestResult) {
if (this._liveTerminal)
this._maybeWriteNewLine();
this._resultIndex.set(result, this._resultIndex.size + 1);
this._testRows.set(test, this._lastRow++);
this._resultIndex.set(result, String(this._resultIndex.size + 1));
if (this._liveTerminal) {
const prefix = this._testPrefix(result, '');
this._testRows.set(test, this._lastRow);
const index = this._resultIndex.get(result)!;
const prefix = this._testPrefix(index, '');
const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result);
process.stdout.write(prefix + this.fitToScreen(line, prefix) + '\n');
this._appendLine(line, prefix);
}
}
@ -70,19 +75,52 @@ class ListReporter extends BaseReporter {
}
onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
if (!this._liveTerminal)
return;
if (step.category !== 'test.step')
return;
this._updateTestLine(test, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(result, ''));
const testIndex = this._resultIndex.get(result)!;
if (!this._printSteps) {
if (this._liveTerminal)
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
return;
}
const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1;
(result as any)[lastStepOrdinalSymbol] = ordinal;
const stepIndex = `${testIndex}.${ordinal}`;
this._stepIndex.set(step, stepIndex);
if (this._liveTerminal)
this._maybeWriteNewLine();
if (this._liveTerminal) {
this._stepRows.set(step, this._lastRow);
const prefix = this._testPrefix(stepIndex, '');
const line = test.title + colors.dim(stepSuffix(step));
this._appendLine(line, prefix);
}
}
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
if (!this._liveTerminal)
return;
if (step.category !== 'test.step')
return;
this._updateTestLine(test, colors.dim(formatTestTitle(this.config, test, step.parent)) + this._retrySuffix(result), this._testPrefix(result, ''));
const testIndex = this._resultIndex.get(result)!;
if (!this._printSteps) {
if (this._liveTerminal)
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step.parent)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
return;
}
const index = this._stepIndex.get(step)!;
const title = test.title + colors.dim(stepSuffix(step));
const prefix = this._testPrefix(index, '');
let text = '';
if (step.error)
text = colors.red(title);
else
text = title;
text += colors.dim(` (${milliseconds(step.duration)})`);
this._updateOrAppendLine(this._stepRows.get(step)!, text, prefix);
}
private _maybeWriteNewLine() {
@ -125,68 +163,77 @@ class ListReporter extends BaseReporter {
const title = formatTestTitle(this.config, test);
let prefix = '';
let text = '';
const index = this._resultIndex.get(result)!;
if (result.status === 'skipped') {
prefix = this._testPrefix(result, colors.green('-'));
prefix = this._testPrefix(index, colors.green('-'));
// Do not show duration for skipped.
text = colors.cyan(title) + this._retrySuffix(result);
} else {
const statusMark = result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK;
if (result.status === test.expectedStatus) {
prefix = this._testPrefix(result, colors.green(statusMark));
text = colors.dim(title);
prefix = this._testPrefix(index, colors.green(statusMark));
text = title;
} else {
prefix = this._testPrefix(result, colors.red(statusMark));
prefix = this._testPrefix(index, colors.red(statusMark));
text = colors.red(title);
}
text += this._retrySuffix(result) + colors.dim(` (${milliseconds(result.duration)})`);
}
this._updateOrAppendLine(this._testRows.get(test)!, text, prefix);
}
private _updateOrAppendLine(row: number, text: string, prefix: string) {
if (this._liveTerminal) {
this._updateTestLine(test, text, prefix);
this._updateLine(row, text, prefix);
} else {
this._maybeWriteNewLine();
process.stdout.write(prefix + text);
process.stdout.write('\n');
this._appendLine(text, prefix);
}
}
private _updateTestLine(test: TestCase, line: string, prefix: string) {
if (process.env.PW_TEST_DEBUG_REPORTERS)
this._updateTestLineForTest(test, line, prefix);
else
this._updateTestLineForTTY(test, line, prefix);
private _appendLine(text: string, prefix: string) {
const line = prefix + this.fitToScreen(text, prefix);
if (process.env.PW_TEST_DEBUG_REPORTERS) {
process.stdout.write(this._lastRow + ' : ' + line + '\n');
} else {
process.stdout.write(line);
process.stdout.write('\n');
}
++this._lastRow;
}
private _updateTestLineForTTY(test: TestCase, line: string, prefix: string) {
const testRow = this._testRows.get(test)!;
private _updateLine(row: number, text: string, prefix: string) {
const line = prefix + this.fitToScreen(text, prefix);
if (process.env.PW_TEST_DEBUG_REPORTERS)
process.stdout.write(row + ' : ' + line + '\n');
else
this._updateLineForTTY(row, line);
}
private _updateLineForTTY(row: number, line: string) {
// Go up if needed
if (testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
if (row !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - row}A`);
// Erase line, go to the start
process.stdout.write('\u001B[2K\u001B[0G');
process.stdout.write(prefix + this.fitToScreen(line, prefix));
process.stdout.write(line);
// Go down if needed.
if (testRow !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
if (row !== this._lastRow)
process.stdout.write(`\u001B[${this._lastRow - row}E`);
if (process.env.PWTEST_TTY_WIDTH)
process.stdout.write('\n'); // For testing.
}
private _testPrefix(result: TestResult, statusMark: string) {
const index = this._resultIndex.get(result)!;
private _testPrefix(index: string, statusMark: string) {
const statusMarkLength = stripAnsiEscapes(statusMark).length;
return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + colors.dim(String(index) + ' ');
return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + colors.dim(index + ' ');
}
private _retrySuffix(result: TestResult) {
return (result.retry ? colors.yellow(` (retry #${result.retry})`) : '');
}
private _updateTestLineForTest(test: TestCase, line: string, prefix: string) {
const testRow = this._testRows.get(test)!;
process.stdout.write(testRow + ' : ' + prefix + line + '\n');
}
override onError(error: TestError): void {
super.onError(error);
this._maybeWriteNewLine();
@ -202,4 +249,6 @@ class ListReporter extends BaseReporter {
}
}
const lastStepOrdinalSymbol = Symbol('lastStepOrdinal');
export default ListReporter;

65
tests/playwright-test/reporter-list.spec.ts
View File

@ -58,30 +58,67 @@ test('render steps', async ({ runInlineTest }) => {
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
const text = stripAnsi(result.output);
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:6:7 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)',
]);
});
test('render steps inlint', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = stripAnsi(result.output);
const lines = text.split('\n').filter(l => l.startsWith('0 :'));
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:6:7 › passes › outer 1.0',
'0 : 1 a.test.ts:6:7 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:6:7 › passes › outer 1.0',
'0 : 1 a.test.ts:6:7 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:6:7 › passes › outer 1.0',
'0 : 1 a.test.ts:6:7 › passes',
'0 : 1 a.test.ts:6:7 › passes › outer 2.0',
'0 : 1 a.test.ts:6:7 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:6:7 › passes › outer 2.0',
'0 : 1 a.test.ts:6:7 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:6:7 › passes › outer 2.0',
'0 : 1 a.test.ts:7:20 › passes › outer 1.0',
'0 : 1 a.test.ts:8:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:7:20 › passes › outer 1.0',
'0 : 1 a.test.ts:9:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:7:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:7 › passes',
'0 : 1 a.test.ts:11:20 › passes › outer 2.0',
'0 : 1 a.test.ts:12:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:11:20 › passes › outer 2.0',
'0 : 1 a.test.ts:13:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:11:20 › passes › outer 2.0',
'0 : 1 a.test.ts:6:7 › passes',
]);
});
@ -119,7 +156,9 @@ test('render retries', async ({ runInlineTest }) => {
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:6:7 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:6:7 › flaky (XXms)`,
`1 : 2 a.test.ts:6:7 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:6:7 › flaky (retry #1) (XXms)`,
]);
});
@ -173,7 +212,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
function simpleAnsiRenderer(text, ttyWidth) {
let lineNumber = 0;
let columnNumber = 0;
const screenLines = [];
const screenLines: string[][] = [];
const ensureScreenSize = () => {
if (lineNumber < 0)
throw new Error('Bad terminal navigation!');