mirror of
https://github.com/microsoft/playwright.git
synced 2025-01-05 19:04:43 +03:00
feat(line reporter): show stats, handle tty (#12695)
- Line reporter now shows stats in addition to the test name: ```log [chromium] › page/page-click-react.spec.ts:108:1 › should not retarget when element changes on hover 23% [21/93] Passed: 17 Flaky: 0 Failed: 0 Skipped: 4 (7s) ``` - When connected to a TTY or with `env.PLAYWRIGHT_LIVE_TERMINAL` set to anything but `'0'` or `'false'`, line reporter updates in place. - When not connected to a TTY, line reporter prints an update after each ~1% of tests done, so it never prints more than 100 lines. - Updated tests to the golden style.
This commit is contained in:
Dmitry Gozman
GitHub
parent
e9fa098bf3
commit
be817d1a53
@ -19,6 +19,7 @@ import colors from 'colors/safe';
|
||||
import fs from 'fs';
|
||||
import milliseconds from 'ms';
|
||||
import path from 'path';
|
||||
import { getAsBooleanFromENV } from 'playwright-core/lib/utils/utils';
|
||||
import StackUtils from 'stack-utils';
|
||||
import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep, Location } from '../../types/testReporter';
|
||||
|
||||
@ -51,23 +52,25 @@ export class BaseReporter implements Reporter {
|
||||
duration = 0;
|
||||
config!: FullConfig;
|
||||
suite!: Suite;
|
||||
totalTestCount = 0;
|
||||
result!: FullResult;
|
||||
private fileDurations = new Map<string, number>();
|
||||
private monotonicStartTime: number = 0;
|
||||
private _omitFailures: boolean;
|
||||
private readonly _ttyWidthForTest: number;
|
||||
readonly liveTerminal: boolean;
|
||||
readonly stats = { skipped: 0, complete: 0, total: 0, passed: 0, failed: 0, flaky: 0 };
|
||||
|
||||
constructor(options: { omitFailures?: boolean } = {}) {
|
||||
this._omitFailures = options.omitFailures || false;
|
||||
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
|
||||
this.liveTerminal = process.stdout.isTTY || getAsBooleanFromENV('PLAYWRIGHT_LIVE_TERMINAL');
|
||||
}
|
||||
|
||||
onBegin(config: FullConfig, suite: Suite) {
|
||||
this.monotonicStartTime = monotonicTime();
|
||||
this.config = config;
|
||||
this.suite = suite;
|
||||
this.totalTestCount = suite.allTests().length;
|
||||
this.stats.total = suite.allTests().length;
|
||||
}
|
||||
|
||||
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
@ -86,6 +89,16 @@ export class BaseReporter implements Reporter {
|
||||
}
|
||||
|
||||
onTestEnd(test: TestCase, result: TestResult) {
|
||||
this.stats.complete++;
|
||||
if (!this.willRetry(test)) {
|
||||
switch (test.outcome()) {
|
||||
case 'skipped': this.stats.skipped++; break;
|
||||
case 'expected': this.stats.passed++; break;
|
||||
case 'unexpected': this.stats.failed++; break;
|
||||
case 'flaky': this.stats.flaky++; break;
|
||||
}
|
||||
}
|
||||
|
||||
// Ignore any tests that are run in parallel.
|
||||
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
|
||||
if ((suite as any)._parallelMode === 'parallel')
|
||||
@ -119,7 +132,23 @@ export class BaseReporter implements Reporter {
|
||||
protected generateStartingMessage() {
|
||||
const jobs = Math.min(this.config.workers, (this.config as any).__testGroupsCount);
|
||||
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
|
||||
return `\nRunning ${this.totalTestCount} test${this.totalTestCount > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
|
||||
return `\nRunning ${this.stats.total} test${this.stats.total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`;
|
||||
}
|
||||
|
||||
protected generateStatsMessage(done: boolean) {
|
||||
// Do not report 100% until done.
|
||||
const percent = Math.min(done ? 100 : 99, Math.round(this.stats.complete / this.stats.total * 100));
|
||||
const maxExpected = done ? this.stats.total : this.stats.total - 1;
|
||||
const retriesSuffix = this.stats.complete > maxExpected ? ` (retries)` : ``;
|
||||
const message = [
|
||||
`${percent}% [${this.stats.complete}/${this.stats.total}]${retriesSuffix}`,
|
||||
`${(this.stats.passed ? colors.green : colors.gray)('Passed: ' + this.stats.passed)}`,
|
||||
`${(this.stats.flaky ? colors.red : colors.gray)('Flaky: ' + this.stats.flaky)}`,
|
||||
`${(this.stats.failed ? colors.red : colors.gray)('Failed: ' + this.stats.failed)}`,
|
||||
`${(this.stats.skipped ? colors.yellow : colors.gray)('Skipped: ' + this.stats.skipped)}`,
|
||||
colors.gray(process.env.PW_TEST_DEBUG_REPORTERS ? `(XXms)` : `(${milliseconds(monotonicTime() - this.monotonicStartTime)})`),
|
||||
].join(' ');
|
||||
return { percent, message };
|
||||
}
|
||||
|
||||
protected getSlowTests(): [string, number][] {
|
||||
|
||||
@ -18,10 +18,13 @@ import colors from 'colors/safe';
|
||||
import { BaseReporter, formatFailure, formatTestTitle } from './base';
|
||||
import { FullConfig, TestCase, Suite, TestResult, FullResult } from '../../types/testReporter';
|
||||
|
||||
const lineUp = process.env.PW_TEST_DEBUG_REPORTERS ? '<lineup>' : '\u001B[1A';
|
||||
const erase = process.env.PW_TEST_DEBUG_REPORTERS ? '<erase>' : '\u001B[2K';
|
||||
|
||||
class LineReporter extends BaseReporter {
|
||||
private _current = 0;
|
||||
private _failures = 0;
|
||||
private _lastTest: TestCase | undefined;
|
||||
private _lastPercent = -1;
|
||||
|
||||
printsToStdio() {
|
||||
return true;
|
||||
@ -30,49 +33,55 @@ class LineReporter extends BaseReporter {
|
||||
override onBegin(config: FullConfig, suite: Suite) {
|
||||
super.onBegin(config, suite);
|
||||
console.log(this.generateStartingMessage());
|
||||
console.log();
|
||||
if (this.liveTerminal)
|
||||
console.log('\n');
|
||||
}
|
||||
|
||||
override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
super.onStdOut(chunk, test, result);
|
||||
this._dumpToStdio(test, chunk, process.stdout);
|
||||
this._dumpToStdio(test, chunk, result, process.stdout);
|
||||
}
|
||||
|
||||
override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
|
||||
super.onStdErr(chunk, test, result);
|
||||
this._dumpToStdio(test, chunk, process.stderr);
|
||||
this._dumpToStdio(test, chunk, result, process.stderr);
|
||||
}
|
||||
|
||||
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
|
||||
private _testTitleLine(test: TestCase, result: TestResult | undefined) {
|
||||
const title = formatTestTitle(this.config, test);
|
||||
const titleSuffix = result?.retry ? ` (retry #${result.retry})` : '';
|
||||
return this.fitToScreen(title, titleSuffix) + colors.yellow(titleSuffix);
|
||||
}
|
||||
|
||||
private _dumpToStdio(test: TestCase | undefined, chunk: string | Buffer, result: TestResult | undefined, stream: NodeJS.WriteStream) {
|
||||
if (this.config.quiet)
|
||||
return;
|
||||
if (!process.env.PW_TEST_DEBUG_REPORTERS)
|
||||
stream.write(`\u001B[1A\u001B[2K`);
|
||||
if (this.liveTerminal)
|
||||
stream.write(lineUp + erase + lineUp + erase);
|
||||
if (test && this._lastTest !== test) {
|
||||
// Write new header for the output.
|
||||
const title = colors.gray(formatTestTitle(this.config, test));
|
||||
stream.write(this.fitToScreen(title) + `\n`);
|
||||
stream.write(this._testTitleLine(test, result) + `\n`);
|
||||
this._lastTest = test;
|
||||
}
|
||||
|
||||
stream.write(chunk);
|
||||
console.log();
|
||||
console.log('\n');
|
||||
}
|
||||
|
||||
override onTestEnd(test: TestCase, result: TestResult) {
|
||||
super.onTestEnd(test, result);
|
||||
++this._current;
|
||||
const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``;
|
||||
const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`;
|
||||
const suffix = result.retry ? ` (retry #${result.retry})` : '';
|
||||
if (process.env.PW_TEST_DEBUG_REPORTERS)
|
||||
process.stdout.write(`${title + suffix}\n`);
|
||||
else
|
||||
process.stdout.write(`\u001B[1A\u001B[2K${this.fitToScreen(title, suffix) + colors.yellow(suffix)}\n`);
|
||||
const stats = this.generateStatsMessage(false);
|
||||
if (this.liveTerminal) {
|
||||
process.stdout.write(lineUp + erase + lineUp + erase + `${this._testTitleLine(test, result)}\n${this.fitToScreen(stats.message)}\n`);
|
||||
} else {
|
||||
if (stats.percent !== this._lastPercent)
|
||||
process.stdout.write(this.fitToScreen(stats.message) + '\n');
|
||||
}
|
||||
this._lastPercent = stats.percent;
|
||||
|
||||
if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
|
||||
if (!process.env.PW_TEST_DEBUG_REPORTERS)
|
||||
process.stdout.write(`\u001B[1A\u001B[2K`);
|
||||
if (this.liveTerminal)
|
||||
process.stdout.write(lineUp + erase + lineUp + erase);
|
||||
console.log(formatFailure(this.config, test, {
|
||||
index: ++this._failures
|
||||
}).message);
|
||||
@ -81,9 +90,9 @@ class LineReporter extends BaseReporter {
|
||||
}
|
||||
|
||||
override async onEnd(result: FullResult) {
|
||||
if (!process.env.PW_TEST_DEBUG_REPORTERS)
|
||||
process.stdout.write(`\u001B[1A\u001B[2K`);
|
||||
await super.onEnd(result);
|
||||
if (this.liveTerminal)
|
||||
process.stdout.write(lineUp + erase + lineUp + erase);
|
||||
this.epilogue(false);
|
||||
}
|
||||
}
|
||||
|
||||
@ -29,11 +29,9 @@ class ListReporter extends BaseReporter {
|
||||
private _lastRow = 0;
|
||||
private _testRows = new Map<TestCase, number>();
|
||||
private _needNewLine = false;
|
||||
private readonly _liveTerminal: string | boolean | undefined;
|
||||
|
||||
constructor(options: { omitFailures?: boolean } = {}) {
|
||||
super(options);
|
||||
this._liveTerminal = process.stdout.isTTY || !!process.env.PWTEST_TTY_WIDTH;
|
||||
}
|
||||
|
||||
printsToStdio() {
|
||||
@ -47,7 +45,7 @@ class ListReporter extends BaseReporter {
|
||||
}
|
||||
|
||||
onTestBegin(test: TestCase, result: TestResult) {
|
||||
if (this._liveTerminal) {
|
||||
if (this.liveTerminal) {
|
||||
if (this._needNewLine) {
|
||||
this._needNewLine = false;
|
||||
process.stdout.write('\n');
|
||||
@ -71,7 +69,7 @@ class ListReporter extends BaseReporter {
|
||||
}
|
||||
|
||||
onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
|
||||
if (!this._liveTerminal)
|
||||
if (!this.liveTerminal)
|
||||
return;
|
||||
if (step.category !== 'test.step')
|
||||
return;
|
||||
@ -79,7 +77,7 @@ class ListReporter extends BaseReporter {
|
||||
}
|
||||
|
||||
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
|
||||
if (!this._liveTerminal)
|
||||
if (!this.liveTerminal)
|
||||
return;
|
||||
if (step.category !== 'test.step')
|
||||
return;
|
||||
@ -91,7 +89,7 @@ class ListReporter extends BaseReporter {
|
||||
return;
|
||||
const text = chunk.toString('utf-8');
|
||||
this._needNewLine = text[text.length - 1] !== '\n';
|
||||
if (this._liveTerminal) {
|
||||
if (this.liveTerminal) {
|
||||
const newLineCount = text.split('\n').length - 1;
|
||||
this._lastRow += newLineCount;
|
||||
}
|
||||
@ -116,7 +114,7 @@ class ListReporter extends BaseReporter {
|
||||
}
|
||||
const suffix = this._retrySuffix(result) + duration;
|
||||
|
||||
if (this._liveTerminal) {
|
||||
if (this.liveTerminal) {
|
||||
this._updateTestLine(test, text, suffix);
|
||||
} else {
|
||||
if (this._needNewLine) {
|
||||
|
||||
@ -631,12 +631,12 @@ test('should not hang and report results when worker process suddenly exits duri
|
||||
test('failing due to afterall', () => {});
|
||||
test.afterAll(() => { process.exit(0); });
|
||||
`
|
||||
}, { reporter: 'line' });
|
||||
}, { reporter: 'line' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('Worker process exited unexpectedly');
|
||||
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall');
|
||||
expect(stripAnsi(result.output)).toContain('a.spec.js:6:7 › failing due to afterall');
|
||||
});
|
||||
|
||||
test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => {
|
||||
|
||||
@ -255,6 +255,10 @@ export function stripAnsi(str: string): string {
|
||||
return str.replace(asciiRegex, '');
|
||||
}
|
||||
|
||||
export function trimLineEnds(text: string): string {
|
||||
return text.split('\n').map(line => line.trimEnd()).join('\n');
|
||||
}
|
||||
|
||||
export function countTimes(s: string, sub: string): number {
|
||||
let result = 0;
|
||||
for (let index = 0; index !== -1;) {
|
||||
|
||||
@ -230,8 +230,8 @@ test('should add line in addition to file json without CI', async ({ runInlineTe
|
||||
expect(1).toBe(1);
|
||||
});
|
||||
`,
|
||||
}, { reporter: '' }, { PW_TEST_DEBUG_REPORTERS: '1' });
|
||||
}, { reporter: '' }, { PLAYWRIGHT_LIVE_TERMINAL: '1' });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(stripAnsi(result.output)).toContain('[1/1] a.test.js:6:7 › one');
|
||||
expect(stripAnsi(result.output)).toContain('a.test.js:6:7 › one');
|
||||
expect(fs.existsSync(testInfo.outputPath('a.json'))).toBeTruthy();
|
||||
});
|
||||
|
||||
@ -14,71 +14,238 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { test, expect, stripAnsi } from './playwright-test-fixtures';
|
||||
import { test, expect, trimLineEnds, stripAnsi } from './playwright-test-fixtures';
|
||||
|
||||
test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
test('should work with tty', async ({ runInlineTest }, testInfo) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
test('one', async ({}) => {
|
||||
expect(1).toBe(0);
|
||||
test('passing test', async ({}) => {
|
||||
});
|
||||
`,
|
||||
}, { retries: 3, reporter: 'line' });
|
||||
const text = stripAnsi(result.output);
|
||||
expect(text).toContain('[1/1] a.test.js:6:7 › one');
|
||||
expect(text).toContain('[2/1] (retries) a.test.js:6:7 › one (retry #1)');
|
||||
expect(text).toContain('[3/1] (retries) a.test.js:6:7 › one (retry #2)');
|
||||
expect(text).toContain('[4/1] (retries) a.test.js:6:7 › one (retry #3)');
|
||||
expect(text).toContain('1 failed');
|
||||
expect(text).toContain('1) a.test');
|
||||
expect(text).not.toContain('2) a.test');
|
||||
expect(text).toContain('Retry #1 ----');
|
||||
expect(text).toContain('Retry #2 ----');
|
||||
expect(text).toContain('Retry #3 ----');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
test('render flaky', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
test('one', async ({}, testInfo) => {
|
||||
expect(testInfo.retry).toBe(3);
|
||||
test.skip('skipped test', async ({}) => {
|
||||
});
|
||||
`,
|
||||
}, { retries: 3, reporter: 'line' });
|
||||
const text = stripAnsi(result.output);
|
||||
expect(text).toContain('1 flaky');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
const { test } = pwt;
|
||||
test('foobar', async ({}, testInfo) => {
|
||||
test('flaky test', async ({}, testInfo) => {
|
||||
expect(testInfo.retry).toBe(1);
|
||||
});
|
||||
`
|
||||
}, { retries: '1', reporter: 'line' });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.flaky).toBe(1);
|
||||
expect(stripAnsi(result.output)).toContain('expect(testInfo.retry).toBe(1)');
|
||||
});
|
||||
|
||||
test('should work on CI', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
test('one', async ({}) => {
|
||||
test('failing test', async ({}) => {
|
||||
expect(1).toBe(0);
|
||||
});
|
||||
`,
|
||||
}, { reporter: 'line' }, { CI: '1' });
|
||||
const text = stripAnsi(result.output);
|
||||
expect(text).toContain('[1/1] a.test.js:6:7 › one');
|
||||
expect(text).toContain('1 failed');
|
||||
expect(text).toContain('1) a.test');
|
||||
}, { retries: '1', reporter: 'line', workers: '1' }, {
|
||||
PLAYWRIGHT_LIVE_TERMINAL: '1',
|
||||
FORCE_COLOR: '0',
|
||||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`Running 4 tests using 1 worker
|
||||
|
||||
|
||||
<lineup><erase><lineup><erase>a.test.js:6:7 › passing test
|
||||
25% [1/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:8:12 › skipped test
|
||||
50% [2/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:10:7 › flaky test
|
||||
75% [3/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:10:7 › flaky test (retry #1)
|
||||
99% [4/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase> 1) a.test.js:10:7 › flaky test ===================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 1
|
||||
Received: 0
|
||||
|
||||
9 | });
|
||||
10 | test('flaky test', async ({}, testInfo) => {
|
||||
> 11 | expect(testInfo.retry).toBe(1);
|
||||
| ^
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
14 | expect(1).toBe(0);
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:11:32
|
||||
|
||||
|
||||
<lineup><erase><lineup><erase>a.test.js:13:7 › failing test
|
||||
99% [5/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase>a.test.js:13:7 › failing test (retry #1)
|
||||
99% [6/4] (retries) Passed: 1 Flaky: 1 Failed: 1 Skipped: 1 (XXms)
|
||||
<lineup><erase><lineup><erase> 2) a.test.js:13:7 › failing test =================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 0
|
||||
Received: 1
|
||||
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
> 14 | expect(1).toBe(0);
|
||||
| ^
|
||||
15 | });
|
||||
16 |
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
Retry #1 ---------------------------------------------------------------------------------------
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 0
|
||||
Received: 1
|
||||
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
> 14 | expect(1).toBe(0);
|
||||
| ^
|
||||
15 | });
|
||||
16 |
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
|
||||
<lineup><erase><lineup><erase>
|
||||
1 failed
|
||||
a.test.js:13:7 › failing test ==================================================================
|
||||
1 flaky
|
||||
a.test.js:10:7 › flaky test ====================================================================
|
||||
1 skipped
|
||||
1 passed`));
|
||||
});
|
||||
|
||||
test('should work with non-tty', async ({ runInlineTest }, testInfo) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
test('passing test', async ({}) => {
|
||||
});
|
||||
test.skip('skipped test', async ({}) => {
|
||||
});
|
||||
test('flaky test', async ({}, testInfo) => {
|
||||
expect(testInfo.retry).toBe(1);
|
||||
});
|
||||
test('failing test', async ({}) => {
|
||||
expect(1).toBe(0);
|
||||
});
|
||||
`,
|
||||
}, { retries: '1', reporter: 'line', workers: '1' }, {
|
||||
FORCE_COLOR: '0',
|
||||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(trimLineEnds(result.output)).toContain(trimLineEnds(`
|
||||
Running 4 tests using 1 worker
|
||||
25% [1/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)
|
||||
50% [2/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
75% [3/4] Passed: 1 Flaky: 0 Failed: 0 Skipped: 1 (XXms)
|
||||
99% [4/4] (retries) Passed: 1 Flaky: 1 Failed: 0 Skipped: 1 (XXms)
|
||||
1) a.test.js:10:7 › flaky test ===================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 1
|
||||
Received: 0
|
||||
|
||||
9 | });
|
||||
10 | test('flaky test', async ({}, testInfo) => {
|
||||
> 11 | expect(testInfo.retry).toBe(1);
|
||||
| ^
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
14 | expect(1).toBe(0);
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:11:32
|
||||
|
||||
|
||||
2) a.test.js:13:7 › failing test =================================================================
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 0
|
||||
Received: 1
|
||||
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
> 14 | expect(1).toBe(0);
|
||||
| ^
|
||||
15 | });
|
||||
16 |
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
Retry #1 ---------------------------------------------------------------------------------------
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 0
|
||||
Received: 1
|
||||
|
||||
12 | });
|
||||
13 | test('failing test', async ({}) => {
|
||||
> 14 | expect(1).toBe(0);
|
||||
| ^
|
||||
15 | });
|
||||
16 |
|
||||
|
||||
at ${testInfo.outputPath('a.test.js')}:14:19
|
||||
|
||||
|
||||
|
||||
1 failed
|
||||
a.test.js:13:7 › failing test ==================================================================
|
||||
1 flaky
|
||||
a.test.js:10:7 › flaky test ====================================================================
|
||||
1 skipped
|
||||
1 passed`));
|
||||
});
|
||||
|
||||
test('should respect tty width', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
test('passing test', async ({}) => {
|
||||
});
|
||||
test.skip('skipped test', async ({}) => {
|
||||
});
|
||||
test('flaky test', async ({}, testInfo) => {
|
||||
expect(testInfo.retry).toBe(1);
|
||||
});
|
||||
test('failing test', async ({}) => {
|
||||
expect(1).toBe(0);
|
||||
});
|
||||
`,
|
||||
}, { retries: '1', reporter: 'line', workers: '1' }, {
|
||||
PLAYWRIGHT_LIVE_TERMINAL: '1',
|
||||
FORCE_COLOR: '0',
|
||||
PWTEST_TTY_WIDTH: '30',
|
||||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
const text = stripAnsi(result.output);
|
||||
expect(text).toContain(`a.test.js:6:7 › passing test`);
|
||||
expect(text).toContain(`25% [1/4] Passed: 1 Flaky: 0 F`);
|
||||
expect(text).not.toContain(`25% [1/4] Passed: 1 Flaky: 0 Fa`);
|
||||
expect(text).toContain(`a.test.js:10:7 › fl (retry #1)`);
|
||||
expect(text).toContain(`99% [4/4] (retries) Passed: 1 `);
|
||||
expect(text).not.toContain(`99% [4/4] (retries) Passed: 1 F`);
|
||||
});
|
||||
|
||||
test('should spare status updates in non-tty mode', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test } = pwt;
|
||||
for (let i = 0; i < 300; i++) {
|
||||
test('test' + i, () => {});
|
||||
}
|
||||
`,
|
||||
}, { reporter: 'line', workers: '1' }, {
|
||||
FORCE_COLOR: '0',
|
||||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
const lines = [`Running 300 tests using 1 worker`, `0% [1/300] Passed: 1 Flaky: 0 Failed: 0 Skipped: 0 (XXms)`];
|
||||
for (let i = 1; i <= 99; i++)
|
||||
lines.push(`${i}% [${3 * i - 1}/300] Passed: ${3 * i - 1} Flaky: 0 Failed: 0 Skipped: 0 (XXms)`);
|
||||
lines.push('');
|
||||
lines.push(' 300 passed');
|
||||
expect(trimLineEnds(result.output)).toContain(lines.join('\n'));
|
||||
});
|
||||
|
||||
@ -63,7 +63,7 @@ test('render steps', async ({ runInlineTest }) => {
|
||||
});
|
||||
});
|
||||
`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' });
|
||||
const text = stripAnsi(result.output);
|
||||
const lines = text.split('\n').filter(l => l.startsWith('0 :'));
|
||||
lines.pop(); // Remove last item that contains [v] and time in ms.
|
||||
@ -91,7 +91,7 @@ test('render retries', async ({ runInlineTest }) => {
|
||||
expect(testInfo.retry).toBe(1);
|
||||
});
|
||||
`,
|
||||
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
|
||||
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIVE_TERMINAL: '1' });
|
||||
const text = stripAnsi(result.output);
|
||||
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/[\dm]+s/, 'XXms'));
|
||||
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
|
||||
@ -121,7 +121,7 @@ test('should truncate long test names', async ({ runInlineTest }) => {
|
||||
test.skip('skipped very long name', async () => {
|
||||
});
|
||||
`,
|
||||
}, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: 50 });
|
||||
}, { reporter: 'list', retries: 0 }, { PLAYWRIGHT_LIVE_TERMINAL: '1', PWTEST_TTY_WIDTH: 50 });
|
||||
const text = stripAnsi(result.output);
|
||||
const positiveStatusMarkPrefix = process.platform === 'win32' ? 'ok' : '✓ ';
|
||||
const negativateStatusMarkPrefix = process.platform === 'win32' ? 'x ' : '✘ ';
|
||||
|
||||
Loading…
Reference in New Issue
Block a user