microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-01-05 19:04:43 +03:00
Code Issues Projects Releases Wiki Activity

chore: run more reporter tests through blob report, some fixes (#23765)

Browse Source
This commit is contained in:
Yury Semikhatsky 2023-06-16 21:30:55 -07:00 committed by GitHub
parent 09b1e3ffa9
commit f6d86c20f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1953 additions and 1908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/playwright-test/src/isomorphic/teleReceiver.ts
View File

@ -232,6 +232,7 @@ export class TeleReporterReceiver {
result.status = payload.status;
result.statusEx = payload.status;
result.errors = payload.errors;
result.error = result.errors?.[0];
result.attachments = this._parseAttachments(payload.attachments);
this._reporter.onTestEnd?.(test, result);
}
@ -242,7 +243,10 @@ export class TeleReporterReceiver {
const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined;
const step: TestStep = {
titlePath: () => [],
titlePath: () => {
const parentPath = parentStep?.titlePath() || [];
return [...parentPath, payload.title];
},
title: payload.title,
category: payload.category,
location: this._absoluteLocation(payload.location),

2
packages/playwright-test/src/reporters/blob.ts
View File

@ -92,7 +92,7 @@ export class BlobReporter extends TeleReporterEmitter {
override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] {
return super._serializeAttachments(attachments).map(attachment => {
if (!attachment.path || !fs.statSync(attachment.path).isFile())
if (!attachment.path || !fs.statSync(attachment.path, { throwIfNoEntry: false })?.isFile())
return attachment;
// Add run guid to avoid clashes between shards.
const sha1 = calculateSha1(attachment.path + this._salt);

9
tests/playwright-test/playwright-test-fixtures.ts
View File

@ -88,7 +88,7 @@ export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-tes
const configFile = (baseDir: string, files: Files): string | undefined => {
for (const [name, content] of Object.entries(files)) {
if (name.includes('playwright.config')) {
if (content.includes('reporter:'))
if (content.includes('reporter:') || content.includes('reportSlowTests:'))
return path.resolve(baseDir, name);
}
}
@ -349,12 +349,7 @@ export const test = base
const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir;
const testProcess = childProcess({
command,
env: cleanEnv({
PW_TEST_DEBUG_REPORTERS: '1',
PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1',
PWTEST_TTY_WIDTH: '80',
...env
}),
env: cleanEnv(env),
cwd,
});
const { exitCode } = await testProcess.exited;

676
tests/playwright-test/reporter-base.spec.ts
View File

@ -17,346 +17,352 @@
import { test, expect } from './playwright-test-fixtures';
import * as path from 'path';
test('handle long test names', async ({ runInlineTest }) => {
const title = 'title'.repeat(30);
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('${title}', async ({}) => {
expect(1).toBe(0);
});
`,
});
expect(result.output).toContain('expect(1).toBe');
expect(result.exitCode).toBe(1);
});
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('print the error name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
test('handle long test names', async ({ runInlineTest }) => {
const title = 'title'.repeat(30);
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('${title}', async ({}) => {
expect(1).toBe(0);
});
`,
});
expect(result.output).toContain('expect(1).toBe');
expect(result.exitCode).toBe(1);
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
});
test('print should print the error name without a message', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error();
error.name = 'FooBarError';
throw error;
test('print the error name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError');
});
test('should print an error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
expect(result.output).not.toContain('at a.spec.ts:5');
expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`);
expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`);
expect(result.output).toContain(`> 4 | const error = new Error('my-message');`);
});
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'node_modules/utils/utils.js': `
function assert(value) {
if (!value)
throw new Error('Assertion error');
}
module.exports = { assert };
`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
const { assert } = require('utils/utils.js');
test('fail', async ({}) => {
assert(false);
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('Error: Assertion error');
expect(output).toContain('a.spec.ts:4:11 › fail');
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
expect(output).toContain(`> 5 | assert(false);`);
expect(output).toContain(` | ^`);
expect(output).toContain(`utils.js:4`);
expect(output).toContain(`a.spec.ts:5:9`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function ohMy() {
throw new Error('oh my');
}
`,
'a.spec.ts': `
import { ohMy } from './helper';
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
ohMy();
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: oh my');
expect(result.output).toContain(` 2 | export function ohMy() {`);
expect(result.output).toContain(` > 3 | throw new Error('oh my');`);
expect(result.output).toContain(` | ^`);
});
test('should print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'foo' },
{ name: 'bar' },
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: { max: 0, threshold: 2400 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 2500));
});
`,
'dir/b.test.js': `
import { test, expect } from '@playwright/test';
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 1));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(8);
expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`);
expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`);
});
test('should not print slow parallel tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 500 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test.describe.parallel('suite', () => {
test('inner slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('inner fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
test('print should print the error name without a message', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error();
error.name = 'FooBarError';
throw error;
});
`
});
`,
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError');
});
test('should print an error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
expect(result.output).not.toContain('at a.spec.ts:5');
expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`);
expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`);
expect(result.output).toContain(`> 4 | const error = new Error('my-message');`);
});
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'node_modules/utils/utils.js': `
function assert(value) {
if (!value)
throw new Error('Assertion error');
}
module.exports = { assert };
`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
const { assert } = require('utils/utils.js');
test('fail', async ({}) => {
assert(false);
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('Error: Assertion error');
expect(output).toContain('a.spec.ts:4:15 › fail');
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
expect(output).toContain(`> 5 | assert(false);`);
expect(output).toContain(` | ^`);
expect(output).toContain(`utils.js:4`);
expect(output).toContain(`a.spec.ts:5:13`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function ohMy() {
throw new Error('oh my');
}
`,
'a.spec.ts': `
import { ohMy } from './helper';
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
ohMy();
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: oh my');
expect(result.output).toContain(` 2 | export function ohMy() {`);
expect(result.output).toContain(` > 3 | throw new Error('oh my');`);
expect(result.output).toContain(` | ^`);
});
test('should print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'foo' },
{ name: 'bar' },
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: { max: 0, threshold: 2400 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 2500));
});
`,
'dir/b.test.js': `
import { test, expect } from '@playwright/test';
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 1));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(8);
expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`);
expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`);
});
test('should not print slow parallel tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 500 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test.describe.parallel('suite', () => {
test('inner slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('inner fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.output).not.toContain('Slow test file');
});
test('should not print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: null,
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.output).not.toContain('Slow test');
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should print flaky timeouts', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
if (!testInfo.retry)
await new Promise(f => setTimeout(f, 2000));
});
`
}, { retries: '1', reporter: 'list', timeout: '1000' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
test('should print stack-less errors', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const e = new Error('Hello');
delete e.stack;
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Hello');
});
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async function myTest({}) {
const e = new Error('Hello');
// Force stack to contain "Hello".
// Otherwise it is computed lazy and will get 'foo bar' instead.
e.stack;
e.message = 'foo bar';
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('foo bar');
expect(output).toContain('function myTest');
});
test('should print "no tests found" error', async ({ runInlineTest }) => {
const result = await runInlineTest({ });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('No tests found');
});
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.attachments.push({
name: 'foo.txt',
body: undefined,
contentType: 'text/plain'
});
expect(1).toBe(2);
});
`,
});
expect(result.output).not.toContain('Error in reporter');
expect(result.failed).toBe(1);
expect(result.exitCode).toBe(1);
});
test('should report fatal errors at the end', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
'b.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
}, { reporter: 'list' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.output).toContain('2 errors were not a part of any test, see above for details');
});
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('should work', () => {});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
});
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.output).not.toContain('Slow test file');
});
test('should not print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: null,
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.output).not.toContain('Slow test');
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should print flaky timeouts', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
if (!testInfo.retry)
await new Promise(f => setTimeout(f, 2000));
});
`
}, { retries: '1', reporter: 'list', timeout: '1000' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
test('should print stack-less errors', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const e = new Error('Hello');
delete e.stack;
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Hello');
});
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async function myTest({}) {
const e = new Error('Hello');
// Force stack to contain "Hello".
// Otherwise it is computed lazy and will get 'foo bar' instead.
e.stack;
e.message = 'foo bar';
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('foo bar');
expect(output).toContain('function myTest');
});
test('should print "no tests found" error', async ({ runInlineTest }) => {
const result = await runInlineTest({ });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('No tests found');
});
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.attachments.push({
name: 'foo.txt',
body: undefined,
contentType: 'text/plain'
});
expect(1).toBe(2);
});
`,
});
expect(result.output).not.toContain('Error in reporter');
expect(result.failed).toBe(1);
expect(result.exitCode).toBe(1);
});
test('should report fatal errors at the end', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
'b.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
}, { reporter: 'list' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.output).toContain('2 errors were not a part of any test, see above for details');
});
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('should work', () => {});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
});
}

2
tests/playwright-test/reporter-blob.spec.ts
View File

@ -361,7 +361,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'list'] });
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0);
const text = stripAnsi(output);

176
tests/playwright-test/reporter-dot.spec.ts
View File

@ -17,93 +17,99 @@
import colors from 'colors/safe';
import { test, expect } from './playwright-test-fixtures';
test('render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(0);
});
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('render unexpected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.red('F'));
expect(result.exitCode).toBe(1);
});
test('render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(0);
});
test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'dot' });
const text = result.output;
expect(text).toContain('×××F');
expect(result.rawOutput).toContain(colors.red('F'));
expect(result.exitCode).toBe(1);
});
test('render unexpected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.red('F'));
expect(result.exitCode).toBe(1);
});
test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'dot' });
const text = result.output;
expect(text).toContain('×××±');
expect(result.rawOutput).toContain(colors.yellow('±'));
expect(text).toContain('1 flaky');
expect(text).toContain('Retry #1');
expect(result.exitCode).toBe(0);
});
test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'dot' });
const text = result.output;
expect(text).toContain('×××F');
expect(result.rawOutput).toContain(colors.red('F'));
expect(result.exitCode).toBe(1);
});
test('should work from config', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { reporter: 'dot' };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(0);
});
test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'dot' });
const text = result.output;
expect(text).toContain('×××±');
expect(result.rawOutput).toContain(colors.yellow('±'));
expect(text).toContain('1 flaky');
expect(text).toContain('Retry #1');
expect(result.exitCode).toBe(0);
});
test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
for (let i = 0; i < 243; i++) {
test('test' + i, () => {});
}
`,
}, { reporter: 'dot' });
expect(result.exitCode).toBe(0);
expect(result.rawOutput).toContain(
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(3));
});
test('should work from config', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { reporter: 'dot' };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(0);
});
test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
for (let i = 0; i < 243; i++) {
test('test' + i, () => {});
}
`,
}, { reporter: 'dot' });
expect(result.exitCode).toBe(0);
expect(result.rawOutput).toContain(
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(3));
});
});
}

140
tests/playwright-test/reporter-github.spec.ts
View File

@ -23,73 +23,79 @@ function relativeFilePath(file: string): string {
return path.relative(process.cwd(), file);
}
test('print GitHub annotations for success', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('example1', async ({}) => {
expect(1 + 1).toBe(2);
});
`
}, { reporter: 'github' });
const text = result.output;
expect(text).not.toContain('::error');
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('example', async ({}) => {
expect(1 + 1).toBe(3);
});
`
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
const text = result.output;
const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
expect(result.exitCode).toBe(1);
});
test('print GitHub annotations for success', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('example1', async ({}) => {
expect(1 + 1).toBe(2);
});
`
}, { reporter: 'github' });
const text = result.output;
expect(text).not.toContain('::error');
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 100 }
};
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 200));
});
`
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' });
const text = result.output;
expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took');
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('example', async ({}) => {
expect(1 + 1).toBe(3);
});
`
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
const text = result.output;
const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
expect(result.exitCode).toBe(1);
});
test('print GitHub annotations for global error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
w: [async ({}, use) => {
await use();
throw new Error('Oh my!');
}, { scope: 'worker' }],
});
test('passes but...', ({w}) => {
});
`,
}, { reporter: 'github' });
const text = result.output;
expect(text).toContain('::error ::Error: Oh my!%0A%0A');
expect(result.exitCode).toBe(1);
});
test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 100 }
};
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 200));
});
`
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' });
const text = result.output;
expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took');
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for global error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
w: [async ({}, use) => {
await use();
throw new Error('Oh my!');
}, { scope: 'worker' }],
});
test('passes but...', ({w}) => {
});
`,
}, { reporter: 'github' });
const text = result.output;
expect(text).toContain('::error ::Error: Oh my!%0A%0A');
expect(result.exitCode).toBe(1);
});
});
}

844
tests/playwright-test/reporter-junit.spec.ts
View File

@ -19,424 +19,432 @@ import path from 'path';
import { test, expect } from './playwright-test-fixtures';
import fs from 'fs';
test('should render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
'b.test.js': `
import { test, expect } from '@playwright/test';
test('two', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
expect(result.exitCode).toBe(0);
});
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('should render unexpected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['$']['failures']).toBe('1');
const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];
expect(failure['$']['message']).toContain('a.test.js');
expect(failure['$']['message']).toContain('one');
expect(failure['$']['type']).toBe('FAILURE');
expect(failure['_']).toContain('expect(1).toBe(0)');
expect(result.exitCode).toBe(1);
});
test('should render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'junit' });
expect(result.output).toContain(`tests="1"`);
expect(result.output).toContain(`failures="1"`);
expect(result.output).toContain(`<failure`);
expect(result.output).toContain('Retry #1');
expect(result.output).toContain('Retry #2');
expect(result.output).toContain('Retry #3');
expect(result.exitCode).toBe(1);
});
test('should render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'junit' });
expect(result.output).not.toContain('Retry #1');
expect(result.exitCode).toBe(0);
});
test('should render stdout', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import colors from 'colors/safe';
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
console.log(colors.yellow('Hello world'));
console.log('Hello again');
console.error('My error');
console.error('\\0'); // null control character
test.expect("abc").toBe('abcd');
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again');
expect(testcase['system-out'][0]).not.toContain('u00');
expect(testcase['system-err'][0]).toContain('My error');
expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character
expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`);
expect(result.exitCode).toBe(1);
});
test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reporter: [ ['junit', { stripANSIControlSequences: true }] ],
};
`,
'a.test.ts': `
import colors from 'colors/safe';
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
console.log(colors.yellow('Hello world'));
});
`,
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe('Hello world');
expect(result.exitCode).toBe(0);
});
test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reporter: [ ['junit'] ],
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
process.stdout.write('Hello world &"\\'<>]]>');
});
`,
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]&gt;');
expect(result.output).toContain(`<system-out>\n<![CDATA[Hello world &"\'<>]]&gt;]]>\n</system-out>`);
expect(result.exitCode).toBe(0);
});
test('should render skipped', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async () => {
console.log('Hello world');
});
test('two', async () => {
test.skip();
console.log('Hello world');
});
`,
}, { retries: 3, reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
expect(result.exitCode).toBe(0);
});
test('should report skipped due to sharding', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async () => {
});
test('two', async () => {
test.skip();
});
`,
'b.test.js': `
import { test, expect } from '@playwright/test';
test('three', async () => {
});
test('four', async () => {
test.skip();
});
test('five', async () => {
});
`,
}, { shard: '1/3', reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
expect(result.exitCode).toBe(0);
});
test('should not render projects if they dont exist', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(result.exitCode).toBe(0);
});
test('should render projects', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2');
expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(result.exitCode).toBe(0);
});
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test.use({ screenshot: 'on' });
test('one', async ({ page }, testInfo) => {
await page.setContent('hello');
const file = testInfo.outputPath('file.txt');
require('fs').writeFileSync(file, 'my file', 'utf8');
testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' });
testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' });
console.log('log here');
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe([
`log here`,
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`,
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`,
].join('\n'));
expect(result.exitCode).toBe(0);
});
function parseXML(xml: string): any {
let result: any;
xml2js.parseString(xml, (err, r) => result = r);
return result;
}
test('should render annotations to custom testcase properties', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(1);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description');
expect(result.exitCode).toBe(0);
});
test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
test.slow();
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(1);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('');
expect(result.exitCode).toBe(0);
});
test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
const xrayOptions = {
embedAnnotationsAsProperties: true
}
module.exports = {
reporter: [ ['junit', xrayOptions] ],
};
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_id', description: '1234' });
testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(4);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234');
expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key');
expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2');
expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary');
expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary');
expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements');
expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6');
expect(result.exitCode).toBe(0);
});
test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
const file = testInfo.outputPath('evidence1.txt');
require('fs').writeFileSync(file, 'hello', 'utf8');
testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' });
testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' });
// await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
// await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
});
`
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).not.toBeTruthy();
expect(result.exitCode).toBe(0);
});
test.describe('report location', () => {
test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'nested/project/playwright.config.ts': `
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
`,
'nested/project/a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: '', config: './nested/project/playwright.config.ts' });
expect(result.exitCode).toBe(0);
expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy();
});
test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'foo/package.json': `{ "name": "foo" }`,
// unused config along "search path"
'foo/bar/playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'foo/bar/baz/tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, {
cwd: 'foo/bar/baz/tests',
test('should render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
'b.test.js': `
import { test, expect } from '@playwright/test';
test('two', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
expect(result.exitCode).toBe(0);
});
test('should render unexpected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['$']['failures']).toBe('1');
const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];
expect(failure['$']['message']).toContain('a.test.js');
expect(failure['$']['message']).toContain('one');
expect(failure['$']['type']).toBe('FAILURE');
expect(failure['_']).toContain('expect(1).toBe(0)');
expect(result.exitCode).toBe(1);
});
test('should render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'junit' });
expect(result.output).toContain(`tests="1"`);
expect(result.output).toContain(`failures="1"`);
expect(result.output).toContain(`<failure`);
expect(result.output).toContain('Retry #1');
expect(result.output).toContain('Retry #2');
expect(result.output).toContain('Retry #3');
expect(result.exitCode).toBe(1);
});
test('should render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'junit' });
expect(result.output).not.toContain('Retry #1');
expect(result.exitCode).toBe(0);
});
test('should render stdout', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import colors from 'colors/safe';
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
console.log(colors.yellow('Hello world'));
console.log('Hello again');
console.error('My error');
console.error('\\0'); // null control character
test.expect("abc").toBe('abcd');
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again');
expect(testcase['system-out'][0]).not.toContain('u00');
expect(testcase['system-err'][0]).toContain('My error');
expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character
expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`);
expect(result.exitCode).toBe(1);
});
test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reporter: [ ['junit', { stripANSIControlSequences: true }] ],
};
`,
'a.test.ts': `
import colors from 'colors/safe';
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
console.log(colors.yellow('Hello world'));
});
`,
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe('Hello world');
expect(result.exitCode).toBe(0);
});
test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reporter: [ ['junit'] ],
};
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
process.stdout.write('Hello world &"\\'<>]]>');
});
`,
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]&gt;');
expect(result.output).toContain(`<system-out>\n<![CDATA[Hello world &"\'<>]]&gt;]]>\n</system-out>`);
expect(result.exitCode).toBe(0);
});
test('should render skipped', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async () => {
console.log('Hello world');
});
test('two', async () => {
test.skip();
console.log('Hello world');
});
`,
}, { retries: 3, reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
expect(result.exitCode).toBe(0);
});
test('should report skipped due to sharding', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async () => {
});
test('two', async () => {
test.skip();
});
`,
'b.test.js': `
import { test, expect } from '@playwright/test';
test('three', async () => {
});
test('four', async () => {
test.skip();
});
test('five', async () => {
});
`,
}, { shard: '1/3', reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
expect(result.exitCode).toBe(0);
});
test('should not render projects if they dont exist', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(result.exitCode).toBe(0);
});
test('should render projects', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] };
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2');
expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(result.exitCode).toBe(0);
});
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
test.skip(useIntermediateMergeReport, 'Blob report hashes attachment paths');
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test.use({ screenshot: 'on' });
test('one', async ({ page }, testInfo) => {
await page.setContent('hello');
const file = testInfo.outputPath('file.txt');
require('fs').writeFileSync(file, 'my file', 'utf8');
testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' });
testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' });
console.log('log here');
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1);
expect(testcase['system-out'][0].trim()).toBe([
`log here`,
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`,
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`,
].join('\n'));
expect(result.exitCode).toBe(0);
});
function parseXML(xml: string): any {
let result: any;
xml2js.parseString(xml, (err, r) => result = r);
return result;
}
test('should render annotations to custom testcase properties', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(1);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description');
expect(result.exitCode).toBe(0);
});
test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
test.slow();
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(1);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('');
expect(result.exitCode).toBe(0);
});
test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
const xrayOptions = {
embedAnnotationsAsProperties: true
}
module.exports = {
reporter: [ ['junit', xrayOptions] ],
};
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_id', description: '1234' });
testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
});
`
}, { reporter: '' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).toBeTruthy();
expect(testcase['properties'][0]['property'].length).toBe(4);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234');
expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key');
expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2');
expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary');
expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary');
expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements');
expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6');
expect(result.exitCode).toBe(0);
});
test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
const file = testInfo.outputPath('evidence1.txt');
require('fs').writeFileSync(file, 'hello', 'utf8');
testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' });
testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' });
// await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
// await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
});
`
}, { reporter: 'junit' });
const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties']).not.toBeTruthy();
expect(result.exitCode).toBe(0);
});
test.describe('report location', () => {
test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
test.skip(useIntermediateMergeReport);
const result = await runInlineTest({
'nested/project/playwright.config.ts': `
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
`,
'nested/project/a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: '', config: './nested/project/playwright.config.ts' });
expect(result.exitCode).toBe(0);
expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy();
});
test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'foo/package.json': `{ "name": "foo" }`,
// unused config along "search path"
'foo/bar/playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'foo/bar/baz/tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, {
cwd: 'foo/bar/baz/tests',
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
});
}

288
tests/playwright-test/reporter-line.spec.ts
View File

@ -16,149 +16,155 @@
import { test, expect } from './playwright-test-fixtures';
test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('[2/1] (retries) a.test.js:3:7 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:3:7 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:3:7 › one (retry #3)');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ────');
expect(text).toContain('Retry #2 ────');
expect(text).toContain('Retry #3 ────');
expect(result.exitCode).toBe(1);
});
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('1 flaky');
expect(result.exitCode).toBe(0);
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should work on CI', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'line' }, { CI: '1' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
});
test('should print output', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
process.stdout.write('one');
process.stdout.write('two');
console.log('full-line');
});
`
}, { reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.output).toContain([
'a.spec.ts:3:11 › foobar',
'one',
'',
'two',
'',
'full-line',
].join('\n'));
});
test('should render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:11 › one');
expect(text).toContain('[2/1] (retries) a.test.js:3:11 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:3:11 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:3:11 › one (retry #3)');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ────');
expect(text).toContain('Retry #2 ────');
expect(text).toContain('Retry #3 ────');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
expect(testInfo.retry).toBe(3);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('1 flaky');
expect(result.exitCode).toBe(0);
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should work on CI', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'line' }, { CI: '1' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:11 › one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
});
test('should print output', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
process.stdout.write('one');
process.stdout.write('two');
console.log('full-line');
});
`
}, { reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.output).toContain([
'a.spec.ts:3:15 › foobar',
'one',
'',
'two',
'',
'full-line',
].join('\n'));
});
test('should render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
await test.step('inner 1.2', async () => {
expect.soft(1).toBe(2);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
});
expect.soft(1).toBe(2);
});
await test.step('inner 1.2', async () => {
expect.soft(1).toBe(2);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
});
expect.soft(1).toBe(2);
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes ──');
expect(result.exitCode).toBe(1);
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes ──');
expect(result.exitCode).toBe(1);
});
});
}

411
tests/playwright-test/reporter-list.spec.ts
View File

@ -20,212 +20,217 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces
const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
test('render each test with project name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
{ name: 'bar' },
] };
`,
'a.test.ts': `
const { test, expect } = require('@playwright/test');
test('fails', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
expect(0).toBe(0);
});
test.skip('skipped', async () => {
});
`,
}, { reporter: 'list', workers: '1' });
const text = result.output;
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:7 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:7 › passes`);
expect(text).toContain(`- 3 [foo] › a.test.ts:9:12 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:7 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:7 › passes`);
expect(text).toContain(`- 6 [bar] › a.test.ts:9:12 › skipped`);
expect(result.exitCode).toBe(1);
});
test('render steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)',
]);
});
test('render steps inline', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:3:11 › passes',
]);
});
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
const TTY_WIDTH = 80;
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
console.log('a'.repeat(80) + 'b'.repeat(20));
});
`,
}, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' });
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
if (process.platform === 'win32')
expect(renderedText).toContain(' ok 1 a.test.ts:3:11 › passes');
else
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:11 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:11 › passes');
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
});
test('render retries', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('flaky', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:3:11 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:11 › flaky (XXms)`,
`1 : 2 a.test.ts:3:11 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:11 › flaky (retry #1) (XXms)`,
]);
});
test('should truncate long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
] };
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('failure in very long name', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
});
test('passes 2 long name', async () => {
});
test.skip('skipped very long name', async () => {
});
`,
}, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' });
expect(result.exitCode).toBe(1);
const lines = result.output.split('\n').slice(3, 11);
expect(lines.every(line => line.length <= 50)).toBe(true);
expect(lines[0]).toBe(` 1 …a.test.ts:3:11 › failure in very long name`);
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
expect(lines[1]).toContain(`:3:11 › failure in very long name (`);
expect(lines[1].length).toBe(50);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:11 › passes`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes (`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:11 › passes 2 long name`);
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
expect(lines[5]).toContain(`test.ts:8:11 › passes 2 long name (`);
expect(lines[5].length).toBe(50);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:12 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`);
});
test('render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
test('render each test with project name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
{ name: 'bar' },
] };
`,
'a.test.ts': `
const { test, expect } = require('@playwright/test');
test('fails', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
expect(0).toBe(0);
});
test.skip('skipped', async () => {
});
`,
}, { reporter: 'list', workers: '1' });
const text = result.output;
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 3 [foo] › a.test.ts:9:16 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 6 [bar] › a.test.ts:9:16 › skipped`);
expect(result.exitCode).toBe(1);
});
test('render steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:15 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)',
]);
});
test('render steps inline', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
});
`,
}, { reporter: 'list' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:3:11 › passes',
]);
});
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
const TTY_WIDTH = 80;
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
console.log('a'.repeat(80) + 'b'.repeat(20));
});
`,
}, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' });
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
if (process.platform === 'win32')
expect(renderedText).toContain(' ok 1 a.test.ts:3:15 › passes');
else
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:15 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:15 › passes');
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
});
test('render retries', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('flaky', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:3:15 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
]);
});
test('should truncate long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
] };
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('failure in very long name', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
});
test('passes 2 long name', async () => {
});
test.skip('skipped very long name', async () => {
});
`,
}, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' });
expect(result.exitCode).toBe(1);
const lines = result.output.split('\n').slice(3, 11);
expect(lines.every(line => line.length <= 50)).toBe(true);
expect(lines[0]).toBe(` 1 …a.test.ts:3:15 › failure in very long name`);
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
expect(lines[1]).toContain(`:3:15 › failure in very long name (`);
expect(lines[1].length).toBe(50);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:15 › passes`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:15 › passes (`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:15 › passes 2 long name`);
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
expect(lines[5]).toContain(`test.ts:8:15 › passes 2 long name (`);
expect(lines[5].length).toBe(50);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:16 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:16 › skipped very long name`);
});
test('render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
});
});
});
`,
}, { reporter: 'list' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
});
}
function simpleAnsiRenderer(text, ttyWidth) {
let lineNumber = 0;

1307
tests/playwright-test/reporter.spec.ts
View File

File diff suppressed because it is too large Load Diff