mirror of
https://github.com/microsoft/playwright.git
synced 2025-01-07 11:46:42 +03:00
chore: run more reporter tests through blob report, some fixes (#23765)
This commit is contained in:
Yury Semikhatsky
GitHub
parent
09b1e3ffa9
commit
f6d86c20f3
@ -232,6 +232,7 @@ export class TeleReporterReceiver {
|
||||
result.status = payload.status;
|
||||
result.statusEx = payload.status;
|
||||
result.errors = payload.errors;
|
||||
result.error = result.errors?.[0];
|
||||
result.attachments = this._parseAttachments(payload.attachments);
|
||||
this._reporter.onTestEnd?.(test, result);
|
||||
}
|
||||
@ -242,7 +243,10 @@ export class TeleReporterReceiver {
|
||||
const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined;
|
||||
|
||||
const step: TestStep = {
|
||||
titlePath: () => [],
|
||||
titlePath: () => {
|
||||
const parentPath = parentStep?.titlePath() || [];
|
||||
return [...parentPath, payload.title];
|
||||
},
|
||||
title: payload.title,
|
||||
category: payload.category,
|
||||
location: this._absoluteLocation(payload.location),
|
||||
|
||||
@ -92,7 +92,7 @@ export class BlobReporter extends TeleReporterEmitter {
|
||||
|
||||
override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] {
|
||||
return super._serializeAttachments(attachments).map(attachment => {
|
||||
if (!attachment.path || !fs.statSync(attachment.path).isFile())
|
||||
if (!attachment.path || !fs.statSync(attachment.path, { throwIfNoEntry: false })?.isFile())
|
||||
return attachment;
|
||||
// Add run guid to avoid clashes between shards.
|
||||
const sha1 = calculateSha1(attachment.path + this._salt);
|
||||
|
||||
@ -88,7 +88,7 @@ export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-tes
|
||||
const configFile = (baseDir: string, files: Files): string | undefined => {
|
||||
for (const [name, content] of Object.entries(files)) {
|
||||
if (name.includes('playwright.config')) {
|
||||
if (content.includes('reporter:'))
|
||||
if (content.includes('reporter:') || content.includes('reportSlowTests:'))
|
||||
return path.resolve(baseDir, name);
|
||||
}
|
||||
}
|
||||
@ -349,12 +349,7 @@ export const test = base
|
||||
const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir;
|
||||
const testProcess = childProcess({
|
||||
command,
|
||||
env: cleanEnv({
|
||||
PW_TEST_DEBUG_REPORTERS: '1',
|
||||
PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1',
|
||||
PWTEST_TTY_WIDTH: '80',
|
||||
...env
|
||||
}),
|
||||
env: cleanEnv(env),
|
||||
cwd,
|
||||
});
|
||||
const { exitCode } = await testProcess.exited;
|
||||
|
||||
@ -17,7 +17,11 @@
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
import * as path from 'path';
|
||||
|
||||
test('handle long test names', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('handle long test names', async ({ runInlineTest }) => {
|
||||
const title = 'title'.repeat(30);
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
@ -29,9 +33,9 @@ test('handle long test names', async ({ runInlineTest }) => {
|
||||
});
|
||||
expect(result.output).toContain('expect(1).toBe');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('print the error name', async ({ runInlineTest }) => {
|
||||
test('print the error name', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -45,9 +49,9 @@ test('print the error name', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('FooBarError: my-message');
|
||||
});
|
||||
});
|
||||
|
||||
test('print should print the error name without a message', async ({ runInlineTest }) => {
|
||||
test('print should print the error name without a message', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -61,9 +65,9 @@ test('print should print the error name without a message', async ({ runInlineTe
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('FooBarError');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print an error in a codeframe', async ({ runInlineTest }) => {
|
||||
test('should print an error in a codeframe', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -83,9 +87,9 @@ test('should print an error in a codeframe', async ({ runInlineTest }) => {
|
||||
expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`);
|
||||
expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`);
|
||||
expect(result.output).toContain(`> 4 | const error = new Error('my-message');`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
|
||||
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'node_modules/utils/utils.js': `
|
||||
function assert(value) {
|
||||
@ -106,15 +110,15 @@ test('should filter out node_modules error in a codeframe', async ({ runInlineTe
|
||||
expect(result.failed).toBe(1);
|
||||
const output = result.output;
|
||||
expect(output).toContain('Error: Assertion error');
|
||||
expect(output).toContain('a.spec.ts:4:11 › fail');
|
||||
expect(output).toContain('a.spec.ts:4:15 › fail');
|
||||
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
|
||||
expect(output).toContain(`> 5 | assert(false);`);
|
||||
expect(output).toContain(` | ^`);
|
||||
expect(output).toContain(`utils.js:4`);
|
||||
expect(output).toContain(`a.spec.ts:5:9`);
|
||||
});
|
||||
expect(output).toContain(`a.spec.ts:5:13`);
|
||||
});
|
||||
|
||||
test('should print codeframe from a helper', async ({ runInlineTest }) => {
|
||||
test('should print codeframe from a helper', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'helper.ts': `
|
||||
export function ohMy() {
|
||||
@ -137,9 +141,9 @@ test('should print codeframe from a helper', async ({ runInlineTest }) => {
|
||||
expect(result.output).toContain(` 2 | export function ohMy() {`);
|
||||
expect(result.output).toContain(` > 3 | throw new Error('oh my');`);
|
||||
expect(result.output).toContain(` | ^`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should print slow tests', async ({ runInlineTest }) => {
|
||||
test('should print slow tests', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -176,9 +180,9 @@ test('should print slow tests', async ({ runInlineTest }) => {
|
||||
expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`);
|
||||
expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`);
|
||||
expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not print slow parallel tests', async ({ runInlineTest }) => {
|
||||
test('should not print slow parallel tests', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -200,9 +204,9 @@ test('should not print slow parallel tests', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.output).not.toContain('Slow test file');
|
||||
});
|
||||
});
|
||||
|
||||
test('should not print slow tests', async ({ runInlineTest }) => {
|
||||
test('should not print slow tests', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -226,9 +230,9 @@ test('should not print slow tests', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.output).not.toContain('Slow test');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -240,9 +244,9 @@ test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.flaky).toBe(1);
|
||||
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print flaky timeouts', async ({ runInlineTest }) => {
|
||||
test('should print flaky timeouts', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -255,9 +259,9 @@ test('should print flaky timeouts', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.flaky).toBe(1);
|
||||
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print stack-less errors', async ({ runInlineTest }) => {
|
||||
test('should print stack-less errors', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -271,9 +275,9 @@ test('should print stack-less errors', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('Hello');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
|
||||
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -292,15 +296,15 @@ test('should print errors with inconsistent message/stack', async ({ runInlineTe
|
||||
const output = result.output;
|
||||
expect(output).toContain('foo bar');
|
||||
expect(output).toContain('function myTest');
|
||||
});
|
||||
});
|
||||
|
||||
test('should print "no tests found" error', async ({ runInlineTest }) => {
|
||||
test('should print "no tests found" error', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({ });
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain('No tests found');
|
||||
});
|
||||
});
|
||||
|
||||
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
|
||||
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -317,9 +321,9 @@ test('should not crash on undefined body with manual attachments', async ({ runI
|
||||
expect(result.output).not.toContain('Error in reporter');
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report fatal errors at the end', async ({ runInlineTest }) => {
|
||||
test('should report fatal errors at the end', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test as base, expect } from '@playwright/test';
|
||||
@ -347,9 +351,9 @@ test('should report fatal errors at the end', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.output).toContain('2 errors were not a part of any test, see above for details');
|
||||
});
|
||||
});
|
||||
|
||||
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
|
||||
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -359,4 +363,6 @@ test('should contain at most 1 decimal for humanized timing', async ({ runInline
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -361,7 +361,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
|
||||
const reportFiles = await fs.promises.readdir(reportDir);
|
||||
reportFiles.sort();
|
||||
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
|
||||
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'list'] });
|
||||
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
|
||||
expect(exitCode).toBe(0);
|
||||
|
||||
const text = stripAnsi(output);
|
||||
|
||||
@ -17,7 +17,11 @@
|
||||
import colors from 'colors/safe';
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
|
||||
test('render expected', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('render expected', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -28,9 +32,9 @@ test('render expected', async ({ runInlineTest }) => {
|
||||
}, { reporter: 'dot' });
|
||||
expect(result.rawOutput).toContain(colors.green('·'));
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('render unexpected', async ({ runInlineTest }) => {
|
||||
test('render unexpected', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -41,9 +45,9 @@ test('render unexpected', async ({ runInlineTest }) => {
|
||||
}, { reporter: 'dot' });
|
||||
expect(result.rawOutput).toContain(colors.red('F'));
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -56,9 +60,9 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
expect(text).toContain('×××F');
|
||||
expect(result.rawOutput).toContain(colors.red('F'));
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('render flaky', async ({ runInlineTest }) => {
|
||||
test('render flaky', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -73,9 +77,9 @@ test('render flaky', async ({ runInlineTest }) => {
|
||||
expect(text).toContain('1 flaky');
|
||||
expect(text).toContain('Retry #1');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should work from config', async ({ runInlineTest }) => {
|
||||
test('should work from config', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = { reporter: 'dot' };
|
||||
@ -89,9 +93,9 @@ test('should work from config', async ({ runInlineTest }) => {
|
||||
}, { reporter: 'dot' });
|
||||
expect(result.rawOutput).toContain(colors.green('·'));
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
|
||||
test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -106,4 +110,6 @@ test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
|
||||
colors.green('·').repeat(80) + '\n' +
|
||||
colors.green('·').repeat(80) + '\n' +
|
||||
colors.green('·').repeat(3));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -23,7 +23,11 @@ function relativeFilePath(file: string): string {
|
||||
return path.relative(process.cwd(), file);
|
||||
}
|
||||
|
||||
test('print GitHub annotations for success', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('print GitHub annotations for success', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -36,9 +40,9 @@ test('print GitHub annotations for success', async ({ runInlineTest }) => {
|
||||
expect(text).not.toContain('::error');
|
||||
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
|
||||
test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test, expect } = require('@playwright/test');
|
||||
@ -49,13 +53,13 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }, test
|
||||
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
|
||||
const text = result.output;
|
||||
const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
|
||||
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
|
||||
test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -73,9 +77,9 @@ test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
|
||||
expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took');
|
||||
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('print GitHub annotations for global error', async ({ runInlineTest }) => {
|
||||
test('print GitHub annotations for global error', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test as base, expect } from '@playwright/test';
|
||||
@ -92,4 +96,6 @@ test('print GitHub annotations for global error', async ({ runInlineTest }) => {
|
||||
const text = result.output;
|
||||
expect(text).toContain('::error ::Error: Oh my!%0A%0A');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -19,7 +19,11 @@ import path from 'path';
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
import fs from 'fs';
|
||||
|
||||
test('should render expected', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('should render expected', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -44,9 +48,9 @@ test('should render expected', async ({ runInlineTest }) => {
|
||||
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
|
||||
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render unexpected', async ({ runInlineTest }) => {
|
||||
test('should render unexpected', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -64,9 +68,9 @@ test('should render unexpected', async ({ runInlineTest }) => {
|
||||
expect(failure['$']['type']).toBe('FAILURE');
|
||||
expect(failure['_']).toContain('expect(1).toBe(0)');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render unexpected after retry', async ({ runInlineTest }) => {
|
||||
test('should render unexpected after retry', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -82,9 +86,9 @@ test('should render unexpected after retry', async ({ runInlineTest }) => {
|
||||
expect(result.output).toContain('Retry #2');
|
||||
expect(result.output).toContain('Retry #3');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render flaky', async ({ runInlineTest }) => {
|
||||
test('should render flaky', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -95,9 +99,9 @@ test('should render flaky', async ({ runInlineTest }) => {
|
||||
}, { retries: 3, reporter: 'junit' });
|
||||
expect(result.output).not.toContain('Retry #1');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render stdout', async ({ runInlineTest }) => {
|
||||
test('should render stdout', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import colors from 'colors/safe';
|
||||
@ -120,9 +124,9 @@ test('should render stdout', async ({ runInlineTest }) => {
|
||||
expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character
|
||||
expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`);
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
|
||||
test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -142,9 +146,9 @@ test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
|
||||
expect(testcase['system-out'].length).toBe(1);
|
||||
expect(testcase['system-out'][0].trim()).toBe('Hello world');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => {
|
||||
test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
@ -164,9 +168,9 @@ test('should render, by default, character data as CDATA sections', async ({ run
|
||||
expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]>');
|
||||
expect(result.output).toContain(`<system-out>\n<![CDATA[Hello world &"\'<>]]>]]>\n</system-out>`);
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render skipped', async ({ runInlineTest }) => {
|
||||
test('should render skipped', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -184,9 +188,9 @@ test('should render skipped', async ({ runInlineTest }) => {
|
||||
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
|
||||
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report skipped due to sharding', async ({ runInlineTest }) => {
|
||||
test('should report skipped due to sharding', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -213,9 +217,9 @@ test('should report skipped due to sharding', async ({ runInlineTest }) => {
|
||||
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
|
||||
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not render projects if they dont exist', async ({ runInlineTest }) => {
|
||||
test('should not render projects if they dont exist', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = { };
|
||||
@ -239,9 +243,9 @@ test('should not render projects if they dont exist', async ({ runInlineTest })
|
||||
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
|
||||
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render projects', async ({ runInlineTest }) => {
|
||||
test('should render projects', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] };
|
||||
@ -274,9 +278,10 @@ test('should render projects', async ({ runInlineTest }) => {
|
||||
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one');
|
||||
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
|
||||
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
|
||||
test.skip(useIntermediateMergeReport, 'Blob report hashes attachment paths');
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -300,15 +305,15 @@ test('should render existing attachments, but not missing ones', async ({ runInl
|
||||
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`,
|
||||
].join('\n'));
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
function parseXML(xml: string): any {
|
||||
function parseXML(xml: string): any {
|
||||
let result: any;
|
||||
xml2js.parseString(xml, (err, r) => result = r);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
test('should render annotations to custom testcase properties', async ({ runInlineTest }) => {
|
||||
test('should render annotations to custom testcase properties', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
|
||||
'a.test.js': `
|
||||
@ -325,9 +330,9 @@ test('should render annotations to custom testcase properties', async ({ runInli
|
||||
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description');
|
||||
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => {
|
||||
test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
|
||||
'a.test.js': `
|
||||
@ -344,9 +349,9 @@ test('should render built-in annotations to testcase properties', async ({ runIn
|
||||
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow');
|
||||
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
|
||||
test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
const xrayOptions = {
|
||||
@ -379,9 +384,9 @@ test('should render all annotations to testcase value based properties, if reque
|
||||
expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements');
|
||||
expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => {
|
||||
test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -399,11 +404,12 @@ test('should not embed attachments to a custom testcase property, if not explici
|
||||
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
|
||||
expect(testcase['properties']).not.toBeTruthy();
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
test.describe('report location', () => {
|
||||
test.describe('report location', () => {
|
||||
test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
|
||||
test.skip(useIntermediateMergeReport);
|
||||
const result = await runInlineTest({
|
||||
'nested/project/playwright.config.ts': `
|
||||
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
|
||||
@ -439,4 +445,6 @@ test.describe('report location', () => {
|
||||
expect(result.passed).toBe(1);
|
||||
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -16,7 +16,11 @@
|
||||
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
|
||||
test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test, expect } = require('@playwright/test');
|
||||
@ -26,10 +30,10 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
`,
|
||||
}, { retries: 3, reporter: 'line' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('[1/1] a.test.js:3:7 › one');
|
||||
expect(text).toContain('[2/1] (retries) a.test.js:3:7 › one (retry #1)');
|
||||
expect(text).toContain('[3/1] (retries) a.test.js:3:7 › one (retry #2)');
|
||||
expect(text).toContain('[4/1] (retries) a.test.js:3:7 › one (retry #3)');
|
||||
expect(text).toContain('[1/1] a.test.js:3:11 › one');
|
||||
expect(text).toContain('[2/1] (retries) a.test.js:3:11 › one (retry #1)');
|
||||
expect(text).toContain('[3/1] (retries) a.test.js:3:11 › one (retry #2)');
|
||||
expect(text).toContain('[4/1] (retries) a.test.js:3:11 › one (retry #3)');
|
||||
expect(text).toContain('1 failed');
|
||||
expect(text).toContain('1) a.test');
|
||||
expect(text).not.toContain('2) a.test');
|
||||
@ -37,9 +41,9 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
|
||||
expect(text).toContain('Retry #2 ────');
|
||||
expect(text).toContain('Retry #3 ────');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('render flaky', async ({ runInlineTest }) => {
|
||||
test('render flaky', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -51,9 +55,9 @@ test('render flaky', async ({ runInlineTest }) => {
|
||||
const text = result.output;
|
||||
expect(text).toContain('1 flaky');
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -65,9 +69,9 @@ test('should print flaky failures', async ({ runInlineTest }) => {
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.flaky).toBe(1);
|
||||
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
|
||||
});
|
||||
});
|
||||
|
||||
test('should work on CI', async ({ runInlineTest }) => {
|
||||
test('should work on CI', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.js': `
|
||||
const { test, expect } = require('@playwright/test');
|
||||
@ -77,13 +81,13 @@ test('should work on CI', async ({ runInlineTest }) => {
|
||||
`,
|
||||
}, { reporter: 'line' }, { CI: '1' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('[1/1] a.test.js:3:7 › one');
|
||||
expect(text).toContain('[1/1] a.test.js:3:11 › one');
|
||||
expect(text).toContain('1 failed');
|
||||
expect(text).toContain('1) a.test');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should print output', async ({ runInlineTest }) => {
|
||||
test('should print output', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -96,16 +100,16 @@ test('should print output', async ({ runInlineTest }) => {
|
||||
}, { reporter: 'line' });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.output).toContain([
|
||||
'a.spec.ts:3:11 › foobar',
|
||||
'a.spec.ts:3:15 › foobar',
|
||||
'one',
|
||||
'',
|
||||
'two',
|
||||
'',
|
||||
'full-line',
|
||||
].join('\n'));
|
||||
});
|
||||
});
|
||||
|
||||
test('should render failed test steps', async ({ runInlineTest }) => {
|
||||
test('should render failed test steps', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -119,11 +123,11 @@ test('should render failed test steps', async ({ runInlineTest }) => {
|
||||
`,
|
||||
}, { reporter: 'line' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
|
||||
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => {
|
||||
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -140,11 +144,11 @@ test('should not render more than one failed test steps in header', async ({ run
|
||||
`,
|
||||
}, { reporter: 'line' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──');
|
||||
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 ──');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
|
||||
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -159,6 +163,8 @@ test('should not render more than one failed test steps in header (2)', async ({
|
||||
`,
|
||||
}, { reporter: 'line' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('1) a.test.ts:3:11 › passes ──');
|
||||
expect(text).toContain('1) a.test.ts:3:15 › passes ──');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -20,7 +20,11 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces
|
||||
const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
|
||||
const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
|
||||
|
||||
test('render each test with project name', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('render each test with project name', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = { projects: [
|
||||
@ -42,16 +46,16 @@ test('render each test with project name', async ({ runInlineTest }) => {
|
||||
}, { reporter: 'list', workers: '1' });
|
||||
const text = result.output;
|
||||
|
||||
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:7 › fails`);
|
||||
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:7 › passes`);
|
||||
expect(text).toContain(`- 3 [foo] › a.test.ts:9:12 › skipped`);
|
||||
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:7 › fails`);
|
||||
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:7 › passes`);
|
||||
expect(text).toContain(`- 6 [bar] › a.test.ts:9:12 › skipped`);
|
||||
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:11 › fails`);
|
||||
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes`);
|
||||
expect(text).toContain(`- 3 [foo] › a.test.ts:9:16 › skipped`);
|
||||
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:11 › fails`);
|
||||
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:11 › passes`);
|
||||
expect(text).toContain(`- 6 [bar] › a.test.ts:9:16 › skipped`);
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('render steps', async ({ runInlineTest }) => {
|
||||
test('render steps', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -71,7 +75,7 @@ test('render steps', async ({ runInlineTest }) => {
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
|
||||
lines.pop(); // Remove last item that contains [v] and time in ms.
|
||||
expect(lines).toEqual([
|
||||
'0 : 1 a.test.ts:3:11 › passes',
|
||||
'0 : 1 a.test.ts:3:15 › passes',
|
||||
'1 : 1.1 passes › outer 1.0',
|
||||
'2 : 1.2 passes › outer 1.0 › inner 1.1',
|
||||
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
|
||||
@ -85,9 +89,9 @@ test('render steps', async ({ runInlineTest }) => {
|
||||
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
|
||||
'4 : 1.4 passes › outer 2.0 (Xms)',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('render steps inline', async ({ runInlineTest }) => {
|
||||
test('render steps inline', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -100,8 +104,7 @@ test('render steps inline', async ({ runInlineTest }) => {
|
||||
await test.step('inner 2.1', async () => {});
|
||||
await test.step('inner 2.2', async () => {});
|
||||
});
|
||||
});
|
||||
`,
|
||||
});`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
|
||||
const text = result.output;
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
|
||||
@ -121,9 +124,9 @@ test('render steps inline', async ({ runInlineTest }) => {
|
||||
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'0 : 1 a.test.ts:3:11 › passes',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
|
||||
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
|
||||
const TTY_WIDTH = 80;
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
@ -136,14 +139,14 @@ test('very long console line should not mess terminal', async ({ runInlineTest }
|
||||
|
||||
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
|
||||
if (process.platform === 'win32')
|
||||
expect(renderedText).toContain(' ok 1 a.test.ts:3:11 › passes');
|
||||
expect(renderedText).toContain(' ok 1 a.test.ts:3:15 › passes');
|
||||
else
|
||||
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:11 › passes');
|
||||
expect(renderedText).not.toContain(' 1 a.test.ts:3:11 › passes');
|
||||
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:15 › passes');
|
||||
expect(renderedText).not.toContain(' 1 a.test.ts:3:15 › passes');
|
||||
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
|
||||
});
|
||||
});
|
||||
|
||||
test('render retries', async ({ runInlineTest }) => {
|
||||
test('render retries', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -156,14 +159,14 @@ test('render retries', async ({ runInlineTest }) => {
|
||||
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
|
||||
|
||||
expect(lines).toEqual([
|
||||
`0 : 1 a.test.ts:3:11 › flaky`,
|
||||
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:11 › flaky (XXms)`,
|
||||
`1 : 2 a.test.ts:3:11 › flaky (retry #1)`,
|
||||
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:11 › flaky (retry #1) (XXms)`,
|
||||
`0 : 1 a.test.ts:3:15 › flaky`,
|
||||
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
|
||||
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
|
||||
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should truncate long test names', async ({ runInlineTest }) => {
|
||||
test('should truncate long test names', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
module.exports = { projects: [
|
||||
@ -188,28 +191,28 @@ test('should truncate long test names', async ({ runInlineTest }) => {
|
||||
const lines = result.output.split('\n').slice(3, 11);
|
||||
expect(lines.every(line => line.length <= 50)).toBe(true);
|
||||
|
||||
expect(lines[0]).toBe(` 1 …a.test.ts:3:11 › failure in very long name`);
|
||||
expect(lines[0]).toBe(` 1 …a.test.ts:3:15 › failure in very long name`);
|
||||
|
||||
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
|
||||
expect(lines[1]).toContain(`:3:11 › failure in very long name (`);
|
||||
expect(lines[1]).toContain(`:3:15 › failure in very long name (`);
|
||||
expect(lines[1].length).toBe(50);
|
||||
|
||||
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:11 › passes`);
|
||||
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:15 › passes`);
|
||||
|
||||
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes (`);
|
||||
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:15 › passes (`);
|
||||
|
||||
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:11 › passes 2 long name`);
|
||||
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:15 › passes 2 long name`);
|
||||
|
||||
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
|
||||
expect(lines[5]).toContain(`test.ts:8:11 › passes 2 long name (`);
|
||||
expect(lines[5]).toContain(`test.ts:8:15 › passes 2 long name (`);
|
||||
expect(lines[5].length).toBe(50);
|
||||
|
||||
expect(lines[6]).toBe(` 4 …› a.test.ts:10:12 › skipped very long name`);
|
||||
expect(lines[6]).toBe(` 4 …› a.test.ts:10:16 › skipped very long name`);
|
||||
|
||||
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`);
|
||||
});
|
||||
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:16 › skipped very long name`);
|
||||
});
|
||||
|
||||
test('render failed test steps', async ({ runInlineTest }) => {
|
||||
test('render failed test steps', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -223,9 +226,11 @@ test('render failed test steps', async ({ runInlineTest }) => {
|
||||
`,
|
||||
}, { reporter: 'list' });
|
||||
const text = result.output;
|
||||
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
|
||||
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
|
||||
expect(result.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function simpleAnsiRenderer(text, ttyWidth) {
|
||||
let lineNumber = 0;
|
||||
|
||||
@ -74,7 +74,11 @@ class Reporter {
|
||||
module.exports = Reporter;
|
||||
`;
|
||||
|
||||
test('should work with custom reporter', async ({ runInlineTest }) => {
|
||||
for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
|
||||
test.use({ useIntermediateMergeReport });
|
||||
|
||||
test('should work with custom reporter', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': `
|
||||
class Reporter {
|
||||
@ -160,9 +164,9 @@ test('should work with custom reporter', async ({ runInlineTest }) => {
|
||||
'reporter-testend-is run-bar%%',
|
||||
'reporter-end-end%%',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should work without a file extension', async ({ runInlineTest }) => {
|
||||
test('should work without a file extension', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -183,9 +187,10 @@ test('should work without a file extension', async ({ runInlineTest }) => {
|
||||
'end',
|
||||
'exit',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report onEnd after global teardown', async ({ runInlineTest }) => {
|
||||
test('should report onEnd after global teardown', async ({ runInlineTest }) => {
|
||||
test.skip(useIntermediateMergeReport);
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'globalSetup.ts': `
|
||||
@ -213,9 +218,9 @@ test('should report onEnd after global teardown', async ({ runInlineTest }) => {
|
||||
'end',
|
||||
'exit',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should load reporter from node_modules', async ({ runInlineTest }) => {
|
||||
test('should load reporter from node_modules', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'node_modules/my-reporter/index.js': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -236,9 +241,9 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
|
||||
'end',
|
||||
'exit',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report expect steps', async ({ runInlineTest }) => {
|
||||
test('should report expect steps', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': stepsReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -300,9 +305,9 @@ test('should report expect steps', async ({ runInlineTest }) => {
|
||||
`end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`,
|
||||
`end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report api steps', async ({ runInlineTest }) => {
|
||||
test('should report api steps', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': stepsReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -409,10 +414,10 @@ test('should report api steps', async ({ runInlineTest }) => {
|
||||
`end {\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`,
|
||||
`end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}]}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
test('should report api step failure', async ({ runInlineTest }) => {
|
||||
test('should report api step failure', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': stepsReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -458,9 +463,9 @@ test('should report api step failure', async ({ runInlineTest }) => {
|
||||
`end {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`,
|
||||
`end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"},{\"title\":\"fixture: browser\",\"category\":\"fixture\"}]}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => {
|
||||
test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test as base, expect } from '@playwright/test';
|
||||
@ -481,9 +486,9 @@ test('should not have internal error when steps are finished after timeout', asy
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).not.toContain('Internal error');
|
||||
});
|
||||
});
|
||||
|
||||
test('should show nice stacks for locators', async ({ runInlineTest }) => {
|
||||
test('should show nice stacks for locators', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': stepsReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -530,9 +535,9 @@ test('should show nice stacks for locators', async ({ runInlineTest }) => {
|
||||
`end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`,
|
||||
`end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report forbid-only error to reporter', async ({ runInlineTest }) => {
|
||||
test('should report forbid-only error to reporter', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -548,9 +553,9 @@ test('should report forbid-only error to reporter', async ({ runInlineTest }) =>
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain(`%%got error: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: \"a.test.ts pass\"`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report no-tests error to reporter', async ({ runInlineTest }) => {
|
||||
test('should report no-tests error to reporter', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -562,9 +567,9 @@ test('should report no-tests error to reporter', async ({ runInlineTest }) => {
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain(`%%got error: No tests found`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report require error to reporter', async ({ runInlineTest }) => {
|
||||
test('should report require error to reporter', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -579,9 +584,9 @@ test('should report require error to reporter', async ({ runInlineTest }) => {
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain(`%%got error: Oh my!`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report global setup error to reporter', async ({ runInlineTest }) => {
|
||||
test('should report global setup error to reporter', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': smallReporterJS,
|
||||
'playwright.config.ts': `
|
||||
@ -603,9 +608,9 @@ test('should report global setup error to reporter', async ({ runInlineTest }) =
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain(`%%got error: Oh my!`);
|
||||
});
|
||||
});
|
||||
|
||||
test('should report correct tests/suites when using grep', async ({ runInlineTest }) => {
|
||||
test('should report correct tests/suites when using grep', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.spec.js': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
@ -633,9 +638,9 @@ test('should report correct tests/suites when using grep', async ({ runInlineTes
|
||||
expect(fileSuite.suites!.length).toBe(1);
|
||||
expect(fileSuite.suites![0].specs.length).toBe(2);
|
||||
expect(fileSuite.specs.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test('should use sourceMap-based file suite names', async ({ runInlineTest }) => {
|
||||
test('should use sourceMap-based file suite names', async ({ runInlineTest }) => {
|
||||
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11028' });
|
||||
const result = await runInlineTest({
|
||||
'reporter.js': `
|
||||
@ -678,9 +683,9 @@ var import_test = __toModule(require("@playwright/test"));
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.output).toContain('a.spec.ts');
|
||||
});
|
||||
});
|
||||
|
||||
test('parallelIndex is presented in onTestEnd', async ({ runInlineTest }) => {
|
||||
test('parallelIndex is presented in onTestEnd', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': `
|
||||
class Reporter {
|
||||
@ -701,9 +706,9 @@ test('parallelIndex is presented in onTestEnd', async ({ runInlineTest }) => {
|
||||
}, { 'reporter': '', 'workers': 1 });
|
||||
|
||||
expect(result.output).toContain('parallelIndex: 0');
|
||||
});
|
||||
});
|
||||
|
||||
test('test and step error should have code snippet', async ({ runInlineTest }) => {
|
||||
test('test and step error should have code snippet', async ({ runInlineTest }) => {
|
||||
const testErrorFile = test.info().outputPath('testError.txt');
|
||||
const stepErrorFile = test.info().outputPath('stepError.txt');
|
||||
const result = await runInlineTest({
|
||||
@ -712,10 +717,12 @@ test('test and step error should have code snippet', async ({ runInlineTest }) =
|
||||
class Reporter {
|
||||
onStepEnd(test, result, step) {
|
||||
console.log('\\n%%onStepEnd: ' + step.error?.snippet?.length);
|
||||
if (step.error?.snippet)
|
||||
fs.writeFileSync('${stepErrorFile.replace(/\\/g, '\\\\')}', step.error?.snippet);
|
||||
}
|
||||
onTestEnd(test, result) {
|
||||
console.log('\\n%%onTestEnd: ' + result.error?.snippet?.length);
|
||||
if (result.error)
|
||||
fs.writeFileSync('${testErrorFile.replace(/\\/g, '\\\\')}', result.error?.snippet);
|
||||
}
|
||||
onError(error) {
|
||||
@ -738,8 +745,8 @@ test('test and step error should have code snippet', async ({ runInlineTest }) =
|
||||
`,
|
||||
}, { 'reporter': '', 'workers': 1 });
|
||||
|
||||
expect(result.output).toContain('onTestEnd: 522');
|
||||
expect(result.output).toContain('onStepEnd: 522');
|
||||
expect(result.output).toContain('onTestEnd: 550');
|
||||
expect(result.output).toContain('onStepEnd: 550');
|
||||
expect(stripAnsi(fs.readFileSync(testErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => {
|
||||
4 | await test.step('step', async () => {
|
||||
> 5 | expect(1).toBe(2);
|
||||
@ -754,9 +761,9 @@ test('test and step error should have code snippet', async ({ runInlineTest }) =
|
||||
6 | });
|
||||
7 | });
|
||||
8 | `);
|
||||
});
|
||||
});
|
||||
|
||||
test('onError should have code snippet', async ({ runInlineTest }) => {
|
||||
test('onError should have code snippet', async ({ runInlineTest }) => {
|
||||
const errorFile = test.info().outputPath('error.txt');
|
||||
const result = await runInlineTest({
|
||||
'reporter.ts': `
|
||||
@ -779,7 +786,7 @@ test('onError should have code snippet', async ({ runInlineTest }) => {
|
||||
`,
|
||||
}, { 'reporter': '', 'workers': 1 });
|
||||
|
||||
expect(result.output).toContain('onError: 396');
|
||||
expect(result.output).toContain('onError: 412');
|
||||
expect(stripAnsi(fs.readFileSync(errorFile, 'utf8'))).toBe(` at a.spec.js:3
|
||||
|
||||
1 |
|
||||
@ -787,4 +794,6 @@ test('onError should have code snippet', async ({ runInlineTest }) => {
|
||||
> 3 | throw new Error('test');
|
||||
| ^
|
||||
4 | `);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user