docs: update test reporters (#21337)

This commit is contained in:
Debbie O'Brien 2023-03-03 16:52:28 +01:00 committed by GitHub
parent eb3f8bfba8
commit e29a6e7bb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -12,19 +12,7 @@ npx playwright test --reporter=line
For more control, you can specify reporters programmatically in the [configuration file](./test-configuration.md).
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'line',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -36,22 +24,7 @@ export default defineConfig({
You can use multiple reporters at the same time. For example you can use `'list'` for nice terminal output and `'json'` to get a comprehensive json file with the test results.
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [
['list'],
['json', { outputFile: 'test-results.json' }]
],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -66,20 +39,7 @@ export default defineConfig({
You can use different reporters locally and on CI. For example, using concise `'dot'` reporter avoids too much output. This is the default on CI.
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
// Concise 'dot' for CI, default 'list' when running locally
reporter: process.env.CI ? 'dot' : 'list',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -100,19 +60,7 @@ List reporter is default (except on CI where the `dot` reporter is default). It
npx playwright test --reporter=list
```
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'list',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -139,19 +87,7 @@ Running 124 tests using 6 workers
You can opt into the step rendering via passing the following config option:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['list', { printSteps: true }]],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -167,19 +103,7 @@ Line reporter is more concise than the list reporter. It uses a single line to r
npx playwright test --reporter=line
```
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'line',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -209,19 +133,7 @@ Dot reporter is very concise - it only produces a single character per successfu
npx playwright test --reporter=dot
```
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'dot',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -250,23 +162,7 @@ By default, HTML report is opened automatically if some of the tests failed. You
You can also configure `host` and `port` that are used to serve the HTML report.
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['html', {
open: 'never',
host: '0.0.0.0',
port: 9223,
}]],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -278,19 +174,8 @@ By default, report is written into the `playwright-report` folder in the current
that location using the `PLAYWRIGHT_HTML_REPORT` environment variable or a reporter configuration.
In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['html', { outputFolder: 'my-report' }]],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -334,19 +219,8 @@ npx playwright test --reporter=json
```
In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['json', { outputFile: 'results.json' }]],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -375,19 +249,8 @@ npx playwright test --reporter=junit
```
In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['junit', { outputFile: 'results.xml' }]],
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -399,34 +262,7 @@ The JUnit reporter provides support for embedding additional information on the
In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead.
```js tab=js-js
// playwright.config.js
// @ts-check
// JUnit reporter config for Xray
const xrayOptions = {
// Whether to add <properties> with all annotations; default is false
embedAnnotationsAsProperties: true,
// By default, annotation is reported as <property name='' value=''>.
// These annotations are reported as <property name=''>value</property>.
textContentAnnotations: ['test_description'],
// This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner <item> element.
// Disables [[ATTACHMENT|path]] in the <system-out>.
embedAttachmentsAsProperty: 'testrun_evidence',
// Where to put the report.
outputFile: './xray-report.xml'
};
module.exports = defineConfig({
reporter: [['junit', xrayOptions]]
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
// JUnit reporter config for Xray
@ -454,20 +290,8 @@ export default defineConfig({
In the previous configuration sample, all annotations will be added as `<property>` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the `<property>`, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective `<property>`.
Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it).
```js tab=js-js
// @ts-check
const { test } = require('@playwright/test');
test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_id', description: '1234' });
testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
});
```
```js tab=js-ts
```js
// example.spec.ts/js
import { test } from '@playwright/test';
test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
@ -486,21 +310,9 @@ Embedding attachments can be used to attach screenshots or any other relevant ev
The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
});
```
```js tab=js-ts
// playwright.config.js
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
});
@ -508,19 +320,8 @@ export default defineConfig({
The following test adds attachments:
```js tab=js-js
// @ts-check
const { test } = require('@playwright/test');
test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
const file = testInfo.outputPath('evidence1.txt');
require('fs').writeFileSync(file, 'hello', 'utf8');
await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
});
```
```js tab=js-ts
```js
// example.spec.ts/js
import { test } from '@playwright/test';
test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
@ -537,21 +338,7 @@ You can use the built in `github` reporter to get automatic failure annotations
Note that all other reporters work on GitHub Actions as well, but do not provide annotations.
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
// 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot'
// default 'list' when running locally
reporter: process.env.CI ? 'github' : 'list',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
@ -565,33 +352,7 @@ export default defineConfig({
You can create a custom reporter by implementing a class with some of the reporter methods. Learn more about the [Reporter] API.
```js tab=js-js
// my-awesome-reporter.js
// @ts-check
/** @implements {import('@playwright/test/reporter').Reporter} */
class MyReporter {
onBegin(config, suite) {
console.log(`Starting the run with ${suite.allTests().length} tests`);
}
onTestBegin(test) {
console.log(`Starting test ${test.title}`);
}
onTestEnd(test, result) {
console.log(`Finished test ${test.title}: ${result.status}`);
}
onEnd(result) {
console.log(`Finished the run: ${result.status}`);
}
}
module.exports = MyReporter;
```
```js tab=js-ts
```js
// my-awesome-reporter.ts
import { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter';
@ -618,31 +379,16 @@ export default MyReporter;
Now use this reporter with [`property: TestConfig.reporter`].
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: './my-awesome-reporter.js',
});
```
```js tab=js-ts
// playwright.config.ts
```js
import { defineConfig } from '@playwright/test';
export default defineConfig({
reporter: './my-awesome-reporter.ts',
});
```
## Third party reporter showcase
* [Allure](https://www.npmjs.com/package/allure-playwright)
* [Monocart](https://github.com/cenfun/monocart-reporter)
* [Tesults](https://www.tesults.com/docs/playwright)
* [ReportPortal](https://github.com/reportportal/agent-js-playwright)