microsoft/playwright
1
1
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2024-11-29 01:53:54 +03:00
Code Issues Projects Releases Wiki Activity

chore: add common env vars for junit and json reporters (#30611)

Browse Source
This commit is contained in:
Yury Semikhatsky 2024-05-01 10:16:49 -07:00 committed by GitHub
parent c3d8b22198
commit 3b7c4fac22
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 167 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/src/test-reporters-js.md
View File

@ -231,7 +231,7 @@ Blob report supports following configuration options and environment variables:
|---|---|---|---|
| `PLAYWRIGHT_BLOB_OUTPUT_DIR` | `outputDir` | Directory to save the output. Existing content is deleted before writing the new report. | `blob-report`
| `PLAYWRIGHT_BLOB_OUTPUT_NAME` | `fileName` | Report file name. | `report-<project>-<hash>-<shard_number>.zip`
| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path for the output. If defined, `outputDir` and `fileName` will be ignored. | `undefined`
| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `outputDir` and `fileName` will be ignored. | `undefined`
### JSON reporter
@ -267,7 +267,9 @@ JSON report supports following configuration options and environment variables:
| Environment Variable Name | Reporter Config Option| Description | Default
|---|---|---|---|
| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JSON report is printed to stdout.
| `PLAYWRIGHT_JSON_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is specified. | `cwd` or config directory.
| `PLAYWRIGHT_JSON_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JSON report is printed to the stdout.
| `PLAYWRIGHT_JSON_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JSON_OUTPUT_DIR` and `PLAYWRIGHT_JSON_OUTPUT_NAME` will be ignored. | JSON report is printed to the stdout.
### JUnit reporter
@ -303,7 +305,9 @@ JUnit report supports following configuration options and environment variables:
| Environment Variable Name | Reporter Config Option| Description | Default
|---|---|---|---|
| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JUnit report is printed to stdout.
| `PLAYWRIGHT_JUNIT_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is not specified. | `cwd` or config directory.
| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JUnit report is printed to the stdout.
| `PLAYWRIGHT_JUNIT_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JUNIT_OUTPUT_DIR` and `PLAYWRIGHT_JUNIT_OUTPUT_NAME` will be ignored. | JUnit report is printed to the stdout.
| | `stripANSIControlSequences` | Whether to remove ANSI control sequences from the text before writing it in the report. | By default output text is added as is.
| | `includeProjectInTestName` | Whether to include Playwright project name in every test case as a name prefix. | By default not included.
| `PLAYWRIGHT_JUNIT_SUITE_ID` | | Value of the `id` attribute on the root `<testsuites/>` report entry. | Empty string.

47
packages/playwright/src/reporters/base.ts
View File

@ -19,6 +19,7 @@ import path from 'path';
import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter';
import { getPackageManagerExecCommand } from 'playwright-core/lib/utils';
import type { ReporterV2 } from './reporterV2';
import { resolveReporterOutputPath } from '../util';
export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
export const kOutputSymbol = Symbol('output');
@ -547,3 +548,49 @@ function fitToWidth(line: string, width: number, prefix?: string): string {
function belongsToNodeModules(file: string) {
return file.includes(`${path.sep}node_modules${path.sep}`);
}
function resolveFromEnv(name: string): string | undefined {
const value = process.env[name];
if (value)
return path.resolve(process.cwd(), value);
return undefined;
}
// In addition to `outputFile` the function returns `outputDir` which should
// be cleaned up if present by some reporters contract.
export function resolveOutputFile(reporterName: string, options: {
configDir: string,
outputDir?: string,
fileName?: string,
outputFile?: string,
default?: {
fileName: string,
outputDir: string,
}
}): { outputFile: string, outputDir?: string } |undefined {
const name = reporterName.toUpperCase();
let outputFile;
if (options.outputFile)
outputFile = path.resolve(options.configDir, options.outputFile);
if (!outputFile)
outputFile = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_FILE`);
// Return early to avoid deleting outputDir.
if (outputFile)
return { outputFile };
let outputDir;
if (options.outputDir)
outputDir = path.resolve(options.configDir, options.outputDir);
if (!outputDir)
outputDir = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_DIR`);
if (!outputDir && options.default)
outputDir = resolveReporterOutputPath(options.default.outputDir, options.configDir, undefined);
if (!outputFile) {
const reportName = options.fileName ?? process.env[`PLAYWRIGHT_${name}_OUTPUT_NAME`] ?? options.default?.fileName;
if (!reportName)
return undefined;
outputFile = path.resolve(outputDir ?? process.cwd(), reportName);
}
return { outputFile, outputDir };
}

34
packages/playwright/src/reporters/blob.ts
View File

@ -24,7 +24,7 @@ import type { FullConfig, FullResult, TestResult } from '../../types/testReporte
import type { JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver';
import { TeleReporterEmitter } from './teleEmitter';
import { yazl } from 'playwright-core/lib/zipBundle';
import { resolveReporterOutputPath } from '../util';
import { resolveOutputFile } from './base';
type BlobReporterOptions = {
configDir: string;
@ -107,17 +107,15 @@ export class BlobReporter extends TeleReporterEmitter {
}
private async _prepareOutputFile() {
let outputFile = reportOutputFileFromEnv();
if (!outputFile && this._options.outputFile)
outputFile = path.resolve(this._options.configDir, this._options.outputFile);
// Explicit `outputFile` overrides `outputDir` and `fileName` options.
if (!outputFile) {
const reportName = this._options.fileName || process.env[`PLAYWRIGHT_BLOB_OUTPUT_NAME`] || this._defaultReportName(this._config);
const outputDir = resolveReporterOutputPath('blob-report', this._options.configDir, this._options.outputDir ?? reportOutputDirFromEnv());
if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE)
await removeFolders([outputDir]);
outputFile = path.resolve(outputDir, reportName);
}
const { outputFile, outputDir } = resolveOutputFile('BLOB', {
...this._options,
default: {
fileName: this._defaultReportName(this._config),
outputDir: 'blob-report',
}
})!;
if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE)
await removeFolders([outputDir!]);
await fs.promises.mkdir(path.dirname(outputFile), { recursive: true });
return outputFile;
}
@ -149,15 +147,3 @@ export class BlobReporter extends TeleReporterEmitter {
});
}
}
function reportOutputDirFromEnv(): string | undefined {
if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`])
return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`]);
return undefined;
}
function reportOutputFileFromEnv(): string | undefined {
if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`])
return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`]);
return undefined;
}

35
packages/playwright/src/reporters/json.ts
View File

@ -17,24 +17,29 @@
import fs from 'fs';
import path from 'path';
import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter';
import { formatError, prepareErrorStack } from './base';
import { MultiMap, assert, toPosixPath } from 'playwright-core/lib/utils';
import { formatError, prepareErrorStack, resolveOutputFile } from './base';
import { MultiMap, toPosixPath } from 'playwright-core/lib/utils';
import { getProjectId } from '../common/config';
import EmptyReporter from './empty';
type JSONOptions = {
outputFile?: string,
configDir: string,
};
class JSONReporter extends EmptyReporter {
config!: FullConfig;
suite!: Suite;
private _errors: TestError[] = [];
private _outputFile: string | undefined;
private _resolvedOutputFile: string | undefined;
constructor(options: { outputFile?: string } = {}) {
constructor(options: JSONOptions) {
super();
this._outputFile = options.outputFile || reportOutputNameFromEnv();
this._resolvedOutputFile = resolveOutputFile('JSON', options)?.outputFile;
}
override printsToStdio() {
return !this._outputFile;
return !this._resolvedOutputFile;
}
override onConfigure(config: FullConfig) {
@ -50,7 +55,7 @@ class JSONReporter extends EmptyReporter {
}
override async onEnd(result: FullResult) {
await outputReport(this._serializeReport(result), this.config, this._outputFile);
await outputReport(this._serializeReport(result), this._resolvedOutputFile);
}
private _serializeReport(result: FullResult): JSONReport {
@ -228,13 +233,11 @@ class JSONReporter extends EmptyReporter {
}
}
async function outputReport(report: JSONReport, config: FullConfig, outputFile: string | undefined) {
async function outputReport(report: JSONReport, resolvedOutputFile: string | undefined) {
const reportString = JSON.stringify(report, undefined, 2);
if (outputFile) {
assert(config.configFile || path.isAbsolute(outputFile), 'Expected fully resolved path if not using config file.');
outputFile = config.configFile ? path.resolve(path.dirname(config.configFile), outputFile) : outputFile;
await fs.promises.mkdir(path.dirname(outputFile), { recursive: true });
await fs.promises.writeFile(outputFile, reportString);
if (resolvedOutputFile) {
await fs.promises.mkdir(path.dirname(resolvedOutputFile), { recursive: true });
await fs.promises.writeFile(resolvedOutputFile, reportString);
} else {
console.log(reportString);
}
@ -250,12 +253,6 @@ function removePrivateFields(config: FullConfig): FullConfig {
return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig;
}
function reportOutputNameFromEnv(): string | undefined {
if (process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`])
return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]);
return undefined;
}
export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] {
if (!Array.isArray(patterns))
patterns = [patterns];

18
packages/playwright/src/reporters/junit.ts
View File

@ -17,7 +17,7 @@
import fs from 'fs';
import path from 'path';
import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter';
import { formatFailure, stripAnsiEscapes } from './base';
import { formatFailure, resolveOutputFile, stripAnsiEscapes } from './base';
import EmptyReporter from './empty';
type JUnitOptions = {
@ -25,7 +25,7 @@ type JUnitOptions = {
stripANSIControlSequences?: boolean,
includeProjectInTestName?: boolean,
configDir?: string,
configDir: string,
};
class JUnitReporter extends EmptyReporter {
@ -40,14 +40,12 @@ class JUnitReporter extends EmptyReporter {
private stripANSIControlSequences = false;
private includeProjectInTestName = false;
constructor(options: JUnitOptions = {}) {
constructor(options: JUnitOptions) {
super();
this.stripANSIControlSequences = options.stripANSIControlSequences || false;
this.includeProjectInTestName = options.includeProjectInTestName || false;
this.configDir = options.configDir || '';
const outputFile = options.outputFile || reportOutputNameFromEnv();
if (outputFile)
this.resolvedOutputFile = path.resolve(this.configDir, outputFile);
this.configDir = options.configDir;
this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile;
}
override printsToStdio() {
@ -261,10 +259,4 @@ function escape(text: string, stripANSIControlSequences: boolean, isCharacterDat
return text;
}
function reportOutputNameFromEnv(): string | undefined {
if (process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`])
return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]);
return undefined;
}
export default JUnitReporter;

6
tests/playwright-test/reporter-blob.spec.ts
View File

@ -1292,12 +1292,18 @@ test('support PLAYWRIGHT_BLOB_OUTPUT_FILE environment variable', async ({ runInl
test('math 1 @smoke', async ({}) => {});
`,
};
const defaultDir = test.info().outputPath('blob-report');
fs.mkdirSync(defaultDir, { recursive: true });
const file = path.join(defaultDir, 'some.file');
fs.writeFileSync(file, 'content');
await runInlineTest(files, { shard: `1/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: 'subdir/report-one.zip' });
await runInlineTest(files, { shard: `2/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: test.info().outputPath('subdir/report-two.zip') });
const reportDir = test.info().outputPath('subdir');
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles.sort()).toEqual(['report-one.zip', 'report-two.zip']);
expect(fs.existsSync(file), 'Default directory should not be cleaned up if output file is specified.').toBe(true);
});
test('keep projects with same name different bot name separate', async ({ runInlineTest, mergeReports, showReport, page }) => {

38
tests/playwright-test/reporter-json.spec.ts
View File

@ -288,4 +288,42 @@ test.describe('report location', () => {
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
});
test('support PLAYWRIGHT_JSON_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'foo/package.json': `{ "name": "foo" }`,
// unused config along "search path"
'foo/bar/playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'foo/bar/baz/tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'json' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_JSON_OUTPUT_FILE': '../my-report.json' }, {
cwd: 'foo/bar/baz/tests',
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
});
test('support PLAYWRIGHT_JSON_OUTPUT_DIR and PLAYWRIGHT_JSON_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'json' }, { 'PLAYWRIGHT_JSON_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JSON_OUTPUT_NAME': 'baz/my-report.json' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
});
});

38
tests/playwright-test/reporter-junit.spec.ts
View File

@ -504,6 +504,44 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
test('support PLAYWRIGHT_JUNIT_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'foo/package.json': `{ "name": "foo" }`,
// unused config along "search path"
'foo/bar/playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'foo/bar/baz/tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_FILE': '../my-report.xml' }, {
cwd: 'foo/bar/baz/tests',
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
test('support PLAYWRIGHT_JUNIT_OUTPUT_DIR and PLAYWRIGHT_JUNIT_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': 'baz/my-report.xml' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
});
test('testsuites time is test run wall time', async ({ runInlineTest }) => {