diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index e1b68487b0..067ab6a76c 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -84,6 +84,12 @@ const config: PlaywrightTestConfig = { export default config; ``` +## property: TestConfig.configFile +* since: v1.27 +- type: ?<[string]> + +Path to config file, if any. + ## property: TestConfig.forbidOnly * since: v1.10 - type: ?<[boolean]> diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 8642b7f435..ed5d14406c 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -150,6 +150,13 @@ Filter to only run tests with a title **not** matching one of the patterns. This `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests). +## property: TestProject.id +* since: v1.27 +- type: ?<[string]> + +Unique project id within this config. + + ## property: TestProject.metadata * since: v1.10 - type: ?<[Metadata]> diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 45fd3d6927..ae4caa79c8 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -125,6 +125,7 @@ export class Loader { config.snapshotDir = path.resolve(configDir, config.snapshotDir); this._fullConfig._configDir = configDir; + this._fullConfig.configFile = this._configFile; this._fullConfig.rootDir = config.testDir || this._configDir; this._fullConfig._globalOutputDir = takeFirst(config.outputDir, throwawayArtifactsPath, baseFullConfig._globalOutputDir); this._fullConfig.forbidOnly = takeFirst(config.forbidOnly, baseFullConfig.forbidOnly); @@ -165,7 +166,7 @@ export class Loader { const candidate = name + (i ? i : ''); if (usedNames.has(candidate)) continue; - p._id = candidate; + p.id = candidate; usedNames.add(candidate); break; } @@ -277,7 +278,7 @@ export class Loader { process.env.PWTEST_USE_SCREENSHOTS_DIR = '1'; } return { - _id: '', + id: '', _fullConfig: fullConfig, _fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined), _expect: takeFirst(projectConfig.expect, config.expect, {}), @@ -391,20 +392,20 @@ class ProjectSuiteBuilder { test.retries = this._project.retries; const repeatEachIndexSuffix = repeatEachIndex ? ` (repeat:${repeatEachIndex})` : ''; // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles. - const testIdExpression = `[project=${this._project._id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`; + const testIdExpression = `[project=${this._project.id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`; const testId = to._fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); test.id = testId; test.repeatEachIndex = repeatEachIndex; - test._projectId = this._project._id; + test._projectId = this._project.id; if (!filter(test)) { to._entries.pop(); to.tests.pop(); } else { const pool = this._buildPool(entry); if (this._project._fullConfig._workerIsolation === 'isolate-pools') - test._workerHash = `run${this._project._id}-${pool.digest}-repeat${repeatEachIndex}`; + test._workerHash = `run${this._project.id}-${pool.digest}-repeat${repeatEachIndex}`; else - test._workerHash = `run${this._project._id}-repeat${repeatEachIndex}`; + test._workerHash = `run${this._project.id}-repeat${repeatEachIndex}`; test._pool = pool; } } @@ -436,7 +437,7 @@ class ProjectSuiteBuilder { (originalFixtures as any)[key] = value; } if (Object.entries(optionsFromConfig).length) - result.push({ fixtures: optionsFromConfig, location: { file: `project#${this._project._id}`, line: 1, column: 1 } }); + result.push({ fixtures: optionsFromConfig, location: { file: `project#${this._project.id}`, line: 1, column: 1 } }); if (Object.entries(originalFixtures).length) result.push({ fixtures: originalFixtures, location: f.location }); } @@ -647,6 +648,7 @@ export const baseFullConfig: FullConfigInternal = { projects: [], reporter: [[process.env.CI ? 'dot' : 'list']], reportSlowTests: { max: 5, threshold: 15000 }, + configFile: '', rootDir: path.resolve(process.cwd()), quiet: false, shard: null, diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 493b627247..7cdf49d6ed 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -18,6 +18,7 @@ import fs from 'fs'; import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep } from '../../types/testReporter'; import { prepareErrorStack } from './base'; +import { MultiMap } from 'playwright-core/lib/utils/multimap'; export function toPosixPath(aPath: string): string { return aPath.split(path.sep).join(path.posix.sep); @@ -53,7 +54,7 @@ class JSONReporter implements Reporter { private _serializeReport(): JSONReport { return { config: { - ...this.config, + ...removePrivateFields(this.config), rootDir: toPosixPath(this.config.rootDir), projects: this.config.projects.map(project => { return { @@ -61,6 +62,7 @@ class JSONReporter implements Reporter { repeatEach: project.repeatEach, retries: project.retries, metadata: project.metadata, + id: project.id, name: project.name, testDir: toPosixPath(project.testDir), testIgnore: serializePatterns(project.testIgnore), @@ -75,23 +77,32 @@ class JSONReporter implements Reporter { } private _mergeSuites(suites: Suite[]): JSONReportSuite[] { - const fileSuites = new Map(); - const result: JSONReportSuite[] = []; + const fileSuites = new MultiMap(); for (const projectSuite of suites) { + const projectId = projectSuite.project()!.id; + const projectName = projectSuite.project()!.name; for (const fileSuite of projectSuite.suites) { const file = fileSuite.location!.file; - if (!fileSuites.has(file)) { - const serialized = this._serializeSuite(fileSuite); - if (serialized) { - fileSuites.set(file, serialized); - result.push(serialized); - } - } else { - this._mergeTestsFromSuite(fileSuites.get(file)!, fileSuite); - } + const serialized = this._serializeSuite(projectId, projectName, fileSuite); + if (serialized) + fileSuites.set(file, serialized); } } - return result; + + const results: JSONReportSuite[] = []; + for (const [, suites] of fileSuites) { + const result: JSONReportSuite = { + title: suites[0].title, + file: suites[0].file, + column: 0, + line: 0, + specs: [], + }; + for (const suite of suites) + this._mergeTestsFromSuite(result, suite); + results.push(result); + } + return results; } private _relativeLocation(location: Location | undefined): Location { @@ -104,63 +115,61 @@ class JSONReporter implements Reporter { }; } - private _locationMatches(s: JSONReportSuite | JSONReportSpec, location: Location | undefined) { - const relative = this._relativeLocation(location); - return s.file === relative.file && s.line === relative.line && s.column === relative.column; + private _locationMatches(s1: JSONReportSuite | JSONReportSpec, s2: JSONReportSuite | JSONReportSpec) { + return s1.file === s2.file && s1.line === s2.line && s1.column === s2.column; } - private _mergeTestsFromSuite(to: JSONReportSuite, from: Suite) { - for (const fromSuite of from.suites) { - const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && this._locationMatches(s, from.location)); + private _mergeTestsFromSuite(to: JSONReportSuite, from: JSONReportSuite) { + for (const fromSuite of from.suites || []) { + const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && this._locationMatches(s, fromSuite)); if (toSuite) { this._mergeTestsFromSuite(toSuite, fromSuite); } else { - const serialized = this._serializeSuite(fromSuite); - if (serialized) { - if (!to.suites) - to.suites = []; - to.suites.push(serialized); - } + if (!to.suites) + to.suites = []; + to.suites.push(fromSuite); } } - for (const test of from.tests) { - const toSpec = to.specs.find(s => s.title === test.title && s.file === toPosixPath(path.relative(this.config.rootDir, test.location.file)) && s.line === test.location.line && s.column === test.location.column); + + for (const spec of from.specs || []) { + const toSpec = to.specs.find(s => s.title === spec.title && s.file === toPosixPath(path.relative(this.config.rootDir, spec.file)) && s.line === spec.line && s.column === spec.column); if (toSpec) - toSpec.tests.push(this._serializeTest(test)); + toSpec.tests.push(...spec.tests); else - to.specs.push(this._serializeTestSpec(test)); + to.specs.push(spec); } } - private _serializeSuite(suite: Suite): null | JSONReportSuite { + private _serializeSuite(projectId: string, projectName: string, suite: Suite): null | JSONReportSuite { if (!suite.allTests().length) return null; - const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[]; + const suites = suite.suites.map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; return { title: suite.title, ...this._relativeLocation(suite.location), - specs: suite.tests.map(test => this._serializeTestSpec(test)), + specs: suite.tests.map(test => this._serializeTestSpec(projectId, projectName, test)), suites: suites.length ? suites : undefined, }; } - private _serializeTestSpec(test: TestCase): JSONReportSpec { + private _serializeTestSpec(projectId: string, projectName: string, test: TestCase): JSONReportSpec { return { title: test.title, ok: test.ok(), tags: (test.title.match(/@[\S]+/g) || []).map(t => t.substring(1)), - tests: [this._serializeTest(test)], + tests: [this._serializeTest(projectId, projectName, test)], id: test.id, ...this._relativeLocation(test.location), }; } - private _serializeTest(test: TestCase): JSONReportTest { + private _serializeTest(projectId: string, projectName: string, test: TestCase): JSONReportTest { return { timeout: test.timeout, annotations: test.annotations, expectedStatus: test.expectedStatus, - projectName: test.titlePath()[1], + projectId, + projectName, results: test.results.map(r => this._serializeTestResult(r, test)), status: test.outcome(), }; @@ -217,6 +226,10 @@ function stdioEntry(s: string | Buffer): any { return { buffer: s.toString('base64') }; } +function removePrivateFields(config: FullConfig): FullConfig { + return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig; +} + export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] { if (!Array.isArray(patterns)) patterns = [patterns]; diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 144c188cb7..7df06bdcb9 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -512,10 +512,10 @@ export class Runner { for (const [project, files] of filesByProject) { for (const file of files) { const group: TestGroup = { - workerHash: `run${project._id}-repeat${repeatEachIndex}`, + workerHash: `run${project.id}-repeat${repeatEachIndex}`, requireFile: file, repeatEachIndex, - projectId: project._id, + projectId: project.id, tests: [], watchMode: true, }; diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts index 743344a389..02b098f9a7 100644 --- a/packages/playwright-test/src/testInfo.ts +++ b/packages/playwright-test/src/testInfo.ts @@ -118,8 +118,8 @@ export class TestInfoImpl implements TestInfo { const fullTitleWithoutSpec = test.titlePath().slice(1).join(' '); let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec)); - if (project._id) - testOutputDir += '-' + sanitizeForFilePath(project._id); + if (project.id) + testOutputDir += '-' + sanitizeForFilePath(project.id); if (this.retry) testOutputDir += '-retry' + this.retry; if (this.repeatEachIndex) diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 45de213264..8bcac6c47d 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -63,7 +63,6 @@ export interface FullConfigInternal extends FullConfigPublic { * increasing the surface area of the public API type called FullProject. */ export interface FullProjectInternal extends FullProjectPublic { - _id: string; _fullConfig: FullConfigInternal; _fullyParallel: boolean; _expect: Project['expect']; diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index 70ca8c82f6..9255d3bca0 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -160,7 +160,7 @@ export class WorkerRunner extends EventEmitter { return; this._loader = await Loader.deserialize(this._params.loader); - this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!; + this._project = this._loader.fullConfig().projects.find(p => p.id === this._params.projectId)!; } async runTestGroup(runPayload: RunPayload) { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index c5750e9cb4..c78ad4a558 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -198,6 +198,10 @@ export interface FullProject { * Metadata that will be put directly to the test report serialized as JSON. */ metadata: Metadata; + /** + * Unique project id within this config. + */ + id: string; /** * Project name is visible in the report and during test execution. */ @@ -578,6 +582,11 @@ interface TestConfig { }; }; + /** + * Path to config file, if any. + */ + configFile?: string; + /** * Whether to exit with an error if any tests or groups are marked as * [test.only(title, testFunction)](https://playwright.dev/docs/api/class-test#test-only) or @@ -1298,6 +1307,10 @@ export interface FullConfig { * */ webServer: TestConfigWebServer | null; + /** + * Path to config file, if any. + */ + configFile?: string; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; @@ -4402,6 +4415,11 @@ interface TestProject { */ grepInvert?: RegExp|Array; + /** + * Unique project id within this config. + */ + id?: string; + /** * Metadata that will be put directly to the test report serialized as JSON. */ diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 9581ea18fa..2fb290cb3e 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -444,6 +444,7 @@ export interface JSONReport { repeatEach: number, retries: number, metadata: Metadata, + id: string, name: string, testDir: string, testIgnore: string[], @@ -480,6 +481,7 @@ export interface JSONReportTest { annotations: { type: string, description?: string }[], expectedStatus: TestStatus; projectName: string; + projectId: string; results: JSONReportTestResult[]; status: 'skipped' | 'expected' | 'unexpected' | 'flaky'; } diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 6d1ccfc937..b8048028da 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -41,6 +41,7 @@ export interface FullProject { grep: RegExp | RegExp[]; grepInvert: RegExp | RegExp[] | null; metadata: Metadata; + id: string; name: string; snapshotDir: string; outputDir: string; @@ -92,6 +93,7 @@ export interface FullConfig { updateSnapshots: 'all' | 'none' | 'missing'; workers: number; webServer: TestConfigWebServer | null; + configFile?: string; // [internal] !!! DO NOT ADD TO THIS !!! See prior note. } diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 1837fca003..e8052fdf8f 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -55,6 +55,7 @@ export interface JSONReport { repeatEach: number, retries: number, metadata: Metadata, + id: string, name: string, testDir: string, testIgnore: string[], @@ -91,6 +92,7 @@ export interface JSONReportTest { annotations: { type: string, description?: string }[], expectedStatus: TestStatus; projectName: string; + projectId: string; results: JSONReportTestResult[]; status: 'skipped' | 'expected' | 'unexpected' | 'flaky'; } diff --git a/utils/upload_flakiness_dashboard.sh b/utils/upload_flakiness_dashboard.sh index 3af4e60692..a9f08600c8 100755 --- a/utils/upload_flakiness_dashboard.sh +++ b/utils/upload_flakiness_dashboard.sh @@ -95,7 +95,7 @@ az storage blob upload --connection-string "${FLAKINESS_CONNECTION_STRING}" -c u UTC_DATE=$(cat <