chore: project.id, configFile in reporter apis (#17346)

docs/src/test-api/class-testconfig.md

@@ -84,6 +84,12 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```
 
+## property: TestConfig.configFile
+* since: v1.27
+- type: ?<[string]>
+
+Path to config file, if any.
+
 ## property: TestConfig.forbidOnly
 * since: v1.10
 - type: ?<[boolean]>

docs/src/test-api/class-testproject.md

@@ -150,6 +150,13 @@ Filter to only run tests with a title **not** matching one of the patterns. This
 
 `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests).
 
+## property: TestProject.id
+* since: v1.27
+- type: ?<[string]>
+
+Unique project id within this config.
+
+
 ## property: TestProject.metadata
 * since: v1.10
 - type: ?<[Metadata]>

packages/playwright-test/src/loader.ts

@@ -125,6 +125,7 @@ export class Loader {
       config.snapshotDir = path.resolve(configDir, config.snapshotDir);
 
     this._fullConfig._configDir = configDir;
+    this._fullConfig.configFile = this._configFile;
     this._fullConfig.rootDir = config.testDir || this._configDir;
     this._fullConfig._globalOutputDir = takeFirst(config.outputDir, throwawayArtifactsPath, baseFullConfig._globalOutputDir);
     this._fullConfig.forbidOnly = takeFirst(config.forbidOnly, baseFullConfig.forbidOnly);
@@ -165,7 +166,7 @@ export class Loader {
         const candidate = name + (i ? i : '');
         if (usedNames.has(candidate))
           continue;
-        p._id = candidate;
+        p.id = candidate;
         usedNames.add(candidate);
         break;
       }
@@ -277,7 +278,7 @@ export class Loader {
       process.env.PWTEST_USE_SCREENSHOTS_DIR = '1';
     }
     return {
-      _id: '',
+      id: '',
       _fullConfig: fullConfig,
       _fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined),
       _expect: takeFirst(projectConfig.expect, config.expect, {}),
@@ -391,20 +392,20 @@ class ProjectSuiteBuilder {
         test.retries = this._project.retries;
         const repeatEachIndexSuffix = repeatEachIndex ? ` (repeat:${repeatEachIndex})` : '';
         // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles.
-        const testIdExpression = `[project=${this._project._id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`;
+        const testIdExpression = `[project=${this._project.id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`;
         const testId = to._fileId + '-' + calculateSha1(testIdExpression).slice(0, 20);
         test.id = testId;
         test.repeatEachIndex = repeatEachIndex;
-        test._projectId = this._project._id;
+        test._projectId = this._project.id;
         if (!filter(test)) {
           to._entries.pop();
           to.tests.pop();
         } else {
           const pool = this._buildPool(entry);
           if (this._project._fullConfig._workerIsolation === 'isolate-pools')
-            test._workerHash = `run${this._project._id}-${pool.digest}-repeat${repeatEachIndex}`;
+            test._workerHash = `run${this._project.id}-${pool.digest}-repeat${repeatEachIndex}`;
           else
-            test._workerHash = `run${this._project._id}-repeat${repeatEachIndex}`;
+            test._workerHash = `run${this._project.id}-repeat${repeatEachIndex}`;
           test._pool = pool;
         }
       }
@@ -436,7 +437,7 @@ class ProjectSuiteBuilder {
           (originalFixtures as any)[key] = value;
       }
       if (Object.entries(optionsFromConfig).length)
-        result.push({ fixtures: optionsFromConfig, location: { file: `project#${this._project._id}`, line: 1, column: 1 } });
+        result.push({ fixtures: optionsFromConfig, location: { file: `project#${this._project.id}`, line: 1, column: 1 } });
       if (Object.entries(originalFixtures).length)
         result.push({ fixtures: originalFixtures, location: f.location });
     }
@@ -647,6 +648,7 @@ export const baseFullConfig: FullConfigInternal = {
   projects: [],
   reporter: [[process.env.CI ? 'dot' : 'list']],
   reportSlowTests: { max: 5, threshold: 15000 },
+  configFile: '',
   rootDir: path.resolve(process.cwd()),
   quiet: false,
   shard: null,

packages/playwright-test/src/reporters/json.ts

@@ -18,6 +18,7 @@ import fs from 'fs';
 import path from 'path';
 import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep } from '../../types/testReporter';
 import { prepareErrorStack } from './base';
+import { MultiMap } from 'playwright-core/lib/utils/multimap';
 
 export function toPosixPath(aPath: string): string {
   return aPath.split(path.sep).join(path.posix.sep);
@@ -53,7 +54,7 @@ class JSONReporter implements Reporter {
   private _serializeReport(): JSONReport {
     return {
       config: {
-        ...this.config,
+        ...removePrivateFields(this.config),
         rootDir: toPosixPath(this.config.rootDir),
         projects: this.config.projects.map(project => {
           return {
@@ -61,6 +62,7 @@ class JSONReporter implements Reporter {
             repeatEach: project.repeatEach,
             retries: project.retries,
             metadata: project.metadata,
+            id: project.id,
             name: project.name,
             testDir: toPosixPath(project.testDir),
             testIgnore: serializePatterns(project.testIgnore),
@@ -75,23 +77,32 @@ class JSONReporter implements Reporter {
   }
 
   private _mergeSuites(suites: Suite[]): JSONReportSuite[] {
-    const fileSuites = new Map<string, JSONReportSuite>();
-    const result: JSONReportSuite[] = [];
+    const fileSuites = new MultiMap<string, JSONReportSuite>();
     for (const projectSuite of suites) {
+      const projectId = projectSuite.project()!.id;
+      const projectName = projectSuite.project()!.name;
       for (const fileSuite of projectSuite.suites) {
         const file = fileSuite.location!.file;
-        if (!fileSuites.has(file)) {
-          const serialized = this._serializeSuite(fileSuite);
-          if (serialized) {
-            fileSuites.set(file, serialized);
-            result.push(serialized);
-          }
-        } else {
-          this._mergeTestsFromSuite(fileSuites.get(file)!, fileSuite);
-        }
+        const serialized = this._serializeSuite(projectId, projectName, fileSuite);
+        if (serialized)
+          fileSuites.set(file, serialized);
       }
     }
-    return result;
+
+    const results: JSONReportSuite[] = [];
+    for (const [, suites] of fileSuites) {
+      const result: JSONReportSuite = {
+        title: suites[0].title,
+        file: suites[0].file,
+        column: 0,
+        line: 0,
+        specs: [],
+      };
+      for (const suite of suites)
+        this._mergeTestsFromSuite(result, suite);
+      results.push(result);
+    }
+    return results;
   }
 
   private _relativeLocation(location: Location | undefined): Location {
@@ -104,63 +115,61 @@ class JSONReporter implements Reporter {
     };
   }
 
-  private _locationMatches(s: JSONReportSuite | JSONReportSpec, location: Location | undefined) {
-    const relative = this._relativeLocation(location);
-    return s.file === relative.file && s.line === relative.line && s.column === relative.column;
+  private _locationMatches(s1: JSONReportSuite | JSONReportSpec, s2: JSONReportSuite | JSONReportSpec) {
+    return s1.file === s2.file && s1.line === s2.line && s1.column === s2.column;
   }
 
-  private _mergeTestsFromSuite(to: JSONReportSuite, from: Suite) {
-    for (const fromSuite of from.suites) {
-      const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && this._locationMatches(s, from.location));
+  private _mergeTestsFromSuite(to: JSONReportSuite, from: JSONReportSuite) {
+    for (const fromSuite of from.suites || []) {
+      const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && this._locationMatches(s, fromSuite));
       if (toSuite) {
         this._mergeTestsFromSuite(toSuite, fromSuite);
       } else {
-        const serialized = this._serializeSuite(fromSuite);
-        if (serialized) {
-          if (!to.suites)
-            to.suites = [];
-          to.suites.push(serialized);
-        }
+        if (!to.suites)
+          to.suites = [];
+        to.suites.push(fromSuite);
       }
     }
-    for (const test of from.tests) {
-      const toSpec = to.specs.find(s => s.title === test.title && s.file === toPosixPath(path.relative(this.config.rootDir, test.location.file)) && s.line === test.location.line && s.column === test.location.column);
+
+    for (const spec of from.specs || []) {
+      const toSpec = to.specs.find(s => s.title === spec.title && s.file === toPosixPath(path.relative(this.config.rootDir, spec.file)) && s.line === spec.line && s.column === spec.column);
       if (toSpec)
-        toSpec.tests.push(this._serializeTest(test));
+        toSpec.tests.push(...spec.tests);
       else
-        to.specs.push(this._serializeTestSpec(test));
+        to.specs.push(spec);
     }
   }
 
-  private _serializeSuite(suite: Suite): null | JSONReportSuite {
+  private _serializeSuite(projectId: string, projectName: string, suite: Suite): null | JSONReportSuite {
     if (!suite.allTests().length)
       return null;
-    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[];
+    const suites = suite.suites.map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[];
     return {
       title: suite.title,
       ...this._relativeLocation(suite.location),
-      specs: suite.tests.map(test => this._serializeTestSpec(test)),
+      specs: suite.tests.map(test => this._serializeTestSpec(projectId, projectName, test)),
       suites: suites.length ? suites : undefined,
     };
   }
 
-  private _serializeTestSpec(test: TestCase): JSONReportSpec {
+  private _serializeTestSpec(projectId: string, projectName: string, test: TestCase): JSONReportSpec {
     return {
       title: test.title,
       ok: test.ok(),
       tags: (test.title.match(/@[\S]+/g) || []).map(t => t.substring(1)),
-      tests: [this._serializeTest(test)],
+      tests: [this._serializeTest(projectId, projectName, test)],
       id: test.id,
       ...this._relativeLocation(test.location),
     };
   }
 
-  private _serializeTest(test: TestCase): JSONReportTest {
+  private _serializeTest(projectId: string, projectName: string, test: TestCase): JSONReportTest {
     return {
       timeout: test.timeout,
       annotations: test.annotations,
       expectedStatus: test.expectedStatus,
-      projectName: test.titlePath()[1],
+      projectId,
+      projectName,
       results: test.results.map(r => this._serializeTestResult(r, test)),
       status: test.outcome(),
     };
@@ -217,6 +226,10 @@ function stdioEntry(s: string | Buffer): any {
   return { buffer: s.toString('base64') };
 }
 
+function removePrivateFields(config: FullConfig): FullConfig {
+  return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig;
+}
+
 export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] {
   if (!Array.isArray(patterns))
     patterns = [patterns];

packages/playwright-test/src/runner.ts

@@ -512,10 +512,10 @@ export class Runner {
     for (const [project, files] of filesByProject) {
       for (const file of files) {
         const group: TestGroup = {
-          workerHash: `run${project._id}-repeat${repeatEachIndex}`,
+          workerHash: `run${project.id}-repeat${repeatEachIndex}`,
           requireFile: file,
           repeatEachIndex,
-          projectId: project._id,
+          projectId: project.id,
           tests: [],
           watchMode: true,
         };

packages/playwright-test/src/testInfo.ts

@@ -118,8 +118,8 @@ export class TestInfoImpl implements TestInfo {
       const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ');
 
       let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec));
-      if (project._id)
-        testOutputDir += '-' + sanitizeForFilePath(project._id);
+      if (project.id)
+        testOutputDir += '-' + sanitizeForFilePath(project.id);
       if (this.retry)
         testOutputDir += '-retry' + this.retry;
       if (this.repeatEachIndex)

packages/playwright-test/src/types.ts

@@ -63,7 +63,6 @@ export interface FullConfigInternal extends FullConfigPublic {
  * increasing the surface area of the public API type called FullProject.
  */
 export interface FullProjectInternal extends FullProjectPublic {
-  _id: string;
   _fullConfig: FullConfigInternal;
   _fullyParallel: boolean;
   _expect: Project['expect'];

packages/playwright-test/src/workerRunner.ts

@@ -160,7 +160,7 @@ export class WorkerRunner extends EventEmitter {
       return;
 
     this._loader = await Loader.deserialize(this._params.loader);
-    this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!;
+    this._project = this._loader.fullConfig().projects.find(p => p.id === this._params.projectId)!;
   }
 
   async runTestGroup(runPayload: RunPayload) {

packages/playwright-test/types/test.d.ts

@@ -198,6 +198,10 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * Metadata that will be put directly to the test report serialized as JSON.
    */
   metadata: Metadata;
+  /**
+   * Unique project id within this config.
+   */
+  id: string;
   /**
    * Project name is visible in the report and during test execution.
    */
@@ -578,6 +582,11 @@ interface TestConfig {
     };
   };
 
+  /**
+   * Path to config file, if any.
+   */
+  configFile?: string;
+
   /**
    * Whether to exit with an error if any tests or groups are marked as
    * [test.only(title, testFunction)](https://playwright.dev/docs/api/class-test#test-only) or
@@ -1298,6 +1307,10 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    *
    */
   webServer: TestConfigWebServer | null;
+  /**
+   * Path to config file, if any.
+   */
+  configFile?: string;
 }
 
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
@@ -4402,6 +4415,11 @@ interface TestProject {
    */
   grepInvert?: RegExp|Array<RegExp>;
 
+  /**
+   * Unique project id within this config.
+   */
+  id?: string;
+
   /**
    * Metadata that will be put directly to the test report serialized as JSON.
    */

packages/playwright-test/types/testReporter.d.ts

@@ -444,6 +444,7 @@ export interface JSONReport {
       repeatEach: number,
       retries: number,
       metadata: Metadata,
+      id: string,
       name: string,
       testDir: string,
       testIgnore: string[],
@@ -480,6 +481,7 @@ export interface JSONReportTest {
   annotations: { type: string, description?: string }[],
   expectedStatus: TestStatus;
   projectName: string;
+  projectId: string;
   results: JSONReportTestResult[];
   status: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 }

utils/generate_types/overrides-test.d.ts

@@ -41,6 +41,7 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   grep: RegExp | RegExp[];
   grepInvert: RegExp | RegExp[] | null;
   metadata: Metadata;
+  id: string;
   name: string;
   snapshotDir: string;
   outputDir: string;
@@ -92,6 +93,7 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   updateSnapshots: 'all' | 'none' | 'missing';
   workers: number;
   webServer: TestConfigWebServer | null;
+  configFile?: string;
   // [internal] !!! DO NOT ADD TO THIS !!! See prior note.
 }
 

utils/generate_types/overrides-testReporter.d.ts

@@ -55,6 +55,7 @@ export interface JSONReport {
       repeatEach: number,
       retries: number,
       metadata: Metadata,
+      id: string,
       name: string,
       testDir: string,
       testIgnore: string[],
@@ -91,6 +92,7 @@ export interface JSONReportTest {
   annotations: { type: string, description?: string }[],
   expectedStatus: TestStatus;
   projectName: string;
+  projectId: string;
   results: JSONReportTestResult[];
   status: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 }

utils/upload_flakiness_dashboard.sh

@@ -95,7 +95,7 @@ az storage blob upload --connection-string "${FLAKINESS_CONNECTION_STRING}" -c u
 
 UTC_DATE=$(cat <<EOF | node
   const date = new Date();
-  console.log([date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()].join('-'));
+  console.log(date.toISOString().substring(0, 10).replace(/-/g, ''));
 EOF
 )