chore: run html reporter tests with merged report (#23626)

packages/playwright-test/src/isomorphic/teleReceiver.ts

@@ -125,12 +125,14 @@ export class TeleReporterReceiver {
   private _tests = new Map<string, TeleTestCase>();
   private _rootDir!: string;
   private _clearPreviousResultsWhenTestBegins: boolean = false;
+  private _reuseTestCases: boolean;
   private _reportConfig: MergeReporterConfig | undefined;
 
-  constructor(pathSeparator: string, reporter: Reporter, reportConfig?: MergeReporterConfig) {
+  constructor(pathSeparator: string, reporter: Reporter, reuseTestCases: boolean, reportConfig?: MergeReporterConfig) {
     this._rootSuite = new TeleSuite('', 'root');
     this._pathSeparator = pathSeparator;
     this._reporter = reporter;
+    this._reuseTestCases = reuseTestCases;
     this._reportConfig = reportConfig;
   }
 
@@ -246,7 +248,7 @@ export class TeleReporterReceiver {
       location: this._absoluteLocation(payload.location),
       parent: parentStep,
       startTime: new Date(payload.startTime),
-      duration: 0,
+      duration: -1,
       steps: [],
     };
     if (parentStep)
@@ -350,7 +352,7 @@ export class TeleReporterReceiver {
 
   private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) {
     for (const jsonTest of jsonTests) {
-      let targetTest = parent.tests.find(s => s.title === jsonTest.title);
+      let targetTest = this._reuseTestCases ? parent.tests.find(s => s.title === jsonTest.title) : undefined;
       if (!targetTest) {
         targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location));
         targetTest.parent = parent;

packages/playwright-test/src/reporters/merge.ts

@@ -33,7 +33,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
     patchAttachmentPaths(events, dir);
 
   const reporters = await createReporters(config, 'merge', reporterDescriptions);
-  const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters), config.config);
+  const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters), false, config.config);
 
   for (const event of events)
     await receiver.dispatch(event);

packages/playwright-test/src/reporters/teleEmitter.ts

@@ -26,11 +26,11 @@ import { serializeRegexPatterns } from '../isomorphic/teleReceiver';
 export class TeleReporterEmitter implements Reporter {
   private _messageSink: (message: JsonEvent) => void;
   private _rootDir!: string;
-  private _receiverIsInBrowser: boolean;
+  private _skipBuffers: boolean;
 
-  constructor(messageSink: (message: JsonEvent) => void, receiverIsInBrowser: boolean) {
+  constructor(messageSink: (message: JsonEvent) => void, skipBuffers: boolean) {
     this._messageSink = messageSink;
-    this._receiverIsInBrowser = receiverIsInBrowser;
+    this._skipBuffers = skipBuffers;
   }
 
   onBegin(config: FullConfig, suite: Suite) {
@@ -206,7 +206,7 @@ export class TeleReporterEmitter implements Reporter {
       return {
         ...a,
         // There is no Buffer in the browser, so there is no point in sending the data there.
-        base64: (a.body && !this._receiverIsInBrowser) ? a.body.toString('base64') : undefined,
+        base64: (a.body && !this._skipBuffers) ? a.body.toString('base64') : undefined,
       };
     });
   }

packages/playwright-test/src/runner/reporters.ts

@@ -64,7 +64,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
     const prints = r.printsToStdio ? r.printsToStdio() : true;
     return prints;
   });
-  if (reporters.length && !someReporterPrintsToStdio && mode !== 'merge') {
+  if (reporters.length && !someReporterPrintsToStdio) {
     // Add a line/dot/list-mode reporter for convenience.
     // Important to put it first, jsut in case some other reporter stalls onEnd.
     if (mode === 'list')

packages/trace-viewer/src/ui/uiModeView.tsx

@@ -640,7 +640,7 @@ const refreshRootSuite = (eraseResults: boolean): Promise<void> => {
       loadErrors.push(error);
       throttleUpdateRootSuite(config, rootSuite, loadErrors, progress);
     },
-  });
+  }, true);
   receiver._setClearPreviousResultsWhenTestBegins();
   return sendMessage('list', {});
 };

tests/playwright-test/playwright-test-fixtures.ts

@@ -85,7 +85,44 @@ export async function writeFiles(testInfo: TestInfo, files: Files, initial: bool
 
 export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-test/cli.js');
 
-async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): Promise<RunResult> {
+const mergeReports = async (childProcess: CommonFixtures['childProcess'], cwd: string, env: NodeJS.ProcessEnv = {}, reporter: string | undefined, configFile: string | undefined) => {
+  const command = ['node', cliEntrypoint, 'merge-reports'];
+  if (reporter)
+    command.push('--reporter', reporter);
+  if (configFile)
+    command.push('--config', configFile);
+  command.push('blob-report');
+  const testProcess = childProcess({
+    command,
+    env: cleanEnv({
+      PW_TEST_DEBUG_REPORTERS: '1',
+      PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1',
+      PWTEST_TTY_WIDTH: '80',
+      ...env
+    }),
+    cwd,
+  });
+  const { exitCode } = await testProcess.exited;
+  return { exitCode, output: testProcess.output.toString() };
+};
+
+const configFile = (baseDir: string, files: Files): string | undefined => {
+  for (const [name, content] of  Object.entries(files)) {
+    if (name.includes('playwright.config')) {
+      if (content.includes('reporter:'))
+        return path.resolve(baseDir, name);
+    }
+  }
+  return undefined;
+};
+
+async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions, files: Files, useIntermediateMergeReport: boolean): Promise<RunResult> {
+  let reporter;
+  if (useIntermediateMergeReport) {
+    reporter = params.reporter;
+    params.reporter = 'blob';
+  }
+
   const paramList: string[] = [];
   for (const key of Object.keys(params)) {
     for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) {
@@ -110,6 +147,12 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
     ...env,
   }, options.sendSIGINTAfter);
 
+  if (useIntermediateMergeReport) {
+    const mergeResult = await mergeReports(childProcess, cwd, env, reporter, configFile(baseDir, files));
+    expect(mergeResult.exitCode).toBe(0);
+    output = mergeResult.output;
+  }
+
   const summary = (re: RegExp) => {
     let result = 0;
     let match = re.exec(output);
@@ -247,6 +290,7 @@ type Fixtures = {
   runListFiles: (files: Files) => Promise<{ output: string, exitCode: number }>;
   runWatchTest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<TestChildProcess>;
   runTSC: (files: Files) => Promise<TSCResult>;
+  useIntermediateMergeReport: boolean;
   nodeVersion: { major: number, minor: number, patch: number };
 };
 
@@ -265,11 +309,11 @@ export const test = base
         });
       },
 
-      runInlineTest: async ({ childProcess }, use, testInfo: TestInfo) => {
+      runInlineTest: async ({ childProcess, useIntermediateMergeReport }, use, testInfo: TestInfo) => {
         const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
         await use(async (files: Files, params: Params = {}, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
           const baseDir = await writeFiles(testInfo, files, true);
-          return await runPlaywrightTest(childProcess, baseDir, params, { ...env, PWTEST_CACHE_DIR: cacheDir }, options);
+          return await runPlaywrightTest(childProcess, baseDir, params, { ...env, PWTEST_CACHE_DIR: cacheDir }, options, files, useIntermediateMergeReport);
         });
         await removeFolderAsync(cacheDir);
       },
@@ -314,6 +358,10 @@ export const test = base
         const [major, minor, patch] = process.versions.node.split('.');
         await use({ major: +major, minor: +minor, patch: +patch });
       },
+
+      useIntermediateMergeReport: async ({}, use) => {
+        await use(process.env.PWTEST_INTERMEDIATE_BLOB_REPORT === '1');
+      },
     });
 
 const TSCONFIG = {

tests/playwright-test/reporter-blob.spec.ts

@@ -64,6 +64,8 @@ test.slow(!!process.env.CI);
 // Slow tests are 90s.
 const expect = baseExpect.configure({ timeout: process.env.CI ? 75000 : 25000 });
 
+test.describe.configure({ mode: 'parallel' });
+
 const echoReporterJs = `
 class EchoReporter {
   onBegin(config, suite) {