chore: warn users on clashing test output and html reporter folders (#14964)

packages/playwright-test/src/reporters/html.ts

@@ -131,9 +131,12 @@ class HtmlReporter implements Reporter {
   private config!: FullConfigInternal;
   private suite!: Suite;
   private _options: HtmlReporterOptions;
+  private _outputFolder!: string;
+  private _open: string | undefined;
 
   constructor(options: HtmlReporterOptions = {}) {
     this._options = options;
+    console.log('HTML REPORTER OPTIONS', options);
   }
 
   printsToStdio() {
@@ -142,6 +145,25 @@ class HtmlReporter implements Reporter {
 
   onBegin(config: FullConfig, suite: Suite) {
     this.config = config as FullConfigInternal;
+    const { outputFolder, open } = this._resolveOptions();
+    this._outputFolder = outputFolder;
+    this._open = open;
+    const reportedWarnings = new Set<string>();
+    for (const project of config.projects) {
+      if (outputFolder.startsWith(project.outputDir) || project.outputDir.startsWith(outputFolder)) {
+        const key = outputFolder + '|' + project.outputDir;
+        if (reportedWarnings.has(key))
+          continue;
+        reportedWarnings.add(key);
+        console.log(colors.red(`Configuration Error: HTML reporter output folder clashes with the tests output folder:`));
+        console.log(`
+    html reporter folder: ${colors.bold(outputFolder)}
+    test results folder: ${colors.bold(project.outputDir)}`);
+        console.log('');
+        console.log(`HTML reporter will clear its output directory prior to being generated, which will lead to the artifact loss.
+`);
+      }
+    }
     this.suite = suite;
   }
 
@@ -156,26 +178,25 @@ class HtmlReporter implements Reporter {
   }
 
   async onEnd() {
-    const { open, outputFolder } = this._resolveOptions();
     const projectSuites = this.suite.suites;
     const reports = projectSuites.map(suite => {
       const rawReporter = new RawReporter();
       const report = rawReporter.generateProjectReport(this.config, suite);
       return report;
     });
-    await removeFolders([outputFolder]);
-    const builder = new HtmlBuilder(outputFolder);
+    await removeFolders([this._outputFolder]);
+    const builder = new HtmlBuilder(this._outputFolder);
     const { ok, singleTestId } = await builder.build(this.config.metadata, reports);
 
     if (process.env.CI)
       return;
 
 
-    const shouldOpen = open === 'always' || (!ok && open === 'on-failure');
+    const shouldOpen = this._open === 'always' || (!ok && this._open === 'on-failure');
     if (shouldOpen) {
-      await showHTMLReport(outputFolder, singleTestId);
+      await showHTMLReport(this._outputFolder, singleTestId);
     } else {
-      const relativeReportPath = outputFolder === standaloneDefaultFolder() ? '' : ' ' + path.relative(process.cwd(), outputFolder);
+      const relativeReportPath = this._outputFolder === standaloneDefaultFolder() ? '' : ' ' + path.relative(process.cwd(), this._outputFolder);
       console.log('');
       console.log('To open last HTML report run:');
       console.log(colors.cyan(`

tests/config/commonFixtures.ts

@@ -32,7 +32,7 @@ export class TestChildProcess {
   process: ChildProcess;
   output = '';
   onOutput?: () => void;
-  exited: Promise<{ exitCode: number | null, signal: string | null }>;
+  exited: Promise<{ exitCode: number, signal: string | null }>;
   exitCode: Promise<number>;
 
   private _outputCallbacks = new Set<() => void>();

tests/playwright-test/playwright-test-fixtures.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type { JSONReport, JSONReportSuite } from '@playwright/test/reporter';
+import type { JSONReport, JSONReportSuite, JSONReportTestResult } from '@playwright/test/reporter';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
@@ -102,7 +102,7 @@ async function writeFiles(testInfo: TestInfo, files: Files) {
 const cliEntrypoint = path.join(__dirname, '../../packages/playwright-core/cli.js');
 
 async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: Env, options: RunOptions): Promise<RunResult> {
-  const paramList = [];
+  const paramList: string[] = [];
   for (const key of Object.keys(params)) {
     for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) {
       const k = key.startsWith('-') ? key : '--' + key;
@@ -114,8 +114,9 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   const args = ['node', cliEntrypoint, 'test'];
   if (!options.usesCustomOutputDir)
     args.push('--output=' + outputDir);
+  if (!options.usesCustomReporters)
+    args.push('--reporter=dot,json');
   args.push(
-      '--reporter=dot,json',
       '--workers=2',
       ...paramList
   );
@@ -181,7 +182,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
     testProcess.output += '\n' + e.toString();
   }
 
-  const results = [];
+  const results: JSONReportTestResult[] = [];
   function visitSuites(suites?: JSONReportSuite[]) {
     if (!suites)
       return;
@@ -211,6 +212,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
 type RunOptions = {
   sendSIGINTAfter?: number;
   usesCustomOutputDir?: boolean;
+  usesCustomReporters?: boolean;
   additionalArgs?: string[];
   cwd?: string,
 };

tests/playwright-test/reporter-html.spec.ts

@@ -16,7 +16,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { test as baseTest, expect, createImage } from './playwright-test-fixtures';
+import { test as baseTest, expect, createImage, stripAnsi } from './playwright-test-fixtures';
 import type { HttpServer } from '../../packages/playwright-core/lib/utils/httpServer';
 import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html';
 import { spawnAsync } from 'playwright-core/lib/utils/spawnAsync';
@@ -860,3 +860,22 @@ test.describe('gitCommitInfo plugin', () => {
     await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible();
   });
 });
+
+test('should report clashing folders', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [['html', { outputFolder: 'test-results/html-report' }]]
+      }
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({}) => {
+      });
+    `,
+  },  {}, {}, { usesCustomReporters: true });
+  expect(result.exitCode).toBe(0);
+  const output = stripAnsi(result.output);
+  expect(output).toContain('Configuration Error');
+  expect(output).toContain('html-report');
+});