feat(html): open show html report when there are failures (#9526)

packages/playwright-core/src/cli/cli.ts

@@ -222,7 +222,6 @@ program
     }).addHelpText('afterAll', `
 Examples:
 
-  $ show-trace trace/directory
   $ show-trace https://example.com/trace.zip`);
 
 if (!process.env.PW_CLI_TARGET_LANG) {
@@ -235,6 +234,7 @@ if (!process.env.PW_CLI_TARGET_LANG) {
 
   if (playwrightTestPackagePath) {
     require(playwrightTestPackagePath).addTestCommand(program);
+    require(playwrightTestPackagePath).addShowReportCommand(program);
   } else {
     const command = program.command('test').allowUnknownOption(true);
     command.description('Run tests with Playwright Test. Available in @playwright/test package.');

packages/playwright-core/src/web/htmlReport/htmlReport.tsx

@@ -34,19 +34,22 @@ export const Report: React.FC = () => {
   const [report, setReport] = React.useState<ProjectTreeItem[]>([]);
   const [fetchError, setFetchError] = React.useState<string | undefined>();
   const [testId, setTestId] = React.useState<TestId | undefined>();
+  const [filter, setFilter] = React.useState<Filter>('failing');
 
   React.useEffect(() => {
     (async () => {
       try {
         const result = await fetch('data/projects.json', { cache: 'no-cache' });
         const json = (await result.json()) as ProjectTreeItem[];
+        const hasFailures = !!json.find(p => !p.stats.ok);
+        if (!hasFailures)
+          setFilter('all');
         setReport(json);
       } catch (e) {
         setFetchError(e.message);
       }
     })();
   }, []);
-  const [filter, setFilter] = React.useState<Filter>('failing');
 
   return <div className='hbox columns'>
     <SplitView sidebarSize={300} orientation='horizontal' sidebarIsFirst={true}>
@@ -57,7 +60,7 @@ export const Report: React.FC = () => {
           <div key='failing' title='Failing tests' className={'tab-element' + ('failing' === filter ? ' selected' : '')} onClick={() => setFilter('failing')}>Failing</div>
         </div>
         {!fetchError && filter === 'all' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={false}></ProjectTreeItemView>)}
-        {!fetchError && filter === 'failing' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={true}></ProjectTreeItemView>)}
+        {!fetchError && filter === 'failing' && report?.filter(p => !p.stats.ok).map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={true}></ProjectTreeItemView>)}
       </div>
     </SplitView>
   </div>;

packages/playwright-test/src/cli.ts

@@ -24,6 +24,7 @@ import { Runner, builtInReporters, BuiltInReporter } from './runner';
 import { stopProfiling, startProfiling } from './profiler';
 import { FilePatternFilter } from './util';
 import { Loader } from './loader';
+import { showHTMLReport } from './reporters/html';
 
 const defaultTimeout = 30000;
 const defaultReporter: BuiltInReporter = process.env.CI ? 'dot' : 'list';
@@ -77,9 +78,22 @@ Arguments [test-filter...]:
   Pass arguments to filter test files. Each argument is treated as a regular expression.
 
 Examples:
-  $ test my.spec.ts
-  $ test --headed
-  $ test --browser=webkit`);
+  $ npx playwright test my.spec.ts
+  $ npx playwright test --headed
+  $ npx playwright test --browser=webkit`);
+}
+
+export function addShowReportCommand(program: Command) {
+  const command = program.command('show-report [report]');
+  command.description('show HTML report');
+  command.action(report => showHTMLReport(report));
+  command.addHelpText('afterAll', `
+Arguments [report]:
+  When specified, opens given report, otherwise opens last generated report.
+
+Examples:
+  $ npx playwright show-report
+  $ npx playwright show-report playwright-report`);
 }
 
 async function createLoader(opts: { [key: string]: any }): Promise<Loader> {

packages/playwright-test/src/reporters/html.ts

@@ -23,6 +23,7 @@ import { HttpServer } from 'playwright-core/src/utils/httpServer';
 import { calculateSha1, removeFolders } from 'playwright-core/src/utils/utils';
 import { toPosixPath } from './json';
 import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep, JsonAttachment } from './raw';
+import assert from 'assert';
 
 export type Stats = {
   total: number;
@@ -114,40 +115,68 @@ class HtmlReporter {
       const report = rawReporter.generateProjectReport(this.config, suite);
       return report;
     });
-    const reportFolder = path.resolve(process.cwd(), process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report');
+    const reportFolder = htmlReportFolder();
     await removeFolders([reportFolder]);
-    new HtmlBuilder(reports, reportFolder, this.config.rootDir);
+    const builder = new HtmlBuilder(reportFolder, this.config.rootDir);
+    const stats = builder.build(reports);
 
-    if (!process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) {
-      const server = new HttpServer();
-      server.routePrefix('/', (request, response) => {
-        let relativePath = new URL('http://localhost' + request.url).pathname;
-        if (relativePath === '/')
-          relativePath = '/index.html';
-        const absolutePath = path.join(reportFolder, ...relativePath.split('/'));
-        return server.serveFile(response, absolutePath);
-      });
-      const url = await server.start(9323);
+    if (!stats.ok && !process.env.CI && !process.env.PWTEST_SKIP_TEST_OUTPUT) {
+      showHTMLReport(reportFolder);
+    } else {
       console.log('');
-      console.log(colors.cyan(`  Serving HTML report at ${url}. Press Ctrl+C to quit.`));
       console.log('');
-      open(url);
-      process.on('SIGINT', () => process.exit(0));
-      await new Promise(() => {});
+      console.log('All tests passed. To open last HTML report run:');
+      console.log(colors.cyan(`
+  npx playwright show-report
+`));
+      console.log('');
     }
   }
 }
 
+export function htmlReportFolder(): string {
+  return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report');
+}
+
+export async function showHTMLReport(reportFolder: string | undefined) {
+  const folder = reportFolder || htmlReportFolder();
+  try {
+    assert(fs.statSync(folder).isDirectory());
+  } catch (e) {
+    console.log(colors.red(`No report found at "${folder}"`));
+    process.exit(1);
+    return;
+  }
+  const server = new HttpServer();
+  server.routePrefix('/', (request, response) => {
+    let relativePath = new URL('http://localhost' + request.url).pathname;
+    if (relativePath === '/')
+      relativePath = '/index.html';
+    const absolutePath = path.join(folder, ...relativePath.split('/'));
+    return server.serveFile(response, absolutePath);
+  });
+  const url = await server.start(9323);
+  console.log('');
+  console.log(colors.cyan(`  Serving HTML report at ${url}. Press Ctrl+C to quit.`));
+  console.log('');
+  open(url);
+  process.on('SIGINT', () => process.exit(0));
+  await new Promise(() => {});
+}
+
 class HtmlBuilder {
   private _reportFolder: string;
   private _tests = new Map<string, JsonTestCase>();
   private _rootDir: string;
   private _dataFolder: string;
 
-  constructor(rawReports: JsonReport[], outputDir: string, rootDir: string) {
+  constructor(outputDir: string, rootDir: string) {
     this._rootDir = rootDir;
     this._reportFolder = path.resolve(process.cwd(), outputDir);
     this._dataFolder = path.join(this._reportFolder, 'data');
+  }
+
+  build(rawReports: JsonReport[]): Stats {
     fs.mkdirSync(this._dataFolder, { recursive: true });
 
     // Copy app.
@@ -187,6 +216,7 @@ class HtmlBuilder {
       });
     }
     fs.writeFileSync(path.join(this._dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
+    return projects.reduce((a, p) => addStats(a, p.stats), emptyStats());
   }
 
   private _createTestCase(test: JsonTestCase): TestCase {