feat(html): auto-open report (#8908)

src/test/reporters/html.ts

@@ -14,13 +14,25 @@
  * limitations under the License.
  */
 
+import colors from 'colors/safe';
 import fs from 'fs';
+import open from 'open';
 import path from 'path';
 import { FullConfig, Suite } from '../../../types/testReporter';
-import { calculateSha1 } from '../../utils/utils';
+import { HttpServer } from '../../utils/httpServer';
+import { calculateSha1, removeFolders } from '../../utils/utils';
 import { toPosixPath } from '../reporters/json';
 import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw';
 
+export type Stats = {
+  total: number;
+  expected: number;
+  unexpected: number;
+  flaky: number;
+  skipped: number;
+  ok: boolean;
+};
+
 export type Location = {
   file: string;
   line: number;
@@ -30,7 +42,7 @@ export type Location = {
 export type ProjectTreeItem = {
   name: string;
   suites: SuiteTreeItem[];
-  failedTests: number;
+  stats: Stats;
 };
 
 export type SuiteTreeItem = {
@@ -39,7 +51,7 @@ export type SuiteTreeItem = {
   duration: number;
   suites: SuiteTreeItem[];
   tests: TestTreeItem[];
-  failedTests: number;
+  stats: Stats;
 };
 
 export type TestTreeItem = {
@@ -49,6 +61,7 @@ export type TestTreeItem = {
   location: Location;
   duration: number;
   outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
+  ok: boolean;
 };
 
 export type TestFile = {
@@ -99,7 +112,26 @@ class HtmlReporter {
       return report;
     });
     const reportFolder = path.resolve(process.cwd(), process.env[`PLAYWRIGHT_HTML_REPORT`] || 'playwright-report');
+    await removeFolders([reportFolder]);
     new HtmlBuilder(reports, reportFolder, this.config.rootDir);
+
+    if (!process.env.CI) {
+      const server = new HttpServer();
+      server.routePrefix('/', (request, response) => {
+        let relativePath = request.url!;
+        if (relativePath === '/')
+          relativePath = '/index.html';
+        const absolutePath = path.join(reportFolder, ...relativePath.split('/'));
+        return server.serveFile(response, absolutePath);
+      });
+      const url = await server.start();
+      console.log('');
+      console.log(colors.cyan(`  Serving HTML report at ${url}. Press Ctrl+C to quit.`));
+      console.log('');
+      open(url);
+      process.on('SIGINT', () => process.exit(0));
+      await new Promise(() => {});
+    }
   }
 }
 
@@ -135,7 +167,7 @@ class HtmlBuilder {
       projects.push({
         name: projectJson.project.name,
         suites,
-        failedTests: suites.reduce((a, s) => a + s.failedTests, 0)
+        stats: suites.reduce((a, s) => addStats(a, s.stats), emptyStats()),
       });
     }
     fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
@@ -154,11 +186,22 @@ class HtmlBuilder {
     const suites = suite.suites.map(s => this._createSuiteTreeItem(s, fileId, testCollector));
     const tests = suite.tests.map(t => this._createTestTreeItem(t, fileId));
     testCollector.push(...suite.tests);
+    const stats = suites.reduce<Stats>((a, s) => addStats(a, s.stats), emptyStats());
+    for (const test of tests) {
+      if (test.outcome === 'expected')
+        ++stats.expected;
+      if (test.outcome === 'unexpected')
+        ++stats.unexpected;
+      if (test.outcome === 'flaky')
+        ++stats.flaky;
+      ++stats.total;
+    }
+    stats.ok = stats.unexpected + stats.flaky === 0;
     return {
       title: suite.title,
       location: this._relativeLocation(suite.location),
       duration: suites.reduce((a, s) => a + s.duration, 0) + tests.reduce((a, t) => a + t.duration, 0),
-      failedTests: suites.reduce((a, s) => a + s.failedTests, 0) + tests.reduce((a, t) => t.outcome === 'unexpected' || t.outcome === 'flaky' ? a + 1 : a, 0),
+      stats,
       suites,
       tests
     };
@@ -173,7 +216,8 @@ class HtmlBuilder {
       location: this._relativeLocation(test.location),
       title: test.title,
       duration,
-      outcome: test.outcome
+      outcome: test.outcome,
+      ok: test.ok
     };
   }
 
@@ -183,7 +227,7 @@ class HtmlBuilder {
       startTime: result.startTime,
       retry: result.retry,
       steps: result.steps.map(s => this._createTestStep(s)),
-      error: result.error?.message,
+      error: result.error,
       status: result.status,
     };
   }
@@ -195,7 +239,7 @@ class HtmlBuilder {
       duration: step.duration,
       steps: step.steps.map(s => this._createTestStep(s)),
       log: step.log,
-      error: step.error?.message
+      error: step.error
     };
   }
 
@@ -210,4 +254,25 @@ class HtmlBuilder {
   }
 }
 
+const emptyStats = (): Stats => {
+  return {
+    total: 0,
+    expected: 0,
+    unexpected: 0,
+    flaky: 0,
+    skipped: 0,
+    ok: true
+  };
+};
+
+const addStats = (stats: Stats, delta: Stats): Stats => {
+  stats.total += delta.total;
+  stats.skipped += delta.skipped;
+  stats.expected += delta.expected;
+  stats.unexpected += delta.unexpected;
+  stats.flaky += delta.flaky;
+  stats.ok = stats.ok && delta.ok;
+  return stats;
+};
+
 export default HtmlReporter;

src/test/reporters/raw.ts

@@ -17,13 +17,14 @@
 import fs from 'fs';
 import path from 'path';
 import { FullProject } from '../../../types/test';
-import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
+import { FullConfig, Location, Suite, TestCase, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
 import { assert, calculateSha1 } from '../../utils/utils';
 import { sanitizeForFilePath } from '../util';
+import { formatResultFailure } from './base';
 import { serializePatterns } from './json';
 
 export type JsonLocation = Location;
-export type JsonError = TestError;
+export type JsonError = string;
 export type JsonStackFrame = { file: string, line: number, column: number };
 
 export type JsonReport = {
@@ -187,7 +188,7 @@ class RawReporter {
       startTime: result.startTime.toISOString(),
       duration: result.duration,
       status: result.status,
-      error: result.error,
+      error: formatResultFailure(test, result, '').join('').trim(),
       attachments: this._createAttachments(result),
       steps: this._serializeSteps(test, result.steps)
     };
@@ -200,7 +201,7 @@ class RawReporter {
         category: step.category,
         startTime: step.startTime.toISOString(),
         duration: step.duration,
-        error: step.error,
+        error: step.error?.message,
         steps: this._serializeSteps(test, step.steps),
         log: step.data.log || undefined,
       };

src/web/components/treeItem.tsx

@@ -27,7 +27,7 @@ export const TreeItem: React.FunctionComponent<{
   const [expanded, setExpanded] = React.useState(expandByDefault || false);
   const className = selected ? 'tree-item-title selected' : 'tree-item-title';
   return <div style={{ display: 'flex', flexDirection: 'column', width: '100%' }}>
-    <div className={className} style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', whiteSpace: 'nowrap', paddingLeft: depth * 20 + 4 }} onClick={() => { onClick?.(); setExpanded(!expanded); }} >
+    <div className={className} style={{ display: 'flex', flexDirection: 'row', alignItems: 'center', whiteSpace: 'nowrap', paddingLeft: depth * 16 + 4 }} onClick={() => { onClick?.(); setExpanded(!expanded); }} >
       <div className={'codicon codicon-' + (expanded ? 'chevron-down' : 'chevron-right')}
         style={{ cursor: 'pointer', color: 'var(--color)', visibility: loadChildren ? 'visible' : 'hidden' }} />
       {title}

src/web/htmlReport/htmlReport.css

@@ -51,8 +51,9 @@
   color: white;
   padding: 5px;
   overflow: auto;
-  margin: 20px 0;
+  margin: 20px;
   flex: none;
+  box-shadow: var(--box-shadow);
 }
 
 .status-icon {
@@ -164,9 +165,6 @@
   display: flex;
   align-items: center;
   padding: 0 10px 10px;
-  color: var(--blue);
-  text-decoration: underline;
-  cursor: pointer;
 }
 
 .test-details-column {
@@ -183,3 +181,24 @@
   overflow: hidden;
   text-overflow: ellipsis;
 }
+
+.stats {
+  background-color: gray;
+  border-radius: 2px;
+  min-width: 10px;
+  color: white;
+  margin: 0 2px;
+  padding: 0 2px;
+}
+
+.stats.expected {
+  background-color: var(--green);
+}
+
+.stats.unexpected {
+  background-color: var(--red);
+}
+
+.stats.flaky {
+  background-color: var(--yellow);
+}

src/web/htmlReport/htmlReport.tsx

@@ -21,7 +21,7 @@ import { SplitView } from '../components/splitView';
 import { TreeItem } from '../components/treeItem';
 import { TabbedPane } from '../traceViewer/ui/tabbedPane';
 import { msToString } from '../uiUtils';
-import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile } from '../../test/reporters/html';
+import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile, Stats } from '../../test/reporters/html';
 
 type Filter = 'Failing' | 'All';
 
@@ -60,7 +60,7 @@ export const Report: React.FC = () => {
             }}>{item}</div>;
           })
         }</div>
-        {!fetchError && filter === 'All' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId}></ProjectTreeItemView>)}
+        {!fetchError && filter === 'All' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={false}></ProjectTreeItemView>)}
         {!fetchError && filter === 'Failing' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={true}></ProjectTreeItemView>)}
       </div>
     </SplitView>
@@ -71,49 +71,52 @@ const ProjectTreeItemView: React.FC<{
   project: ProjectTreeItem;
   testId?: TestId,
   setTestId: (id: TestId) => void;
-  failingOnly?: boolean;
+  failingOnly: boolean;
 }> = ({ project, testId, setTestId, failingOnly }) => {
+  const hasChildren = !(failingOnly && project.stats.ok);
   return <TreeItem title={<div className='hbox'>
-    {statusIconForFailedTests(project.failedTests)}<div className='tree-text'>{project.name || 'Project'}</div>
+    <div className='tree-text'>{project.name || 'Project'}</div>
+    <div style={{ flex: 'auto' }}></div>
+    <StatsView stats={project.stats}></StatsView>
   </div>
-  } loadChildren={() => {
+  } loadChildren={hasChildren ? () => {
-    return project.suites.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={1} showFileName={true}></SuiteTreeItemView>) || [];
+    return project.suites.filter(s => !(failingOnly && s.stats.ok)).map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={1} showFileName={true} failingOnly={failingOnly}></SuiteTreeItemView>) || [];
-  }} depth={0} expandByDefault={true}></TreeItem>;
+  } : undefined} depth={0} expandByDefault={true}></TreeItem>;
 };
 
 const SuiteTreeItemView: React.FC<{
   suite: SuiteTreeItem,
   testId?: TestId,
   setTestId: (id: TestId) => void;
+  failingOnly: boolean;
   depth: number,
   showFileName: boolean,
-}> = ({ suite, testId, setTestId, showFileName, depth }) => {
+}> = ({ suite, testId, setTestId, showFileName, failingOnly, depth }) => {
   const location = renderLocation(suite.location, showFileName);
   return <TreeItem title={<div className='hbox'>
-    {statusIconForFailedTests(suite.failedTests)}<div className='tree-text'>{suite.title}</div>
+    <div className='tree-text' title={suite.title}>{suite.title}</div>
+    <div style={{ flex: 'auto' }}></div>
+    <StatsView stats={suite.stats}></StatsView>
     {!!suite.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
   </div>
   } loadChildren={() => {
-    const suiteChildren = suite.suites.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={depth + 1} showFileName={false}></SuiteTreeItemView>) || [];
+    const suiteChildren = suite.suites.filter(s => !(failingOnly && s.stats.ok)).map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={depth + 1} showFileName={false} failingOnly={failingOnly}></SuiteTreeItemView>) || [];
     const suiteCount = suite.suites.length;
-    const testChildren = suite.tests.map((t, i) => <TestTreeItemView key={i + suiteCount} test={t} setTestId={setTestId} testId={testId} showFileName={false} depth={depth + 1}></TestTreeItemView>) || [];
+    const testChildren = suite.tests.filter(t => !(failingOnly && t.ok)).map((t, i) => <TestTreeItemView key={i + suiteCount} test={t} setTestId={setTestId} testId={testId} depth={depth + 1}></TestTreeItemView>) || [];
     return [...suiteChildren, ...testChildren];
   }} depth={depth}></TreeItem>;
 };
 
 const TestTreeItemView: React.FC<{
   test: TestTreeItem,
-  showFileName: boolean,
   testId?: TestId,
   setTestId: (id: TestId) => void;
   depth: number,
-}> = ({ test, testId, setTestId, showFileName, depth }) => {
+}> = ({ test, testId, setTestId, depth }) => {
-  const fileName = test.location.file;
-  const name = fileName.substring(fileName.lastIndexOf('/') + 1);
   return <TreeItem title={<div className='hbox'>
-    {statusIcon(test.outcome)}<div className='tree-text'>{test.title}</div>
+    {statusIcon(test.outcome)}<div className='tree-text' title={test.title}>{test.title}</div>
-    {showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{name}:{test.location.line}</div>}
+    <div style={{ flex: 'auto' }}></div>
-    {!showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{msToString(test.duration)}</div>}
+    {<div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{msToString(test.duration)}</div>}
   </div>
   } selected={test.testId === testId?.testId} depth={depth} onClick={() => setTestId({ testId: test.testId, fileId: test.fileId })}></TreeItem>;
 };
@@ -189,9 +192,16 @@ const StepTreeItem: React.FC<{
   } : undefined} depth={depth}></TreeItem>;
 };
 
-function statusIconForFailedTests(failedTests: number) {
+const StatsView: React.FC<{
-  return failedTests ? statusIcon('failed') : statusIcon('passed');
+  stats: Stats
-}
+}> = ({ stats }) => {
+  return <div className='hbox' style={{flex: 'none'}}>
+    {!!stats.expected && <div className='stats expected' title='Passed'>{stats.expected}</div>}
+    {!!stats.unexpected && <div className='stats unexpected' title='Failed'>{stats.unexpected}</div>}
+    {!!stats.flaky && <div className='stats flaky' title='Flaky'>{stats.flaky}</div>}
+    {!!stats.skipped && <div className='stats skipped' title='Skipped'>{stats.skipped}</div>}
+  </div>;
+};
 
 function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element {
   switch (status) {