feat(reporters): Add error position to JSON Report (#9151)

src/test/reporters/base.ts

@@ -26,6 +26,7 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul
 const stackUtils = new StackUtils();
 
 type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
+export type PositionInFile = { column: number; line: number };
 const kOutputSymbol = Symbol('output');
 
 export class BaseReporter implements Reporter  {
@@ -240,28 +241,24 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in
 
 export function formatError(error: TestError, file?: string) {
   const stack = error.stack;
-  const tokens = [];
+  const tokens = [''];
   if (stack) {
+    const { message, stackLines, position } = prepareErrorStack(
+        stack,
+        file
+    );
+    tokens.push(message);
+
+    const codeFrame = generateCodeFrame(file, position);
+    if (codeFrame) {
       tokens.push('');
-    const lines = stack.split('\n');
-    let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
-    if (firstStackLine === -1)
-      firstStackLine = lines.length;
-    tokens.push(lines.slice(0, firstStackLine).join('\n'));
-    const stackLines = lines.slice(firstStackLine);
-    const position = file ? positionInFile(stackLines, file) : null;
-    if (position) {
-      const source = fs.readFileSync(file!, 'utf8');
-      tokens.push('');
-      tokens.push(codeFrameColumns(source, { start: position }, { highlightCode: colors.enabled }));
+      tokens.push(codeFrame);
     }
     tokens.push('');
     tokens.push(colors.dim(stackLines.join('\n')));
   } else if (error.message) {
-    tokens.push('');
     tokens.push(error.message);
-  } else {
-    tokens.push('');
+  } else if (error.value) {
     tokens.push(error.value);
   }
   return tokens.join('\n');
@@ -277,6 +274,38 @@ function indent(lines: string, tab: string) {
   return lines.replace(/^(?=.+$)/gm, tab);
 }
 
+function generateCodeFrame(file?: string, position?: PositionInFile): string | undefined {
+  if (!position || !file)
+    return;
+
+  const source = fs.readFileSync(file!, 'utf8');
+  const codeFrame = codeFrameColumns(
+      source,
+      { start: position },
+      { highlightCode: colors.enabled }
+  );
+
+  return codeFrame;
+}
+
+export function prepareErrorStack(stack: string, file?: string): {
+  message: string;
+  stackLines: string[];
+  position?: PositionInFile;
+} {
+  const lines = stack.split('\n');
+  let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
+  if (firstStackLine === -1) firstStackLine = lines.length;
+  const message = lines.slice(0, firstStackLine).join('\n');
+  const stackLines = lines.slice(firstStackLine);
+  const position = file ? positionInFile(stackLines, file) : undefined;
+  return {
+    message,
+    stackLines,
+    position,
+  };
+}
+
 function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined {
   // Stack will have /private/var/folders instead of /var/folders on Mac.
   file = fs.realpathSync(file);

src/test/reporters/json.ts

@@ -17,6 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, TestStatus, Location, Reporter } from '../../../types/testReporter';
+import { PositionInFile, prepareErrorStack } from './base';
 
 export interface JSONReport {
   config: Omit<FullConfig, 'projects'> & {
@@ -65,11 +66,17 @@ export interface JSONReportTestResult {
   status: TestStatus | undefined;
   duration: number;
   error: TestError | undefined;
-  stdout: JSONReportSTDIOEntry[],
-  stderr: JSONReportSTDIOEntry[],
+  stdout: JSONReportSTDIOEntry[];
+  stderr: JSONReportSTDIOEntry[];
   retry: number;
   steps?: JSONReportTestStep[];
-  attachments: { name: string, path?: string, body?: string, contentType: string }[];
+  attachments: {
+    name: string;
+    path?: string;
+    body?: string;
+    contentType: string;
+  }[];
+  errorLocation?: PositionInFile
 }
 export interface JSONReportTestStep {
   title: string;
@@ -216,14 +223,14 @@ class JSONReporter implements Reporter {
       annotations: test.annotations,
       expectedStatus: test.expectedStatus,
       projectName: test.titlePath()[1],
-      results: test.results.map(r => this._serializeTestResult(r)),
+      results: test.results.map(r => this._serializeTestResult(r, test.location.file)),
       status: test.outcome(),
     };
   }
 
-  private _serializeTestResult(result: TestResult): JSONReportTestResult {
+  private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult {
     const steps = result.steps.filter(s => s.category === 'test.step');
-    return {
+    const jsonResult: JSONReportTestResult = {
       workerIndex: result.workerIndex,
       status: result.status,
       duration: result.duration,
@@ -239,6 +246,15 @@ class JSONReporter implements Reporter {
         body: a.body?.toString('base64')
       })),
     };
+    if (result.error?.stack) {
+      const { position } = prepareErrorStack(
+          result.error.stack,
+          file
+      );
+      if (position)
+        jsonResult.errorLocation = position;
+    }
+    return jsonResult;
   }
 
   private _serializeTestStep(step: TestStep): JSONReportTestStep {

tests/playwright-test/json-reporter.spec.ts

@@ -183,3 +183,20 @@ test('should have relative always-posix paths', async ({ runInlineTest }) => {
   expect(result.report.suites[0].specs[0].line).toBe(6);
   expect(result.report.suites[0].specs[0].column).toBe(7);
 });
+
+test('should have error position in results', async ({
+  runInlineTest,
+}) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('math works!', async ({}) => {
+        expect(1 + 1).toBe(3);
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].file).toBe('a.test.js');
+  expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.line).toBe(7);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].errorLocation.column).toBe(23);
+});