feat(test-runner): Add GitHub Actions reporter (#9191)

2025-01-02 16:30:39 +03:00 · 2021-10-04 14:02:56 +05:30 · 2021-10-04 14:02:56 +05:30 · be30f9f1c4
commit be30f9f1c4
parent 771dd83c16
12 changed files with 421 additions and 80 deletions
--- a/docs/src/test-advanced-js.md
+++ b/docs/src/test-advanced-js.md
@ -38,7 +38,7 @@ These options would be typically different between local development and CI oper
  - `'failures-only'` - only preserve output for failed tests.
 - `projects: Project[]` - Multiple [projects](#projects) configuration.
 - `quiet: boolean` - Whether to suppress stdout and stderr from the tests.
- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details.
+- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit' | 'github'` - The reporter to use. See [reporters](./test-reporters.md) for details.
 - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold.
 - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shard-tests-between-multiple-machines) information.
 - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
--- a/docs/src/test-reporters-js.md
+++ b/docs/src/test-reporters-js.md
@ -98,6 +98,34 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```

+### Reporter for GitHub Actions
+
+You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions.
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally
+  reporter: process.env.CI ? 'github' : 'list',
+};
+
+module.exports = config;
+```
+
+```js js-flavor=ts
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally
+  reporter: process.env.CI ? 'github' : 'list',
+};
+export default config;
+```
+
 ## Built-in reporters

 All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs.
--- a/package-lock.json
+++ b/package-lock.json
@ -62,6 +62,7 @@
        "@types/extract-zip": "^1.6.2",
        "@types/mime": "^2.0.3",
        "@types/minimatch": "^3.0.3",
+        "@types/ms": "^0.7.31",
        "@types/node": "^14.17.15",
        "@types/pixelmatch": "^5.2.1",
        "@types/pngjs": "^3.4.2",
@ -1485,6 +1486,12 @@
      "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==",
      "dev": true
    },
+    "node_modules/@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
+      "dev": true
+    },
    "node_modules/@types/node": {
      "version": "14.17.15",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.15.tgz",
@ -11936,6 +11943,12 @@
      "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==",
      "dev": true
    },
+    "@types/ms": {
+      "version": "0.7.31",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz",
+      "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
+      "dev": true
+    },
    "@types/node": {
      "version": "14.17.15",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.15.tgz",
--- a/package.json
+++ b/package.json
@ -91,6 +91,7 @@
    "@types/extract-zip": "^1.6.2",
    "@types/mime": "^2.0.3",
    "@types/minimatch": "^3.0.3",
+    "@types/ms": "^0.7.31",
    "@types/node": "^14.17.15",
    "@types/pixelmatch": "^5.2.1",
    "@types/pngjs": "^3.4.2",
@ -136,4 +137,4 @@
    "xml2js": "^0.4.23",
    "yaml": "^1.10.0"
  }
-}
+}
--- a/src/test/reporters/base.ts
+++ b/src/test/reporters/base.ts
@ -17,7 +17,6 @@
 import { codeFrameColumns } from '@babel/code-frame';
 import colors from 'colors/safe';
 import fs from 'fs';
-// @ts-ignore
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
@ -25,9 +24,35 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul

 const stackUtils = new StackUtils();

-type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
+export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
+export const kOutputSymbol = Symbol('output');
 export type PositionInFile = { column: number; line: number };
-const kOutputSymbol = Symbol('output');
+
+type Annotation = {
+  filePath: string;
+  title: string;
+  message: string;
+  position?: PositionInFile;
+};
+
+type FailureDetails = {
+  tokens: string[];
+  position?: PositionInFile;
+};
+
+type ErrorDetails = {
+  message: string;
+  position?: PositionInFile;
+};
+
+type TestSummary = {
+  skipped: number;
+  expected: number;
+  skippedWithError: TestCase[];
+  unexpected: TestCase[];
+  flaky: TestCase[];
+  failuresToPrint: TestCase[];
+};

 export class BaseReporter implements Reporter  {
  duration = 0;
@ -76,21 +101,40 @@ export class BaseReporter implements Reporter  {
    this.result = result;
  }

-  private _printSlowTests() {
+  protected getSlowTests(): [string, number][] {
    if (!this.config.reportSlowTests)
-      return;
+      return [];
    const fileDurations = [...this.fileDurations.entries()];
    fileDurations.sort((a, b) => b[1] - a[1]);
    const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY);
-    for (let i = 0; i < count; ++i) {
-      const duration = fileDurations[i][1];
-      if (duration <= this.config.reportSlowTests.threshold)
-        break;
-      console.log(colors.yellow('  Slow test: ') + fileDurations[i][0] + colors.yellow(` (${milliseconds(duration)})`));
-    }
+    const threshold =  this.config.reportSlowTests.threshold;
+    return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count);
  }

-  epilogue(full: boolean) {
+  protected generateSummaryMessage({ skipped, expected, unexpected, flaky }: TestSummary) {
+    const tokens: string[] = [];
+    tokens.push('');
+    if (unexpected.length) {
+      tokens.push(colors.red(`  ${unexpected.length} failed`));
+      for (const test of unexpected)
+        tokens.push(colors.red(formatTestHeader(this.config, test, '    ')));
+    }
+    if (flaky.length) {
+      tokens.push(colors.yellow(`  ${flaky.length} flaky`));
+      for (const test of flaky)
+        tokens.push(colors.yellow(formatTestHeader(this.config, test, '    ')));
+    }
+    if (skipped)
+      tokens.push(colors.yellow(`  ${skipped} skipped`));
+    if (expected)
+      tokens.push(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
+    if (this.result.status === 'timedout')
+      tokens.push(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
+
+    return tokens.join('\n');
+  }
+
+  protected generateSummary(): TestSummary {
    let skipped = 0;
    let expected = 0;
    const skippedWithError: TestCase[] = [];
@ -112,96 +156,126 @@ export class BaseReporter implements Reporter  {
    });

    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
-    if (full && failuresToPrint.length) {
-      console.log('');
-      this._printFailures(failuresToPrint);
-    }
+    return {
+      skipped,
+      expected,
+      skippedWithError,
+      unexpected,
+      flaky,
+      failuresToPrint
+    };
+  }

+  epilogue(full: boolean) {
+    const summary = this.generateSummary();
+    const summaryMessage = this.generateSummaryMessage(summary);
+    if (full && summary.failuresToPrint.length)
+      this._printFailures(summary.failuresToPrint);
    this._printSlowTests();
-
-    console.log('');
-    if (unexpected.length) {
-      console.log(colors.red(`  ${unexpected.length} failed`));
-      for (const test of unexpected)
-        console.log(colors.red(formatTestHeader(this.config, test, '    ')));
-    }
-    if (flaky.length) {
-      console.log(colors.yellow(`  ${flaky.length} flaky`));
-      for (const test of flaky)
-        console.log(colors.yellow(formatTestHeader(this.config, test, '    ')));
-    }
-    if (skipped)
-      console.log(colors.yellow(`  ${skipped} skipped`));
-    if (expected)
-      console.log(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
-    if (this.result.status === 'timedout')
-      console.log(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
+    this._printSummary(summaryMessage);
  }

  private _printFailures(failures: TestCase[]) {
+    console.log('');
    failures.forEach((test, index) => {
-      console.log(formatFailure(this.config, test, index + 1, this.printTestOutput));
+      console.log(formatFailure(this.config, test, {
+        index: index + 1,
+        includeStdio: this.printTestOutput
+      }).message);
    });
  }

+  private _printSlowTests() {
+    this.getSlowTests().forEach(([file, duration]) => {
+      console.log(colors.yellow('  Slow test: ') + file + colors.yellow(` (${milliseconds(duration)})`));
+    });
+  }
+
+  private _printSummary(summary: string){
+    console.log('');
+    console.log(summary);
+  }
+
  willRetry(test: TestCase): boolean {
    return test.outcome() === 'unexpected' && test.results.length <= test.retries;
  }
 }

-export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean): string {
+export function formatFailure(config: FullConfig, test: TestCase, options: {index?: number, includeStdio?: boolean, includeAttachments?: boolean, filePath?: string} = {}): {
+  message: string,
+  annotations: Annotation[]
+} {
+  const { index, includeStdio, includeAttachments = true, filePath } = options;
  const lines: string[] = [];
-  lines.push(colors.red(formatTestHeader(config, test, '  ', index)));
+  const title = formatTestTitle(config, test);
+  const annotations: Annotation[] = [];
+  const header = formatTestHeader(config, test, '  ', index);
+  lines.push(colors.red(header));
  for (const result of test.results) {
-    const resultTokens = formatResultFailure(test, result, '    ');
+    const resultLines: string[] = [];
+    const { tokens: resultTokens, position } = formatResultFailure(test, result, '    ');
    if (!resultTokens.length)
      continue;
    if (result.retry) {
-      lines.push('');
-      lines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
+      resultLines.push('');
+      resultLines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
    }
-    lines.push(...resultTokens);
-    for (let i = 0; i < result.attachments.length; ++i) {
-      const attachment = result.attachments[i];
-      lines.push('');
-      lines.push(colors.cyan(pad(`    attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-')));
-      if (attachment.path) {
-        const relativePath = path.relative(process.cwd(), attachment.path);
-        lines.push(colors.cyan(`    ${relativePath}`));
-        // Make this extensible
-        if (attachment.name === 'trace') {
-          lines.push(colors.cyan(`    Usage:`));
-          lines.push('');
-          lines.push(colors.cyan(`        npx playwright show-trace ${relativePath}`));
-          lines.push('');
-        }
-      } else {
-        if (attachment.contentType.startsWith('text/')) {
-          let text = attachment.body!.toString();
-          if (text.length > 300)
-            text = text.slice(0, 300) + '...';
-          lines.push(colors.cyan(`    ${text}`));
+    resultLines.push(...resultTokens);
+    if (includeAttachments) {
+      for (let i = 0; i < result.attachments.length; ++i) {
+        const attachment = result.attachments[i];
+        resultLines.push('');
+        resultLines.push(colors.cyan(pad(`    attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-')));
+        if (attachment.path) {
+          const relativePath = path.relative(process.cwd(), attachment.path);
+          resultLines.push(colors.cyan(`    ${relativePath}`));
+          // Make this extensible
+          if (attachment.name === 'trace') {
+            resultLines.push(colors.cyan(`    Usage:`));
+            resultLines.push('');
+            resultLines.push(colors.cyan(`        npx playwright show-trace ${relativePath}`));
+            resultLines.push('');
+          }
+        } else {
+          if (attachment.contentType.startsWith('text/')) {
+            let text = attachment.body!.toString();
+            if (text.length > 300)
+              text = text.slice(0, 300) + '...';
+            resultLines.push(colors.cyan(`    ${text}`));
+          }
        }
+        resultLines.push(colors.cyan(pad('   ', '-')));
      }
-      lines.push(colors.cyan(pad('   ', '-')));
    }
    const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
-    if (stdio && output.length) {
+    if (includeStdio && output.length) {
      const outputText = output.map(({ chunk, type }) => {
        const text = chunk.toString('utf8');
        if (type === 'stderr')
          return colors.red(stripAnsiEscapes(text));
        return text;
      }).join('');
-      lines.push('');
-      lines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
+      resultLines.push('');
+      resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
    }
+    if (filePath) {
+      annotations.push({
+        filePath,
+        position,
+        title,
+        message: [header, ...resultLines].join('\n'),
+      });
+    }
+    lines.push(...resultLines);
  }
  lines.push('');
-  return lines.join('\n');
+  return {
+    message: lines.join('\n'),
+    annotations
+  };
 }

-export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): string[] {
+export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): FailureDetails {
  const resultTokens: string[] = [];
  if (result.status === 'timedOut') {
    resultTokens.push('');
@ -211,9 +285,15 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
    resultTokens.push('');
    resultTokens.push(indent(colors.red(`Expected to fail, but passed.`), initialIndent));
  }
-  if (result.error !== undefined)
-    resultTokens.push(indent(formatError(result.error, test.location.file), initialIndent));
-  return resultTokens;
+  let error: ErrorDetails | undefined = undefined;
+  if (result.error !== undefined) {
+    error = formatError(result.error, test.location.file);
+    resultTokens.push(indent(error.message, initialIndent));
+  }
+  return {
+    tokens: resultTokens,
+    position: error?.position,
+  };
 }

 function relativeTestPath(config: FullConfig, test: TestCase): string {
@ -239,14 +319,16 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in
  return pad(header, '=');
 }

-export function formatError(error: TestError, file?: string) {
+export function formatError(error: TestError, file?: string): ErrorDetails {
  const stack = error.stack;
  const tokens = [''];
+  let positionInFile: PositionInFile | undefined;
  if (stack) {
    const { message, stackLines, position } = prepareErrorStack(
        stack,
        file
    );
+    positionInFile = position;
    tokens.push(message);

    const codeFrame = generateCodeFrame(file, position);
@ -261,7 +343,10 @@ export function formatError(error: TestError, file?: string) {
  } else if (error.value) {
    tokens.push(error.value);
  }
-  return tokens.join('\n');
+  return {
+    position: positionInFile,
+    message: tokens.join('\n'),
+  };
 }

 function pad(line: string, char: string): string {
@ -306,7 +391,7 @@ export function prepareErrorStack(stack: string, file?: string): {
  };
 }

-function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined {
+function positionInFile(stackLines: string[], file: string): PositionInFile | undefined {
  // Stack will have /private/var/folders instead of /var/folders on Mac.
  file = fs.realpathSync(file);
  for (const line of stackLines) {
--- a/src/test/reporters/github.ts
+++ b/src/test/reporters/github.ts
@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import milliseconds from 'ms';
+import path from 'path';
+import { BaseReporter, formatFailure } from './base';
+import { TestCase, FullResult } from '../../../types/testReporter';
+
+type GitHubLogType = 'debug' | 'notice' | 'warning' | 'error';
+
+type GitHubLogOptions = Partial<{
+  title: string;
+  file: string;
+  col: number;
+  endColumn: number;
+  line: number;
+  endLine: number;
+}>;
+
+class GitHubLogger {
+  private _isGitHubAction: boolean = !!process.env.GITHUB_ACTION;
+
+  private _log(message: string, type: GitHubLogType = 'notice', options: GitHubLogOptions = {}) {
+    if (this._isGitHubAction)
+      message = message.replace(/\n/g, '%0A');
+    const configs = Object.entries(options)
+        .map(([key, option]) => `${key}=${option}`)
+        .join(',');
+    console.log(`::${type} ${configs}::${message}`);
+  }
+
+  debug(message: string, options?: GitHubLogOptions) {
+    this._log(message, 'debug', options);
+  }
+
+  error(message: string, options?: GitHubLogOptions) {
+    this._log(message, 'error', options);
+  }
+
+  notice(message: string, options?: GitHubLogOptions) {
+    this._log(message, 'notice', options);
+  }
+
+  warning(message: string, options?: GitHubLogOptions) {
+    this._log(message, 'warning', options);
+  }
+}
+
+export class GitHubReporter extends BaseReporter {
+  githubLogger = new GitHubLogger();
+
+  override async onEnd(result: FullResult) {
+    super.onEnd(result);
+    this._printAnnotations();
+  }
+
+  private _printAnnotations() {
+    const summary = this.generateSummary();
+    const summaryMessage = this.generateSummaryMessage(summary);
+    if (summary.failuresToPrint.length)
+      this._printFailureAnnotations(summary.failuresToPrint);
+    this._printSlowTestAnnotations();
+    this._printSummaryAnnotation(summaryMessage);
+  }
+
+  private _printSlowTestAnnotations() {
+    this.getSlowTests().forEach(([file, duration]) => {
+      const filePath = workspaceRelativePath(path.join(process.cwd(), file));
+      this.githubLogger.warning(`${filePath} took ${milliseconds(duration)}`, {
+        title: 'Slow Test',
+        file: filePath,
+      });
+    });
+  }
+
+  private _printSummaryAnnotation(summary: string){
+    this.githubLogger.notice(summary, {
+      title: '🎭 Playwright Run Summary'
+    });
+  }
+
+  private _printFailureAnnotations(failures: TestCase[]) {
+    failures.forEach((test, index) => {
+      const filePath = workspaceRelativePath(test.location.file);
+      const { annotations } = formatFailure(this.config, test, {
+        filePath,
+        index: index + 1,
+        includeStdio: true,
+        includeAttachments: false,
+      });
+      annotations.forEach(({ filePath, title, message, position }) => {
+        const options: GitHubLogOptions = {
+          file: filePath,
+          title,
+        };
+        if (position) {
+          options.line = position.line;
+          options.col = position.column;
+        }
+        this.githubLogger.error(message, options);
+      });
+    });
+  }
+}
+
+function workspaceRelativePath(filePath: string): string {
+  return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath);
+}
+
+export default GitHubReporter;
--- a/src/test/reporters/junit.ts
+++ b/src/test/reporters/junit.ts
@ -142,7 +142,7 @@ class JUnitReporter implements Reporter {
          message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
          type: 'FAILURE',
        },
-        text: stripAnsiEscapes(formatFailure(this.config, test))
+        text: stripAnsiEscapes(formatFailure(this.config, test).message)
      });
    }

--- a/src/test/reporters/line.ts
+++ b/src/test/reporters/line.ts
@ -61,7 +61,9 @@ class LineReporter extends BaseReporter {
    process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
    if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
      process.stdout.write(`\u001B[1A\u001B[2K`);
-      console.log(formatFailure(this.config, test, ++this._failures));
+      console.log(formatFailure(this.config, test, {
+        index: ++this._failures
+      }).message);
      console.log();
    }
  }
--- a/src/test/reporters/list.ts
+++ b/src/test/reporters/list.ts
@ -16,7 +16,6 @@

 /* eslint-disable no-console */
 import colors from 'colors/safe';
-// @ts-ignore
 import milliseconds from 'ms';
 import { BaseReporter, formatTestTitle } from './base';
 import { FullConfig, FullResult, Suite, TestCase, TestResult, TestStep } from '../../../types/testReporter';
--- a/src/test/reporters/raw.ts
+++ b/src/test/reporters/raw.ts
@ -181,7 +181,7 @@ class RawReporter {
      startTime: result.startTime.toISOString(),
      duration: result.duration,
      status: result.status,
-      error: formatResultFailure(test, result, '').join('').trim(),
+      error: formatResultFailure(test, result, '').tokens.join('').trim(),
      attachments: this._createAttachments(result),
      steps: this._serializeSteps(test, result.steps)
    };
--- a/src/test/runner.ts
+++ b/src/test/runner.ts
@ -27,6 +27,7 @@ import { Loader } from './loader';
 import { Reporter } from '../../types/testReporter';
 import { Multiplexer } from './reporters/multiplexer';
 import DotReporter from './reporters/dot';
+import GitHubReporter from './reporters/github';
 import LineReporter from './reporters/line';
 import ListReporter from './reporters/list';
 import JSONReporter from './reporters/json';
@ -68,6 +69,7 @@ export class Runner {
      dot: list ? ListModeReporter : DotReporter,
      line: list ? ListModeReporter : LineReporter,
      list: list ? ListModeReporter : ListReporter,
+      github: GitHubReporter,
      json: JSONReporter,
      junit: JUnitReporter,
      null: EmptyReporter,
@ -539,5 +541,5 @@ class ListModeReporter implements Reporter {
  }
 }

-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github'] as const;
 export type BuiltInReporter = typeof builtInReporters[number];
--- a/tests/playwright-test/github-reporter.spec.ts
+++ b/tests/playwright-test/github-reporter.spec.ts
@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+import { relativeFilePath } from '../../src/test/util';
+
+test('print GitHub annotations for success', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { reporter: 'github' }, { GITHUB_ACTION: 'true' });
+  const text = stripAscii(result.output);
+  expect(text).not.toContain('::error');
+  expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A  1 passed');
+  expect(result.exitCode).toBe(0);
+});
+
+test('print GitHub annotations with newline if not in CI', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { reporter: 'github' }, { GITHUB_ACTION: '' });
+  const text = stripAscii(result.output);
+  expect(text).not.toContain('::error');
+  expect(text).toContain(`::notice title=🎭 Playwright Run Summary::
+  1 passed `);
+  expect(result.exitCode).toBe(0);
+});
+
+
+test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('example', async ({}) => {
+        expect(1 + 1).toBe(3);
+      });
+    `
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: process.cwd() });
+  const text = stripAscii(result.output);
+  const testPath =  relativeFilePath(testInfo.outputPath('a.test.js'));
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #1`);
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #2`);
+  expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23::  1) a.test.js:6:7 › example =======================================================================%0A%0A    Retry #3`);
+  expect(result.exitCode).toBe(1);
+});
+
+
+test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        reportSlowTests: { max: 0, threshold: 100 }
+      };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('slow test', async ({}) => {
+        await new Promise(f => setTimeout(f, 200));
+      });
+    `
+  }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: '' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took 2');
+  expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A  1 passed');
+  expect(result.exitCode).toBe(0);
+});