feat(testrunner): allow unexpected passes (#3665)

2024-09-11 20:37:54 +03:00 · 2020-08-28 00:32:00 -07:00 · 2020-08-28 00:32:00 -07:00 · 6ffdd4dfa1
commit 6ffdd4dfa1
parent 5c0f93301d
47 changed files with 299 additions and 127 deletions
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -40,8 +40,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json && npm run coverage"
      env:
        BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
        PWRUNNER_JSON_REPORT: "test-results/report.json"
    - uses: actions/upload-artifact@v1
      if: always()
@ -67,8 +65,6 @@ jobs:
    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json
      env:
        BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
        PWRUNNER_JSON_REPORT: "test-results/report.json"
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
@ -98,8 +94,6 @@ jobs:
      shell: bash
      env:
        BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
        PWRUNNER_JSON_REPORT: "test-results/report.json"
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
@ -152,7 +146,6 @@ jobs:
      env:
        BROWSER: ${{ matrix.browser }}
        HEADLESS: "false"
-        DEBUG_FILE: "test-results/debug.log"
        PWRUNNER_JSON_REPORT: "test-results/report.json"
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
@ -184,8 +177,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json"
      env:
        BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
        PWWIRE: true
        PWRUNNER_JSON_REPORT: "test-results/report.json"
    - uses: actions/upload-artifact@v1
--- a/test-runner/src/builtin.fixtures.ts
+++ b/test-runner/src/builtin.fixtures.ts
@ -32,6 +32,7 @@ declare global {

  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
    fail(condition: boolean): ItFunction<STATE>;
+    fixme(condition: boolean): ItFunction<STATE>;
    flaky(condition: boolean): ItFunction<STATE>;
    skip(condition: boolean): ItFunction<STATE>;
    slow(): ItFunction<STATE>;
--- a/test-runner/src/reporters/base.ts
+++ b/test-runner/src/reporters/base.ts
@ -30,10 +30,9 @@ const stackUtils = new StackUtils();

 export class BaseReporter implements Reporter  {
  skipped: Test[] = [];
-  passed: Test[] = [];
+  asExpected: Test[] = [];
+  unexpected = new Set<Test>();
  flaky: Test[] = [];
-  failed: Test[] = [];
-  timedOut: Test[] = [];
  duration = 0;
  startTime: number;
  config: RunnerConfig;
@ -66,28 +65,24 @@ export class BaseReporter implements Reporter  {
  }

  onTestEnd(test: Test, result: TestResult) {
-    switch (result.status) {
-      case 'skipped': {
-        this.skipped.push(test);
-        return;
-      }
-      case 'passed':
-        if (test.results.length === 1)
-          this.passed.push(test);
-        else
-          this.flaky.push(test);
-        return;
-      case 'failed':
-        // Fall through.
-      case 'timedOut': {
-        if (test.results.length === this.config.retries + 1) {
-          if (result.status === 'timedOut')
-            this.timedOut.push(test);
-          else
-            this.failed.push(test);
-        }
-        return;
+    if (result.status === 'skipped') {
+      this.skipped.push(test);
+      return;
+    }
+
+    if (result.status === result.expectedStatus) {
+      if (test.results.length === 1) {
+        // as expected from the first attempt
+        this.asExpected.push(test);
+      } else {
+        // as expected after unexpected -> flaky.
+        this.flaky.push(test);
      }
+      return;
+    }
+    if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) {
+      // We made as many retries as we could, still failing.
+      this.unexpected.add(test);
    }
  }

@ -98,15 +93,16 @@ export class BaseReporter implements Reporter  {
  epilogue() {
    console.log('');

-    console.log(colors.green(`  ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
+    console.log(colors.green(`  ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));

    if (this.skipped.length)
      console.log(colors.yellow(`  ${this.skipped.length} skipped`));

-    if (this.failed.length) {
-      console.log(colors.red(`  ${this.failed.length} failed`));
+    const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut'));
+    if (filteredUnexpected.length) {
+      console.log(colors.red(`  ${filteredUnexpected.length} failed`));
      console.log('');
-      this._printFailures(this.failed);
+      this._printFailures(filteredUnexpected);
    }

    if (this.flaky.length) {
@ -115,11 +111,13 @@ export class BaseReporter implements Reporter  {
      this._printFailures(this.flaky);
    }

-    if (this.timedOut.length) {
-      console.log(colors.red(`  ${this.timedOut.length} timed out`));
+    const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
+    if (timedOut.length) {
+      console.log(colors.red(`  ${timedOut.length} timed out`));
      console.log('');
-      this._printFailures(this.timedOut);
+      this._printFailures(timedOut);
    }
+    console.log('');
  }

  private _printFailures(failures: Test[]) {
@ -131,7 +129,8 @@ export class BaseReporter implements Reporter  {
  formatFailure(test: Test, index?: number): string {
    const tokens: string[] = [];
    const relativePath = path.relative(process.cwd(), test.file);
-    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`;
+    const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
+    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`;
    tokens.push(colors.bold(colors.red(header)));
    for (const result of test.results) {
      if (result.status === 'passed')
--- a/test-runner/src/reporters/dot.ts
+++ b/test-runner/src/reporters/dot.ts
@ -23,8 +23,8 @@ class DotReporter extends BaseReporter {
    super.onTestEnd(test, result);
    switch (result.status) {
      case 'skipped': process.stdout.write(colors.yellow('∘')); break;
-      case 'passed': process.stdout.write(colors.green('·')); break;
-      case 'failed': process.stdout.write(colors.red(test.results.length > 1 ? '' + test.results.length : 'F')); break;
+      case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break;
+      case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break;
      case 'timedOut': process.stdout.write(colors.red('T')); break;
    }
  }
--- a/test-runner/src/reporters/list.ts
+++ b/test-runner/src/reporters/list.ts
@ -35,12 +35,14 @@ class ListReporter extends BaseReporter {
  onTestEnd(test: Test, result: TestResult) {
    super.onTestEnd(test, result);
    let text = '';
-    switch (result.status) {
-      case 'skipped': text = colors.green('  - ') + colors.cyan(test.fullTitle()); break;
-      case 'passed': text = '\u001b[2K\u001b[0G' + colors.green('  ✓ ') + colors.gray(test.fullTitle()); break;
-      case 'failed':
-        // fall through
-      case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(`  ${++this._failure}) ` + test.fullTitle()); break;
+    if (result.status === 'skipped') {
+      text = colors.green('  - ') + colors.cyan(test.fullTitle());
+    } else {
+      const statusMark = result.status === 'passed' ? colors.green('  ✓ ') : colors.red('  x ');
+      if (result.status === result.expectedStatus)
+        text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle());
+      else
+        text = '\u001b[2K\u001b[0G' + colors.red(`  ${++this._failure}) ` + test.fullTitle());
    }
    process.stdout.write(text + '\n');
  }
--- a/test-runner/src/reporters/pytest.ts
+++ b/test-runner/src/reporters/pytest.ts
@ -149,14 +149,15 @@ class PytestReporter extends BaseReporter {
    }

    const status = [];
-    if (this.passed.length)
-      status.push(colors.green(`${this.passed.length} passed`));
+    if (this.asExpected.length)
+      status.push(colors.green(`${this.asExpected.length} as expected`));
    if (this.skipped.length)
      status.push(colors.yellow(`${this.skipped.length} skipped`));
-    if (this.failed.length)
-      status.push(colors.red(`${this.failed.length} failed`));
-    if (this.timedOut.length)
-      status.push(colors.red(`${this.timedOut.length} timed out`));
+    const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
+    if (this.unexpected.size - timedOut.length)
+      status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`));
+    if (timedOut.length)
+      status.push(colors.red(`${timedOut.length} timed out`));
    status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));

    for (let i = lines.length; i < this._visibleRows; ++i)
--- a/test-runner/src/runner.ts
+++ b/test-runner/src/runner.ts
@ -103,7 +103,11 @@ export class Runner {
    let doneCallback;
    const result = new Promise(f => doneCallback = f);
    worker.once('done', params => {
-      if (!params.failedTestId && !params.fatalError) {
+      // We won't file remaining if:
+      // - there are no remaining
+      // - we are here not because something failed
+      // - no unrecoverable worker error
+      if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
        this._workerAvailable(worker);
        doneCallback();
        return;
@ -112,8 +116,8 @@ export class Runner {
      // When worker encounters error, we will restart it.
      this._restartWorker(worker);

-      // In case of fatal error without test id, we are done with the entry.
-      if (params.fatalError && !params.failedTestId) {
+      // In case of fatal error, we are done with the entry.
+      if (params.fatalError) {
        // Report all the tests are failing with this error.
        for (const id of entry.ids) {
          const { test, result } = this._testById.get(id);
@ -127,13 +131,16 @@ export class Runner {
      }

      const remaining = params.remaining;
-      if (this._config.retries) {
+
+      // Only retry expected failures, not passes and only if the test failed.
+      if (this._config.retries && params.failedTestId) {
        const pair = this._testById.get(params.failedTestId);
-        if (pair.test.results.length < this._config.retries + 1) {
+        if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) {
          pair.result = pair.test._appendResult();
          remaining.unshift(pair.test._id);
        }
      }
+
      if (remaining.length)
        this._queue.unshift({ ...entry, ids: remaining });

@ -169,6 +176,7 @@ export class Runner {
      const { test } = this._testById.get(params.id);
      test._skipped = params.skipped;
      test._flaky = params.flaky;
+      test._expectedStatus = params.expectedStatus;
      this._reporter.onTestBegin(test);
    });
    worker.on('testEnd', params => {
--- a/test-runner/src/spec.ts
+++ b/test-runner/src/spec.ts
@ -53,7 +53,7 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
  const suites = [suite];
  suite.file = file;

-  const it = specBuilder(['skip', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
+  const it = specBuilder(['skip', 'fixme', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
    const suite = suites[0];
    const test = new Test(title, fn);
    test.file = file;
@ -65,8 +65,10 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
      test.only = true;
    if (!only && specs.skip && specs.skip[0])
      test._skipped = true;
-    if (!only && specs.fail && specs.fail[0])
+    if (!only && specs.fixme && specs.fixme[0])
      test._skipped = true;
+    if (specs.fail && specs.fail[0])
+      test._expectedStatus = 'failed';
    if (specs.flaky && specs.flaky[0])
      test._flaky = true;
    suite._addTest(test);
@ -81,7 +83,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
    if (only)
      child.only = true;
    if (!only && specs.skip && specs.skip[0])
-      child.skipped = true;
+      child._skipped = true;
+    if (!only && specs.fixme && specs.fixme[0])
+      child._skipped = true;
    suites.unshift(child);
    fn();
    suites.shift();
--- a/test-runner/src/test.ts
+++ b/test-runner/src/test.ts
@ -16,6 +16,8 @@

 export type Configuration = { name: string, value: string }[];

+type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
+
 export class Test {
  suite: Suite;
  title: string;
@ -27,10 +29,13 @@ export class Test {
  results: TestResult[] = [];

  _id: string;
+  // Skipped & flaky are resolved based on options in worker only
+  // We will compute them there and send to the runner (front-end)
  _skipped = false;
  _flaky = false;
  _overriddenFn: Function;
  _startTime: number;
+  _expectedStatus: TestStatus = 'passed';

  constructor(title: string, fn: Function) {
    this.title = title;
@ -48,7 +53,7 @@ export class Test {
  _appendResult(): TestResult {
    const result: TestResult = {
      duration: 0,
-      status: 'none',
+      expectedStatus: 'passed',
      stdout: [],
      stderr: [],
      data: {}
@ -58,17 +63,21 @@ export class Test {
  }

  _ok(): boolean {
-    if (this._skipped)
+    if (this._skipped || this.suite._isSkipped())
      return true;
-    const hasFailedResults = !!this.results.find(r => r.status !== 'passed' && r.status !== 'skipped');
+    const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus);
    if (!hasFailedResults)
      return true;
    if (!this._flaky)
      return false;
-    const hasPassedResults = !!this.results.find(r => r.status === 'passed');
+    const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus);
    return hasPassedResults;
  }

+  _hasResultWithStatus(status: TestStatus): boolean {
+    return !!this.results.find(r => r.status === status);
+  }
+
  _clone(): Test {
    const test = new Test(this.title, this.fn);
    test.suite = this.suite;
@ -83,7 +92,8 @@ export class Test {

 export type TestResult = {
  duration: number;
-  status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped';
+  status?: TestStatus;
+  expectedStatus: TestStatus;
  error?: any;
  stdout: (string | Buffer)[];
  stderr: (string | Buffer)[];
@ -96,9 +106,12 @@ export class Suite {
  suites: Suite[] = [];
  tests: Test[] = [];
  only = false;
-  skipped = false;
  file: string;
  configuration: Configuration;
+
+  // Skipped & flaky are resolved based on options in worker only
+  // We will compute them there and send to the runner (front-end)
+  _skipped = false;
  _configurationString: string;

  _hooks: { type: string, fn: Function } [] = [];
@ -124,7 +137,7 @@ export class Suite {
  }

  _isSkipped(): boolean {
-    return this.skipped || (this.parent && this.parent._isSkipped());
+    return this._skipped || (this.parent && this.parent._isSkipped());
  }

  _addTest(test: Test) {
@ -163,7 +176,7 @@ export class Suite {
    const suite = new Suite(this.title);
    suite.only = this.only;
    suite.file = this.file;
-    suite.skipped = this.skipped;
+    suite._skipped = this._skipped;
    return suite;
  }

--- a/test-runner/src/testRunner.ts
+++ b/test-runner/src/testRunner.ts
@ -46,7 +46,6 @@ export class TestRunner extends EventEmitter {
  private _ids: Set<string>;
  private _remaining: Set<string>;
  private _trialRun: any;
-  private _configuredFile: any;
  private _parsedGeneratorConfiguration: any = {};
  private _config: RunnerConfig;
  private _timeout: number;
@ -55,6 +54,7 @@ export class TestRunner extends EventEmitter {
  private _stdErrBuffer: (string | Buffer)[] = [];
  private _testResult: TestResult | null = null;
  private _suite: Suite;
+  private _loaded = false;

  constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) {
    super();
@ -66,7 +66,6 @@ export class TestRunner extends EventEmitter {
    this._trialRun = config.trialRun;
    this._timeout = config.timeout;
    this._config = config;
-    this._configuredFile = entry.file + `::[${entry.configurationString}]`;
    for (const {name, value} of entry.configuration)
      this._parsedGeneratorConfiguration[name] = value;
    this._parsedGeneratorConfiguration['parallelIndex'] = workerId;
@ -77,14 +76,16 @@ export class TestRunner extends EventEmitter {
    this._trialRun = true;
  }

-  fatalError(error: Error | any) {
-    this._fatalError = serializeError(error);
+  unhandledError(error: Error | any) {
    if (this._testResult) {
-      this._testResult.error = this._fatalError;
+      this._testResult.error = serializeError(error);
      this.emit('testEnd', {
        id: this._testId,
        result: this._testResult
      });
+    } else if (!this._loaded) {
+      // No current test - fatal error.
+      this._fatalError = serializeError(error);
    }
    this._reportDone();
  }
@ -114,6 +115,7 @@ export class TestRunner extends EventEmitter {
    require(this._suite.file);
    revertBabelRequire();
    this._suite._renumber();
+    this._loaded = true;

    rerunRegistrations(this._suite.file, 'test');
    await this._runSuite(this._suite);
@ -132,7 +134,6 @@ export class TestRunner extends EventEmitter {
        await this._runSuite(entry);
      else
        await this._runTest(entry);
-
    }
    try {
      await this._runHooks(suite, 'afterAll', 'after');
@ -151,6 +152,9 @@ export class TestRunner extends EventEmitter {

    const id = test._id;
    this._testId = id;
+    // We only know resolved skipped/flaky value in the worker,
+    // send it to the runner.
+    test._skipped = test._skipped || test.suite._isSkipped();
    this.emit('testBegin', {
      id,
      skipped: test._skipped,
@ -160,13 +164,14 @@ export class TestRunner extends EventEmitter {
    const result: TestResult = {
      duration: 0,
      status: 'passed',
+      expectedStatus: test._expectedStatus,
      stdout: [],
      stderr: [],
      data: {}
    };
    this._testResult = result;

-    if (test._skipped || test.suite._isSkipped()) {
+    if (test._skipped) {
      result.status = 'skipped';
      this.emit('testEnd', { id, result });
      return;
@ -181,7 +186,7 @@ export class TestRunner extends EventEmitter {
        await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo);
        await this._runHooks(test.suite, 'afterEach', 'after', testInfo);
      } else {
-        result.status = 'passed';
+        result.status = result.expectedStatus;
      }
    } catch (error) {
      // Error in the test fixture teardown.
--- a/test-runner/src/worker.ts
+++ b/test-runner/src/worker.ts
@ -49,12 +49,12 @@ let testRunner: TestRunner;

 process.on('unhandledRejection', (reason, promise) => {
  if (testRunner)
-    testRunner.fatalError(reason);
+    testRunner.unhandledError(reason);
 });

 process.on('uncaughtException', error => {
  if (testRunner)
-    testRunner.fatalError(error);
+    testRunner.unhandledError(error);
 });

 process.on('message', async message => {
--- a/test-runner/test/assets/expected-failure.js
+++ b/test-runner/test/assets/expected-failure.js
@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../lib');
+
+it.fail(true)('fails',() => {
+  expect(1 + 1).toBe(3);
+});
+
+it('non-empty remaining',() => {
+  expect(1 + 1).toBe(2);
+});
--- a/test-runner/test/assets/nested-skip.js
+++ b/test-runner/test/assets/nested-skip.js
@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+describe.skip(true)('skipped', () => {
+  it('succeeds',() => {
+    expect(1 + 1).toBe(2);
+  });  
+});
--- a/test-runner/test/assets/one-timeout.js
+++ b/test-runner/test/assets/one-timeout.js
@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it('timeout', async () => {
+  await new Promise(f => setTimeout(f, 10000));
+});
--- a/test-runner/test/assets/unexpected-pass.js
+++ b/test-runner/test/assets/unexpected-pass.js
@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it.fail(true)('succeeds',() => {
+  expect(1 + 1).toBe(2);
+});
--- a/test-runner/test/exit-code.spec.ts
+++ b/test-runner/test/exit-code.spec.ts
@ -14,6 +14,7 @@
 * limitations under the License.
 */

+import colors from 'colors/safe';
 import { spawnSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
@ -30,6 +31,14 @@ it('should fail', async () => {
  expect(result.failed).toBe(1);
 });

+it('should timeout', async () => {
+  const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(timedOut).toBe(1);
+});
+
 it('should succeed', async () => {
  const result = await runTest('one-success.js');
  expect(result.exitCode).toBe(0);
@ -85,6 +94,15 @@ it('should retry failures', async () => {
  expect(result.flaky).toBe(1);
 });

+it('should retry timeout', async () => {
+  const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(timedOut).toBe(1);
+  expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3));
+});
+
 it('should repeat each', async () => {
  const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 });
  expect(exitCode).toBe(0);
@ -106,6 +124,44 @@ it('should allow flaky', async () => {
  expect(result.flaky).toBe(1);
 });

+it('should fail on unexpected pass', async () => {
+  const { exitCode, failed, output } = await runTest('unexpected-pass.js');
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+it('should fail on unexpected pass with retries', async () => {
+  const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 });
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+it('should not retry unexpected pass', async () => {
+  const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(1);
+  expect(output.split('\n')[0]).toBe(colors.red('P'));
+});
+
+it('should not retry expected failure', async () => {
+  const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+  expect(failed).toBe(0);
+  expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·'));
+});
+
+it('should respect nested skip', async () => {
+  const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js');
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(1);
+});
+
 async function runTest(filePath: string, params: any = {}) {
  const outputDir = path.join(__dirname, 'test-results');
  const reportFile = path.join(outputDir, 'results.json');
@ -125,14 +181,20 @@ async function runTest(filePath: string, params: any = {}) {
  });
  const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
  const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
+  const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1];
  const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1];
+  const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1];
  const report = JSON.parse(fs.readFileSync(reportFile).toString());
+  let outputStr = output.toString();
+  outputStr = outputStr.substring(1, outputStr.length - 1);
  return {
    exitCode: status,
-    output: output.toString(),
+    output: outputStr,
    passed: parseInt(passed, 10),
    failed: parseInt(failed || '0', 10),
+    timedOut: parseInt(timedOut || '0', 10),
    flaky: parseInt(flaky || '0', 10),
+    skipped: parseInt(skipped || '0', 10),
    report
  };
 }
--- a/test/snapshots/page-screenshot/firefox/screenshot-webgl.png
+++ b/test/snapshots/page-screenshot/firefox/screenshot-webgl.png
--- a/test/snapshots/page-screenshot/webkit/screenshot-webgl.png
+++ b/test/snapshots/page-screenshot/webkit/screenshot-webgl.png
--- a/test/browsercontext-credentials.spec.ts
+++ b/test/browsercontext-credentials.spec.ts
@ -48,7 +48,7 @@ it('should work with correct credentials', async({browser, server}) => {
  await context.close();
 });

-it.fail(options.CHROMIUM && !options.HEADLESS)('should fail with wrong credentials', async({browser, server}) => {
+it('should fail with wrong credentials', async({browser, server}) => {
  server.setAuth('/empty.html', 'user', 'pass');
  const context = await browser.newContext({
    httpCredentials: { username: 'foo', password: 'bar' }
--- a/test/browsercontext-locale.spec.ts
+++ b/test/browsercontext-locale.spec.ts
@ -137,7 +137,7 @@ it('should be isolated between contexts', async({browser, server}) => {
  ]);
 });

-it.fail(options.FIREFOX)('should not change default locale in another context', async({browser, server}) => {
+it('should not change default locale in another context', async({browser, server}) => {
  async function getContextLocale(context) {
    const page = await context.newPage();
    return await page.evaluate(() => (new Intl.NumberFormat()).resolvedOptions().locale);
--- a/test/browsercontext-page-event.spec.ts
+++ b/test/browsercontext-page-event.spec.ts
@ -156,7 +156,7 @@ it('should fire page lifecycle events', async function({browser, server}) {
  await context.close();
 });

-it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
+it.fixme(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
  // WebKit: Shift+Click does not open a new window.
  const context = await browser.newContext();
  const page = await context.newPage();
@ -170,7 +170,7 @@ it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, serve
  await context.close();
 });

-it.fail(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
+it.fixme(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
  // Firefox: reports an opener in this case.
  // WebKit: Ctrl+Click does not open a new tab.
  const context = await browser.newContext();
--- a/test/browsercontext-timezone-id.spec.ts
+++ b/test/browsercontext-timezone-id.spec.ts
@ -69,7 +69,7 @@ it('should work for multiple pages sharing same process', async({browser, server
  await context.close();
 });

-it.fail(options.FIREFOX)('should not change default timezone in another context', async({browser, server}) => {
+it('should not change default timezone in another context', async({browser, server}) => {
  async function getContextTimezone(context) {
    const page = await context.newPage();
    return await page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone);
--- a/test/browsertype-connect.spec.ts
+++ b/test/browsertype-connect.spec.ts
@ -81,7 +81,7 @@ it.skip(options.WIRE).slow()('disconnected event should be emitted when browser
  expect(disconnected2).toBe(1);
 });

-it.skip(options.WIRE).fail(options.CHROMIUM && WIN).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
+it.skip(options.WIRE).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
  const __testHookBeforeCreateBrowser = () => { throw new Error('Dummy') };
  const error = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint(), __testHookBeforeCreateBrowser } as any).catch(e => e);
  expect(error.message).toContain('Dummy');
--- a/test/browsertype-launch.spec.ts
+++ b/test/browsertype-launch.spec.ts
@ -43,7 +43,7 @@ it.skip(options.FIREFOX)('should throw if page argument is passed', async({brows
  expect(waitError.message).toContain('can not specify page');
 });

-it.fail(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
+it.fixme(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
  // I'm getting ENCONRESET on this one.
  const options = Object.assign({}, defaultBrowserOptions, {executablePath: path.join(__dirname, 'assets', 'dummy_bad_browser_executable.js')});
  let waitError = null;
--- a/test/capabilities.spec.ts
+++ b/test/capabilities.spec.ts
@ -48,7 +48,7 @@ it('should respect CSP', async({page, server}) => {
  expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS');
 });

-it.fail(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
+it.fixme(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
  // TODO: the test passes on Windows locally but fails on GitHub Action bot,
  // apparently due to a Media Pack issue in the Windows Server.
  // Also the test is very flaky on Linux WebKit.
--- a/test/chromium/oopif.spec.ts
+++ b/test/chromium/oopif.spec.ts
@ -65,7 +65,7 @@ it.skip(!options.CHROMIUM)('should handle remote -> local -> remote transitions'
  expect(await countOOPIFs(browser)).toBe(1);
 });

-it.fail(true)('should get the proper viewport', async({browser, page, server}) => {
+it.fixme(options.CHROMIUM).skip(!options.CHROMIUM)('should get the proper viewport', async({browser, page, server}) => {
  expect(page.viewportSize()).toEqual({width: 1280, height: 720});
  await page.goto(server.PREFIX + '/dynamic-oopif.html');
  expect(page.frames().length).toBe(2);
--- a/test/click-react.spec.ts
+++ b/test/click-react.spec.ts
@ -39,7 +39,7 @@ it.fail(true)('should report that selector does not match anymore', async ({page
  expect(error.message).toContain('element does not match the selector anymore');
 });

-it.fail(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
+it.fixme(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
  await page.goto(server.PREFIX + '/react.html');
  await page.evaluate(() => {
    renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
@ -66,7 +66,7 @@ it('should timeout when click opens alert', async({page, server}) => {
  await dialog.dismiss();
 });

-it.fail(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
+it.fixme(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
  await page.goto(server.PREFIX + '/react.html');
  await page.evaluate(() => {
    renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2' })] ));
@ -81,7 +81,7 @@ it.fail(true)('should retarget when element is recycled during hit testing', asy
  expect(await page.evaluate('window.button2')).toBe(undefined);
 });

-it.fail(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
+it.fixme(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
  await page.goto(server.PREFIX + '/react.html');
  await page.evaluate(() => {
    renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
--- a/test/click.spec.ts
+++ b/test/click.spec.ts
@ -322,7 +322,7 @@ it('should click the button inside an iframe', async({page, server}) => {
  expect(await frame.evaluate(() => window['result'])).toBe('Clicked');
 });

-it.fail(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
+it.fixme(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
  // @see https://github.com/GoogleChrome/puppeteer/issues/4110
  // @see https://bugs.chromium.org/p/chromium/issues/detail?id=986390
  // @see https://chromium-review.googlesource.com/c/chromium/src/+/1742784
--- a/test/coverage.js
+++ b/test/coverage.js
@ -24,12 +24,12 @@
 function traceAPICoverage(apiCoverage, api, events) {
  const uninstalls = [];
  for (const [name, classType] of Object.entries(api)) {
-    // console.log('trace', name);
    const className = name.substring(0, 1).toLowerCase() + name.substring(1);
    for (const methodName of Reflect.ownKeys(classType.prototype)) {
      const method = Reflect.get(classType.prototype, methodName);
      if (methodName === 'constructor' || typeof methodName !== 'string' || methodName.startsWith('_') || typeof method !== 'function')
        continue;
+ 
      apiCoverage.set(`${className}.${methodName}`, false);
      const override = function(...args) {
        apiCoverage.set(`${className}.${methodName}`, true);
--- a/test/defaultbrowsercontext-2.spec.ts
+++ b/test/defaultbrowsercontext-2.spec.ts
@ -75,7 +75,7 @@ it('should support extraHTTPHeaders option', async ({server, launchPersistent})
  expect(request.headers['foo']).toBe('bar');
 });

-it('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
+it.flaky(options.CHROMIUM)('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
  const {page, context} = await launchPersistent();
  // Note: we need an open page to make sure its functional.
  expect(fs.readdirSync(tmpDir).length).toBeGreaterThan(0);
@ -111,7 +111,7 @@ it.slow()('should restore state from userDataDir', async({browserType, defaultBr
  await removeUserDataDir(userDataDir2);
 });

-it.fail(options.CHROMIUM && (WIN || MAC)).slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions,  server, launchPersistent}) => {
+it.slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions,  server, launchPersistent}) => {
  const userDataDir = await makeUserDataDir();
  const browserContext = await browserType.launchPersistentContext(userDataDir, defaultBrowserOptions);
  const page = await browserContext.newPage();
--- a/test/dialog.spec.ts
+++ b/test/dialog.spec.ts
@ -62,7 +62,7 @@ it('should dismiss the confirm prompt', async({page}) => {
  expect(result).toBe(false);
 });

-it.fail(options.WEBKIT)('should be able to close context with open alert', async({browser}) => {
+it.fixme(options.WEBKIT && MAC)('should be able to close context with open alert', async({browser}) => {
  const context = await browser.newContext();
  const page = await context.newPage();
  const alertPromise = page.waitForEvent('dialog');
--- a/test/download.spec.ts
+++ b/test/download.spec.ts
@ -250,7 +250,7 @@ it(`should report download path within page.on('download', …) handler for Blob
  expect(fs.readFileSync(path).toString()).toBe('Hello world');
  await page.close();
 })
-it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
+it.fixme(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
  // Firefox does not download on alt-click by default.
  // Our WebKit embedder does not download on alt-click, although Safari does.
  server.setRoute('/download', (req, res) => {
@ -271,7 +271,7 @@ it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads',
  await page.close();
 });

-it.fail(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
+it.fixme(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
  // TODO: - the test fails in headful Chromium as the popup page gets closed along
  // with the session before download completed event arrives.
  // - WebKit doesn't close the popup page
--- a/test/elementhandle-owner-frame.spec.ts
+++ b/test/elementhandle-owner-frame.spec.ts
@ -17,6 +17,7 @@

 import './playwright.fixtures';
 import utils from './utils';
+import { options } from './playwright.fixtures';

 it('should work', async ({ page, server }) => {
  await page.goto(server.EMPTY_PAGE);
@ -34,7 +35,7 @@ it('should work for cross-process iframes', async ({ page, server }) => {
  expect(await elementHandle.ownerFrame()).toBe(frame);
 });

-it('should work for document', async ({ page, server }) => {
+it.flaky(WIN && options.WEBKIT)('should work for document', async ({ page, server }) => {
  await page.goto(server.EMPTY_PAGE);
  await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
  const frame = page.frames()[1];
--- a/test/elementhandle-screenshot.spec.ts
+++ b/test/elementhandle-screenshot.spec.ts
@ -351,7 +351,7 @@ it.skip(ffheadful || options.WIRE)('should restore viewport after element screen
  await context.close();
 });

-it.skip(ffheadful)('should wait for element to stop moving', async({page, server, golden}) => {
+it.skip(ffheadful).flaky(options.WEBKIT && !options.HEADLESS && LINUX)('should wait for element to stop moving', async ({ page, server, golden }) => {
  await page.setViewportSize({ width: 500, height: 500 });
  await page.goto(server.PREFIX + '/grid.html');
  const elementHandle = await page.$('.box:nth-of-type(3)');
--- a/test/elementhandle-wait-for-element-state.spec.ts
+++ b/test/elementhandle-wait-for-element-state.spec.ts
@ -16,6 +16,7 @@
 */

 import './playwright.fixtures';
+import { options } from './playwright.fixtures';

 async function giveItAChanceToResolve(page) {
  for (let i = 0; i < 5; i++)
--- a/test/frame-evaluate.spec.ts
+++ b/test/frame-evaluate.spec.ts
@ -147,7 +147,7 @@ it.fail(options.CHROMIUM || options.FIREFOX)('should work in iframes that failed
  expect(await page.frames()[1].$('div')).toBeTruthy();
 });

-it.fail(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
+it.fixme(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
  // Chromium does not report isolated world for the iframe.
  await page.goto(server.EMPTY_PAGE);
  await page.evaluate(() => {
--- a/test/frame-hierarchy.spec.ts
+++ b/test/frame-hierarchy.spec.ts
@ -171,7 +171,7 @@ it('should report different frame instance when frame re-attaches', async({page,
  expect(frame1).not.toBe(frame2);
 });

-it.fail(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
+it.fixme(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
  server.setRoute('/x-frame-options-deny.html', async (req, res) => {
    res.setHeader('Content-Type', 'text/html');
    res.setHeader('X-Frame-Options', 'DENY');
--- a/test/headful.spec.ts
+++ b/test/headful.spec.ts
@ -128,7 +128,7 @@ it('should(not) block third party cookies', async({browserType, defaultBrowserOp
  await browser.close();
 });

-it.fail(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
+it.fixme(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
  // Our WebKit embedder does not respect window features.
  const browser = await browserType.launch({...defaultBrowserOptions, headless: false });
  const context = await browser.newContext({ viewport: null });
--- a/test/mouse.spec.ts
+++ b/test/mouse.spec.ts
@ -27,7 +27,7 @@ function dimensions() {
  };
 }

-it.fail(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
+it.flaky(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
  // Occasionally times out on options.FIREFOX on Windows: https://github.com/microsoft/playwright/pull/1911/checks?check_run_id=607149016
  await page.evaluate(() => {
    window["clickPromise"] = new Promise(resolve => {
--- a/test/page-add-script-tag.spec.ts
+++ b/test/page-add-script-tag.spec.ts
@ -88,7 +88,7 @@ it('should work with content', async({page, server}) => {
  expect(await page.evaluate(() => window['__injected'])).toBe(35);
 });

-it.fail(options.FIREFOX)('should throw when added with content to the CSP page', async({page, server}) => {
+it('should throw when added with content to the CSP page', async({page, server}) => {
  // Firefox fires onload for blocked script before it issues the CSP console error.
  await page.goto(server.PREFIX + '/csp.html');
  let error = null;
--- a/test/page-emulate-media.spec.ts
+++ b/test/page-emulate-media.spec.ts
@ -114,7 +114,7 @@ it('should work in cross-process iframe', async({browser, server}) => {
  await page.close();
 });

-it.fail(options.FIREFOX)('should change the actual colors in css', async({page}) => {
+it('should change the actual colors in css', async({page}) => {
  await page.setContent(`
    <style>
      @media (prefers-color-scheme: dark) {
--- a/test/page-evaluate.spec.ts
+++ b/test/page-evaluate.spec.ts
@ -415,7 +415,7 @@ it('should not throw an error when evaluation does a navigation', async ({ page,
  expect(result).toEqual([42]);
 });

-it.fail(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
+it.fixme(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
  // It is imporant to be on about:blank for sync reload.
  const result = await page.evaluate(() => {
    window.location.reload();
@ -433,7 +433,7 @@ it('should not throw an error when evaluation does a synchronous navigation and
  expect(result).toBe(undefined);
 });

-it.fail(options.WIRE)('should transfer 100Mb of data from page to node.js', async ({ page }) => {
+it('should transfer 100Mb of data from page to node.js', async ({ page }) => {
  // This is too slow with wire.
  const a = await page.evaluate(() => Array(100 * 1024 * 1024 + 1).join('a'));
  expect(a.length).toBe(100 * 1024 * 1024);
@ -453,7 +453,7 @@ it('should work even when JSON is set to null', async ({ page }) => {
  expect(result).toEqual({ abc: 123 });
 });

-it.fail(options.FIREFOX)('should await promise from popup', async ({ page, server }) => {
+it('should await promise from popup', async ({ page, server }) => {
  // Something is wrong about the way Firefox waits for the chained promise
  await page.goto(server.EMPTY_PAGE);
  const result = await page.evaluate(() => {
--- a/test/page-event-crash.spec.ts
+++ b/test/page-event-crash.spec.ts
@ -17,8 +17,6 @@

 import { options } from './playwright.fixtures';

-const CRASH_FAIL = (options.FIREFOX && WIN) || options.WIRE;
-// Firefox Win: it just doesn't crash sometimes.
 function crash(pageImpl, browserName) {
  if (browserName === 'chromium')
    pageImpl.mainFrame().goto('chrome://crash').catch(e => {});
@ -28,13 +26,13 @@ function crash(pageImpl, browserName) {
    pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
 }

-it.fail(CRASH_FAIL)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
  await page.setContent(`<div>This page should crash</div>`);
  crash(toImpl(page), browserName);
  await new Promise(f => page.on('crash', f));
 });

-it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
  await page.setContent(`<div>This page should crash</div>`);
  crash(toImpl(page), browserName);
  await page.waitForEvent('crash');
@ -43,7 +41,7 @@ it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page
  expect(err.message).toContain('crash');
 });

-it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
  await page.setContent(`<div>This page should crash</div>`);
  const promise = page.waitForEvent('response').catch(e => e);
  crash(toImpl(page), browserName);
@ -51,7 +49,7 @@ it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page,
  expect(error.message).toContain('Page crashed');
 });

-it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
+it.fixme(options.WIRE)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
  await page.setContent(`<div>This page should crash</div>`);
  server.setRoute('/one-style.css', () => {});
  const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e);
@ -61,7 +59,7 @@ it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, b
  expect(error.message).toContain('Navigation failed because page crashed');
 });

-it.fail(CRASH_FAIL)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
+it.fixme(options.WIRE)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
  await page.setContent(`<div>This page should crash</div>`);
  crash(toImpl(page), browserName);
  await page.waitForEvent('crash');
--- a/test/page-screenshot.spec.ts
+++ b/test/page-screenshot.spec.ts
@ -214,7 +214,7 @@ it.skip(ffheadful)('should work for translateZ', async({page, server, golden}) =
  expect(screenshot).toMatchImage(golden('screenshot-translateZ.png'));
 });

-it.fail(options.FIREFOX || options.WEBKIT)('should work for webgl', async({page, server, golden}) => {
+it.fixme(options.FIREFOX).flaky(options.WEBKIT && LINUX)('should work for webgl', async({page, server, golden}) => {
  await page.setViewportSize({width: 640, height: 480});
  await page.goto(server.PREFIX + '/screenshots/webgl.html');
  const screenshot = await page.screenshot();
--- a/test/page-wait-for-navigation.spec.ts
+++ b/test/page-wait-for-navigation.spec.ts
@ -141,7 +141,7 @@ it('should work with DOM history.back()/history.forward()', async({page, server}
  expect(page.url()).toBe(server.PREFIX + '/second.html');
 });

-it.fail(options.FIREFOX)('should work when subframe issues window.stop()', async({page, server}) => {
+it('should work when subframe issues window.stop()', async({page, server}) => {
  server.setRoute('/frames/style.css', (req, res) => {});
  const navigationPromise = page.goto(server.PREFIX + '/frames/one-frame.html');
  const frame = await new Promise<Frame>(f => page.once('frameattached', f));
--- a/test/permissions.spec.ts
+++ b/test/permissions.spec.ts
@ -98,7 +98,7 @@ describe.skip(options.WEBKIT)('permissions', () => {
    expect(await getPermission(page, 'geolocation')).toBe('prompt');
  });

-  it.fail(options.WEBKIT || options.FIREFOX || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
+  it.fail(options.WEBKIT || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
    //TODO: flaky
    // - Linux: https://github.com/microsoft/playwright/pull/1790/checks?check_run_id=587327883
    // - Win: https://ci.appveyor.com/project/aslushnikov/playwright/builds/32402536
--- a/test/screencast.spec.ts
+++ b/test/screencast.spec.ts
@ -173,7 +173,7 @@ class VideoPlayer {
  }
 }

-it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
  if (!toImpl)
    return;
  const videoFile = path.join(tmpDir, 'v.webm');
@ -199,7 +199,7 @@ it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, vid
  expectAll(pixels, almostRed);
 });

-it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM).flaky(options.WEBKIT)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
  if (!toImpl)
    return;
  const videoFile = path.join(tmpDir, 'v.webm');
@ -233,7 +233,7 @@ it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, serv
 });

 // Accelerated compositing is disabled in WebKit on Windows.
-it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM || (options.WEBKIT && WIN)).flaky(options.WEBKIT && LINUX)('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
  if (!toImpl)
    return;
  const videoFile = path.join(tmpDir, 'v.webm');
@ -258,7 +258,7 @@ it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transfo
  }
 });

-it.fail(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
+it.slow().fixme(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
  if (!toImpl)
   return;
  // Use server side of the context. All the code below also uses server side APIs.
@ -280,7 +280,7 @@ it.fail(options.CHROMIUM)('should fire start/stop events when page created/close
  await context.close();
 });

-it.fail(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
+it.fixme(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
  if (!toImpl)
   return;
  // Use server side of the context. All the code below also uses server side APIs.