feat(test): use metafunc in describes (#3682)

test-runner/src/builtin.fixtures.ts

@@ -20,29 +20,26 @@ import { promisify } from 'util';
 import fs from 'fs';
 import rimraf from 'rimraf';
 import { registerFixture } from './fixtures';
-import { Test } from './test';
+import { Test, Suite } from './test';
 
-interface Describers<STATE> {
+interface DescribeFunction {
+  describe(name: string, inner: () => void): void;
+  describe(name: string, modifier: (suite: Suite) => any, inner: () => void): void;
+}
+
+interface ItFunction<STATE> {
   it(name: string, inner: (state: STATE) => Promise<void> | void): void;
   it(name: string, modifier: (test: Test) => any, inner: (state: STATE) => Promise<void> | void): void;
 }
 
 declare global {
-  type DescribeFunction = ((name: string, inner: () => void) => void) & {
-    fail(condition: boolean): DescribeFunction;
-    skip(condition: boolean): DescribeFunction;
-    fixme(condition: boolean): DescribeFunction;
-    flaky(condition: boolean): DescribeFunction;
-    slow(): DescribeFunction;
-    repeat(n: number): DescribeFunction;
-  };
+  const describe: DescribeFunction['describe'];
+  const fdescribe: DescribeFunction['describe'];
+  const xdescribe: DescribeFunction['describe'];
 
-  const describe: DescribeFunction;
-  const fdescribe: DescribeFunction;
-  const xdescribe: DescribeFunction;
-  const it: Describers<TestState & WorkerState & FixtureParameters>['it'];
-  const fit: Describers<TestState & WorkerState & FixtureParameters>['it'];
-  const xit: Describers<TestState & WorkerState & FixtureParameters>['it'];
+  const it: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
+  const fit: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
+  const xit: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
 
   const beforeEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
   const afterEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;

test-runner/src/index.ts

@@ -85,7 +85,7 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor
   const testCollector = new TestCollector(files, matrix, config);
   const suite = testCollector.suite;
   if (config.forbidOnly) {
-    const hasOnly = suite.findTest(t => t.only) || suite.eachSuite(s => s.only);
+    const hasOnly = suite.findTest(t => t._only) || suite.eachSuite(s => s._only);
     if (hasOnly)
       return 'forbid-only';
   }

test-runner/src/reporters/base.ts

@@ -137,7 +137,7 @@ export class BaseReporter implements Reporter  {
         continue;
       if (result.status === 'timedOut') {
         tokens.push('');
-        tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), '    '));
+        tokens.push(indent(colors.red(`Timeout of ${test._timeout}ms exceeded.`), '    '));
       } else {
         const stack = result.error.stack;
         if (stack) {

test-runner/src/reporters/json.ts

@@ -49,9 +49,9 @@ class JSONReporter extends BaseReporter {
     return {
       title: test.title,
       file: test.file,
-      only: test.only,
-      slow: test.slow,
-      timeout: test.timeout,
+      only: test.isOnly(),
+      slow: test.isSlow(),
+      timeout: test.timeout(),
       results: test.results.map(r => this._serializeTestResult(r))
     };
   }

test-runner/src/reporters/pytest.ts

@@ -173,7 +173,7 @@ class PytestReporter extends BaseReporter {
   }
 
   private _id(test: Test): string {
-    for (let suite = test.suite; suite; suite = suite.parent) {
+    for (let suite = test.parent; suite; suite = suite.parent) {
       if (this._suiteIds.has(suite))
         return this._suiteIds.get(suite);
     }

test-runner/src/spec.ts

@@ -63,30 +63,31 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
     if (metaFn)
       metaFn(test);
     test.file = file;
-    test.timeout = timeout;
+    test._timeout = timeout;
     const only = specs._only && specs._only[0];
     if (only)
-      test.only = true;
+      test._only = true;
     if (!only && specs._skip && specs._skip[0])
       test._skipped = true;
     suite._addTest(test);
     return test;
   });
 
-  const describe = specBuilder(['skip', 'fixme', 'flaky', 'only', 'slow'], (specs, title, fn) => {
+  const describe = specBuilder(['_skip', '_only'], (specs: any, title: string, metaFn: (suite: Suite) => void | Function, fn?: Function) => {
+    if (typeof fn !== 'function') {
+      fn = metaFn;
+      metaFn = null;
+    }
     const child = new Suite(title, suites[0]);
+    if (metaFn)
+      metaFn(child);
     suites[0]._addSuite(child);
     child.file = file;
-    child._slow = specs.slow && specs.slow[0];
-    const only = specs.only && specs.only[0];
+    const only = specs._only && specs._only[0];
     if (only)
-      child.only = true;
-    if (!only && specs.skip && specs.skip[0])
+      child._only = true;
+    if (!only && specs._skip && specs._skip[0])
       child._skipped = true;
-    if (!only && specs.fixme && specs.fixme[0])
-      child._skipped = true;
-    if (specs.flaky && specs.flaky[0])
-      child._flaky = true;
     suites.unshift(child);
     fn();
     suites.shift();
@@ -97,8 +98,8 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
   (global as any).beforeAll = fn => suite._addHook('beforeAll', fn);
   (global as any).afterAll = fn => suite._addHook('afterAll', fn);
   (global as any).describe = describe;
-  (global as any).fdescribe = describe.only(true);
-  (global as any).xdescribe = describe.skip(true);
+  (global as any).fdescribe = describe._only(true);
+  (global as any).xdescribe = describe._skip(true);
   (global as any).it = it;
   (global as any).fit = it._only(true);
   (global as any).xit = it._skip(true);

test-runner/src/test.ts

@@ -18,37 +18,23 @@ export type Configuration = { name: string, value: string }[];
 
 type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
 
-export class Test {
-  suite: Suite;
+export class Runnable {
   title: string;
   file: string;
-  only = false;
-  timeout = 0;
-  fn: Function;
-  results: TestResult[] = [];
+  parent?: Suite;
 
-  _id: string;
-  // Skipped & flaky are resolved based on options in worker only
-  // We will compute them there and send to the runner (front-end)
+  _only = false;
   _skipped = false;
   _flaky = false;
   _slow = false;
   _expectedStatus: TestStatus = 'passed';
 
-  _overriddenFn: Function;
-  _startTime: number;
-
-  constructor(title: string, fn: Function) {
-    this.title = title;
-    this.fn = fn;
+  isOnly(): boolean {
+    return this._only;
   }
 
-  titlePath(): string[] {
-    return [...this.suite.titlePath(), this.title];
-  }
-
-  fullTitle(): string {
-    return this.titlePath().join(' ');
+  isSlow(): boolean {
+    return this._slow;
   }
 
   slow(): void;
@@ -56,7 +42,7 @@ export class Test {
   slow(description: string): void;
   slow(condition: boolean, description: string): void;
   slow(arg?: boolean | string, description?: string) {
-    const condition = typeof arg === 'boolean' ? arg : true;
+    const { condition } = this._interpretCondition(arg, description);
     if (condition)
       this._slow = true;
   }
@@ -66,7 +52,7 @@ export class Test {
   skip(description: string): void;
   skip(condition: boolean, description: string): void;
   skip(arg?: boolean | string, description?: string) {
-    const condition = typeof arg === 'boolean' ? arg : true;
+    const { condition } = this._interpretCondition(arg, description);
     if (condition)
       this._skipped = true;
   }
@@ -76,7 +62,7 @@ export class Test {
   fixme(description: string): void;
   fixme(condition: boolean, description: string): void;
   fixme(arg?: boolean | string, description?: string) {
-    const condition = typeof arg === 'boolean' ? arg : true;
+    const { condition } = this._interpretCondition(arg, description);
     if (condition)
       this._skipped = true;
   }
@@ -86,7 +72,7 @@ export class Test {
   flaky(description: string): void;
   flaky(condition: boolean, description: string): void;
   flaky(arg?: boolean | string, description?: string) {
-    const condition = typeof arg === 'boolean' ? arg : true;
+    const { condition } = this._interpretCondition(arg, description);
     if (condition)
       this._flaky = true;
   }
@@ -96,11 +82,64 @@ export class Test {
   fail(description: string): void;
   fail(condition: boolean, description: string): void;
   fail(arg?: boolean | string, description?: string) {
-    const condition = typeof arg === 'boolean' ? arg : true;
+    const { condition } = this._interpretCondition(arg, description);
     if (condition)
       this._expectedStatus = 'failed';
   }
 
+  private _interpretCondition(arg?: boolean | string, description?: string): { condition: boolean, description?: string } {
+    if (arg === undefined && description === undefined)
+      return { condition: true };
+    if (typeof arg === 'string')
+      return { condition: true, description: arg };
+    return { condition: !!arg, description };
+  }
+
+  _isSkipped(): boolean {
+    return this._skipped || (this.parent && this.parent._isSkipped());
+  }
+
+  _isSlow(): boolean {
+    return this._slow || (this.parent && this.parent._isSlow());
+  }
+
+  _isFlaky(): boolean {
+    return this._flaky || (this.parent && this.parent._isFlaky());
+  }
+
+  titlePath(): string[] {
+    if (!this.parent)
+      return [];
+    return [...this.parent.titlePath(), this.title];
+  }
+
+  fullTitle(): string {
+    return this.titlePath().join(' ');
+  }
+
+  _copyFrom(other: Runnable) {
+    this.file = other.file;
+    this._only = other._only;
+    this._flaky = other._flaky;
+    this._skipped = other._skipped;
+    this._slow = other._slow;
+  }
+}
+
+export class Test extends Runnable {
+  fn: Function;
+  results: TestResult[] = [];
+  _id: string;
+  _overriddenFn: Function;
+  _startTime: number;
+  _timeout = 0;
+
+  constructor(title: string, fn: Function) {
+    super();
+    this.title = title;
+    this.fn = fn;
+  }
+
   _appendResult(): TestResult {
     const result: TestResult = {
       duration: 0,
@@ -113,13 +152,17 @@ export class Test {
     return result;
   }
 
+  timeout(): number {
+    return this._timeout;
+  }
+
   _ok(): boolean {
-    if (this._skipped || this.suite._isSkipped())
+    if (this._isSkipped())
       return true;
     const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus);
     if (!hasFailedResults)
       return true;
-    if (!this._flaky)
+    if (!this._isFlaky())
       return false;
     const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus);
     return hasPassedResults;
@@ -131,12 +174,8 @@ export class Test {
 
   _clone(): Test {
     const test = new Test(this.title, this.fn);
-    test.suite = this.suite;
-    test.only = this.only;
-    test.file = this.file;
-    test.timeout = this.timeout;
-    test._flaky = this._flaky;
-    test._slow = this._slow;
+    test._copyFrom(this);
+    test._timeout = this._timeout;
     test._overriddenFn = this._overriddenFn;
     return test;
   }
@@ -152,36 +191,21 @@ export type TestResult = {
   data: any;
 }
 
-export class Suite {
-  title: string;
-  parent?: Suite;
+export class Suite extends Runnable {
   suites: Suite[] = [];
   tests: Test[] = [];
-  only = false;
-  file: string;
-  _flaky = false;
-  _slow = false;
   configuration: Configuration;
-
-  // Skipped & flaky are resolved based on options in worker only
-  // We will compute them there and send to the runner (front-end)
-  _skipped = false;
   _configurationString: string;
 
   _hooks: { type: string, fn: Function } [] = [];
   _entries: (Suite | Test)[] = [];
 
   constructor(title: string, parent?: Suite) {
+    super();
     this.title = title;
     this.parent = parent;
   }
 
-  titlePath(): string[] {
-    if (!this.parent)
-      return [];
-    return [...this.parent.titlePath(), this.title];
-  }
-
   total(): number {
     let count = 0;
     this.findTest(fn => {
@@ -190,20 +214,8 @@ export class Suite {
     return count;
   }
 
-  _isSkipped(): boolean {
-    return this._skipped || (this.parent && this.parent._isSkipped());
-  }
-
-  _isSlow(): boolean {
-    return this._slow || (this.parent && this.parent._isSlow());
-  }
-
-  _isFlaky(): boolean {
-    return this._flaky || (this.parent && this.parent._isFlaky());
-  }
-
   _addTest(test: Test) {
-    test.suite = this;
+    test.parent = this;
     this.tests.push(test);
     this._entries.push(test);
   }
@@ -236,11 +248,7 @@ export class Suite {
 
   _clone(): Suite {
     const suite = new Suite(this.title);
-    suite.only = this.only;
-    suite.file = this.file;
-    suite._flaky = this._flaky;
-    suite._skipped = this._skipped;
-    suite._slow = this._slow;
+    suite._copyFrom(this);
     return suite;
   }
 
@@ -259,7 +267,7 @@ export class Suite {
   _hasTestsToRun(): boolean {
     let found = false;
     this.findTest(test => {
-      if (!test._skipped) {
+      if (!test._isSkipped()) {
         found = true;
         return true;
       }

test-runner/src/testCollector.ts

@@ -110,7 +110,6 @@ export class TestCollector {
 
   private _cloneSuite(suite: Suite, tests: Set<Test>) {
     const copy = suite._clone();
-    copy.only = suite.only;
     for (const entry of suite._entries) {
       if (entry instanceof Suite) {
         copy._addSuite(this._cloneSuite(entry, tests));
@@ -121,7 +120,6 @@ export class TestCollector {
         if (this._grep && !this._grep.test(test.fullTitle()))
           continue;
         const testCopy = test._clone();
-        testCopy.only = test.only;
         copy._addTest(testCopy);
       }
     }
@@ -129,8 +127,8 @@ export class TestCollector {
   }
 
   private _filterOnly(suite) {
-    const onlySuites = suite.suites.filter(child => this._filterOnly(child) || child.only);
-    const onlyTests = suite.tests.filter(test => test.only);
+    const onlySuites = suite.suites.filter((child: Suite) => this._filterOnly(child) || child._only);
+    const onlyTests = suite.tests.filter((test: Test) => test._only);
     if (onlySuites.length || onlyTests.length) {
       suite.suites = onlySuites;
       suite.tests = onlyTests;

test-runner/src/testRunner.ts

@@ -154,9 +154,9 @@ export class TestRunner extends EventEmitter {
     this._testId = id;
     // We only know resolved skipped/flaky value in the worker,
     // send it to the runner.
-    test._skipped = test._skipped || test.suite._isSkipped();
-    test._flaky = test._flaky || test.suite._isFlaky();
-    test._slow = test._slow || test.suite._isSlow();
+    test._skipped = test._isSkipped();
+    test._flaky = test._isFlaky();
+    test._slow = test._isSlow();
     this.emit('testBegin', {
       id,
       skipped: test._skipped,
@@ -184,10 +184,10 @@ export class TestRunner extends EventEmitter {
     try {
       const testInfo = { config: this._config, test, result };
       if (!this._trialRun) {
-        await this._runHooks(test.suite, 'beforeEach', 'before', testInfo);
-        const timeout = test._slow || test.suite._isSlow() ? this._timeout * 3 : this._timeout;
+        await this._runHooks(test.parent, 'beforeEach', 'before', testInfo);
+        const timeout = test._isSlow() ? this._timeout * 3 : this._timeout;
         await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo);
-        await this._runHooks(test.suite, 'afterEach', 'after', testInfo);
+        await this._runHooks(test.parent, 'afterEach', 'after', testInfo);
       } else {
         result.status = result.expectedStatus;
       }

test-runner/test/assets/nested-skip.js

@@ -15,7 +15,9 @@
  */
 require('../../');
 
-describe.skip(true)('skipped', () => {
+describe('skipped', suite => {
+  suite.skip(true);
+}, () => {
   it('succeeds',() => {
     expect(1 + 1).toBe(2);
   });  

test/accessibility.spec.ts

@@ -202,9 +202,10 @@ it('rich text editable fields with role should have children', test => {
   expect(snapshot.children[0]).toEqual(golden);
 });
 
-describe.skip(options.FIREFOX || options.WEBKIT)('contenteditable', () => {
-  // Firefox does not support contenteditable="plaintext-only".
-  // WebKit rich text accessibility is iffy
+describe('contenteditable', suite => {
+  suite.skip(options.FIREFOX, 'Firefox does not support contenteditable="plaintext-only"');
+  suite.skip(options.WEBKIT, 'WebKit rich text accessibility is iffy');
+}, () => {
   it('plain text field with role should not have children', async function({page}) {
     await page.setContent(`
       <div contenteditable="plaintext-only" role='textbox'>Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);

test/browsercontext-device.spec.ts

@@ -16,7 +16,9 @@
  */
 import { options } from './playwright.fixtures';
 
-describe.skip(options.FIREFOX)('device', () => {
+describe('device', suite => {
+  suite.skip(options.FIREFOX);
+}, () => {
   it('should work', async ({playwright, browser, server}) => {
     const iPhone = playwright.devices['iPhone 6'];
     const context = await browser.newContext({ ...iPhone });

test/browsercontext-viewport-mobile.spec.ts

@@ -17,7 +17,9 @@
 
 import { options } from './playwright.fixtures';
 
-describe.skip(options.FIREFOX)('mobile viewport', () => {
+describe('mobile viewport', suite => {
+  suite.skip(options.FIREFOX);
+}, () => {
   it('should support mobile emulation', async ({playwright, browser, server}) => {
     const iPhone = playwright.devices['iPhone 6'];
     const context = await browser.newContext({ ...iPhone });

test/browsertype-connect.spec.ts

@@ -19,7 +19,10 @@ import { options } from './playwright.fixtures';
 import utils from './utils';
 import './remoteServer.fixture';
 
-describe.skip(options.WIRE).slow()('connect', () => {
+describe('connect', suite => {
+  suite.skip(options.WIRE);
+  suite.slow();
+}, () => {
   it('should be able to reconnect to a browser', async ({browserType, remoteServer, server}) => {
     {
       const browser = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint() });

test/browsertype-launch-server.spec.ts

@@ -17,7 +17,9 @@
 
 import { options } from './playwright.fixtures';
 
-describe.skip(options.WIRE)('lauch server', () => {
+describe('lauch server', suite => {
+  suite.skip(options.WIRE);
+}, () => {
   it('should work', async ({browserType, defaultBrowserOptions}) => {
     const browserServer = await browserType.launchServer(defaultBrowserOptions);
     expect(browserServer.wsEndpoint()).not.toBe(null);

test/chromium-css-coverage.spec.ts

@@ -15,7 +15,9 @@
  */
 import { options } from './playwright.fixtures';
 
-describe.skip(!options.CHROMIUM)('oopif', () => {
+describe('oopif', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should work', async function({browserType, page, server}) {
     await page.coverage.startCSSCoverage();
     await page.goto(server.PREFIX + '/csscoverage/simple.html');

test/chromium-js-coverage.spec.ts

@@ -22,7 +22,9 @@ async function({page}) {
   expect(page.coverage).toBe(null);
 });
 
-describe.skip(!options.CHROMIUM)('oopif', () => {
+describe('oopif', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should work', async function({page, server}) {
     await page.coverage.startJSCoverage();
     await page.goto(server.PREFIX + '/jscoverage/simple.html', { waitUntil: 'load' });

test/chromium/chromium.spec.ts

@@ -16,7 +16,9 @@
 import { options } from '../playwright.fixtures';
 import type { ChromiumBrowserContext } from '../..';
 
-describe.skip(!options.CHROMIUM)('chromium', () => {
+describe('chromium', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should create a worker from a service worker', async ({page, server, context}) => {
     const [worker] = await Promise.all([
       (context as ChromiumBrowserContext).waitForEvent('serviceworker'),

test/chromium/oopif.spec.ts

@@ -26,7 +26,9 @@ registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, te
   await browser.close();
 });
 
-describe.skip(!options.CHROMIUM)('oopif', () => {
+describe('oopif', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should report oopif frames', async function({browser, page, server}) {
     await page.goto(server.PREFIX + '/dynamic-oopif.html');
     expect(await countOOPIFs(browser)).toBe(1);

test/chromium/session.spec.ts

@@ -16,7 +16,9 @@
 import { options } from '../playwright.fixtures';
 import type { ChromiumBrowserContext, ChromiumBrowser } from '../../types/types';
 
-describe.skip(!options.CHROMIUM)('session', () => {
+describe('session', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should work', async function({page}) {
     const client = await (page.context() as ChromiumBrowserContext).newCDPSession(page);
 

test/chromium/tracing.spec.ts

@@ -34,7 +34,9 @@ registerFixture('outputFile', async ({tmpDir}, test) => {
     fs.unlinkSync(outputFile);
 });
 
-describe.skip(!options.CHROMIUM)('oopif', () => {
+describe('oopif', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should output a trace', async ({browser, page, server, outputFile}) => {
     await (browser as ChromiumBrowser).startTracing(page, {screenshots: true, path: outputFile});
     await page.goto(server.PREFIX + '/grid.html');

test/electron/electron-app.spec.ts

@@ -20,7 +20,9 @@ import './electron.fixture';
 import path from 'path';
 const electronName = process.platform === 'win32' ? 'electron.cmd' : 'electron';
 
-describe.skip(!options.CHROMIUM)('electron app', () => {
+describe('electron app', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should fire close event', async ({ playwright }) => {
     const electronPath = path.join(__dirname, '..', '..', 'node_modules', '.bin', electronName);
     const application = await playwright.electron.launch(electronPath, {

test/electron/electron-window.spec.ts

@@ -17,7 +17,9 @@
 import { options } from '../playwright.fixtures';
 import './electron.fixture';
 
-describe.skip(!options.CHROMIUM)('electron window', () => {
+describe('electron window', suite => {
+  suite.skip(!options.CHROMIUM);
+}, () => {
   it('should click the button', async ({window, server}) => {
     await window.goto(server.PREFIX + '/input/button.html');
     await window.click('button');

test/elementhandle-screenshot.spec.ts

@@ -24,7 +24,9 @@ import fs from 'fs';
 // Firefox headful produces a different image.
 const ffheadful = options.FIREFOX && !options.HEADLESS;
 
-describe.skip(ffheadful)('element screenshot', () => {
+describe('element screenshot', suite => {
+  suite.skip(ffheadful);
+}, () => {
   it('should work', async ({page, server, golden}) => {
     await page.setViewportSize({width: 500, height: 500});
     await page.goto(server.PREFIX + '/grid.html');

test/fixtures.spec.ts

@@ -32,7 +32,10 @@ it('should close the browser when the node process closes', test => {
   // so we don't check it here.
 });
 
-describe.skip(WIN || !options.HEADLESS).slow()('fixtures', () => {
+describe('fixtures', suite => {
+  suite.skip(WIN || !options.HEADLESS);
+  suite.slow();
+}, () => {
   // Cannot reliably send signals on Windows.
   it('should report browser close signal', async ({remoteServer}) => {
     const pid = await remoteServer.out('pid');

test/page-event-crash.spec.ts

@@ -26,7 +26,10 @@ function crash(pageImpl, browserName) {
     pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
 }
 
-describe.fixme(options.WIRE).flaky(options.FIREFOX && WIN)('', () => {
+describe('', suite => {
+  suite.fixme(options.WIRE);
+  suite.flaky(options.FIREFOX && WIN);
+}, () => {
   it('should emit crash event when page crashes', async ({page, browserName, toImpl}) => {
     await page.setContent(`<div>This page should crash</div>`);
     crash(toImpl(page), browserName);

test/page-screenshot.spec.ts

@@ -23,7 +23,9 @@ import fs from 'fs';
 // Firefox headful produces a different image.
 const ffheadful = options.FIREFOX && !options.HEADLESS;
 
-describe.skip(ffheadful)('page screenshot', () => {
+describe('page screenshot', suite => {
+  suite.skip(ffheadful);
+}, () => {
   it('should work', async ({page, server, golden}) => {
     await page.setViewportSize({width: 500, height: 500});
     await page.goto(server.PREFIX + '/grid.html');

test/permissions.spec.ts

@@ -20,7 +20,9 @@ function getPermission(page, name) {
   return page.evaluate(name => navigator.permissions.query({name}).then(result => result.state), name);
 }
 
-describe.skip(options.WEBKIT)('permissions', () => {
+describe('permissions', suite => {
+  suite.skip(options.WEBKIT);
+}, () => {
   it('should be prompt by default', async ({page, server, context}) => {
     // Permissions API is not implemented in WebKit (see https://developer.mozilla.org/en-US/docs/Web/API/Permissions_API)
     await page.goto(server.EMPTY_PAGE);

test/screencast.spec.ts

@@ -172,7 +172,10 @@ class VideoPlayer {
   }
 }
 
-describe.skip(options.WIRE).fixme(options.CHROMIUM)('screencast', () => {
+describe('screencast', suite => {
+  suite.skip(options.WIRE);
+  suite.fixme(options.CHROMIUM);
+}, () => {
   it('should capture static page', test => {
     test.fixme();
   }, async ({page, tmpDir, videoPlayer, toImpl}) => {

test/slowmo.spec.ts

@@ -44,7 +44,9 @@ async function checkPageSlowMo(toImpl, page, task) {
   `);
   await checkSlowMo(toImpl, page, task);
 }
-describe.skip(options.WIRE)('slowMo', () => {
+describe('slowMo', suite => {
+  suite.skip(options.WIRE);
+}, () => {
   it('Page SlowMo $$eval', async ({page, toImpl}) => {
     await checkPageSlowMo(toImpl, page, () => page.$$eval('button', () => void 0));
   });