feat(test runner): describe.parallel (#8662)

docs/src/test-api/class-test.md

@@ -245,6 +245,78 @@ A callback that is run immediately when calling [`method: Test.describe.only`].
 
 
 
+## method: Test.describe.parallel
+
+Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel.
+
+```js js-flavor=js
+test.describe.parallel('group', () => {
+  test('runs in parallel 1', async ({ page }) => {
+  });
+  test('runs in parallel 2', async ({ page }) => {
+  });
+});
+```
+
+```js js-flavor=ts
+test.describe.parallel('group', () => {
+  test('runs in parallel 1', async ({ page }) => {
+  });
+  test('runs in parallel 2', async ({ page }) => {
+  });
+});
+```
+
+Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks.
+
+### param: Test.describe.parallel.title
+- `title` <[string]>
+
+Group title.
+
+### param: Test.describe.parallel.callback
+- `callback` <[function]>
+
+A callback that is run immediately when calling [`method: Test.describe.parallel`]. Any tests added in this callback will belong to the group.
+
+
+
+## method: Test.describe.parallel.only
+
+Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else.
+
+```js js-flavor=js
+test.describe.parallel.only('group', () => {
+  test('runs in parallel 1', async ({ page }) => {
+  });
+  test('runs in parallel 2', async ({ page }) => {
+  });
+});
+```
+
+```js js-flavor=ts
+test.describe.parallel.only('group', () => {
+  test('runs in parallel 1', async ({ page }) => {
+  });
+  test('runs in parallel 2', async ({ page }) => {
+  });
+});
+```
+
+Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks.
+
+### param: Test.describe.parallel.only.title
+- `title` <[string]>
+
+Group title.
+
+### param: Test.describe.parallel.only.callback
+- `callback` <[function]>
+
+A callback that is run immediately when calling [`method: Test.describe.parallel.only`]. Any tests added in this callback will belong to the group.
+
+
+
 ## method: Test.describe.serial
 
 Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together.

src/test/dispatcher.ts

@@ -161,7 +161,7 @@ export class Dispatcher {
 
         let outermostSerialSuite: Suite | undefined;
         for (let parent = this._testById.get(params.failedTestId)!.test.parent; parent; parent = parent.parent) {
-          if (parent._serial)
+          if (parent._parallelMode ===  'serial')
             outermostSerialSuite = parent;
         }
 

src/test/runner.ts

@@ -472,47 +472,61 @@ function createTestGroups(rootSuite: Suite): TestGroup[] {
   // - They have a different repeatEachIndex - requires different workers.
   // - They have a different set of worker fixtures in the pool - requires different workers.
   // - They have a different requireFile - reuses the worker, but runs each requireFile separately.
+  // - They belong to a parallel suite.
 
-  // We try to preserve the order of tests when they require different workers
+  // Using the map "workerHash -> requireFile -> group" makes us preserve the natural order
-  // by ordering different worker hashes sequentially.
+  // of worker hashes and require files for the simple cases.
-  const workerHashToOrdinal = new Map<string, number>();
+  const groups = new Map<string, Map<string, { general: TestGroup, parallel: TestGroup[] }>>();
-  const requireFileToOrdinal = new Map<string, number>();
+
+  const createGroup = (test: TestCase): TestGroup => {
+    return {
+      workerHash: test._workerHash,
+      requireFile: test._requireFile,
+      repeatEachIndex: test._repeatEachIndex,
+      projectIndex: test._projectIndex,
+      tests: [],
+    };
+  };
 
-  const groupById = new Map<number, TestGroup>();
   for (const projectSuite of rootSuite.suites) {
     for (const test of projectSuite.allTests()) {
-      let workerHashOrdinal = workerHashToOrdinal.get(test._workerHash);
+      let withWorkerHash = groups.get(test._workerHash);
-      if (!workerHashOrdinal) {
+      if (!withWorkerHash) {
-        workerHashOrdinal = workerHashToOrdinal.size + 1;
+        withWorkerHash = new Map();
-        workerHashToOrdinal.set(test._workerHash, workerHashOrdinal);
+        groups.set(test._workerHash, withWorkerHash);
       }
-
+      let withRequireFile = withWorkerHash.get(test._requireFile);
-      let requireFileOrdinal = requireFileToOrdinal.get(test._requireFile);
+      if (!withRequireFile) {
-      if (!requireFileOrdinal) {
+        withRequireFile = {
-        requireFileOrdinal = requireFileToOrdinal.size + 1;
+          general: createGroup(test),
-        requireFileToOrdinal.set(test._requireFile, requireFileOrdinal);
+          parallel: [],
-      }
-
-      const id = workerHashOrdinal * 10000 + requireFileOrdinal;
-      let group = groupById.get(id);
-      if (!group) {
-        group = {
-          workerHash: test._workerHash,
-          requireFile: test._requireFile,
-          repeatEachIndex: test._repeatEachIndex,
-          projectIndex: test._projectIndex,
-          tests: [],
         };
-        groupById.set(id, group);
+        withWorkerHash.set(test._requireFile, withRequireFile);
+      }
+
+      let insideParallel = false;
+      for (let parent = test.parent; parent; parent = parent.parent)
+        insideParallel = insideParallel || parent._parallelMode === 'parallel';
+
+      if (insideParallel) {
+        const group = createGroup(test);
+        group.tests.push(test);
+        withRequireFile.parallel.push(group);
+      } else {
+        withRequireFile.general.tests.push(test);
       }
-      group.tests.push(test);
     }
   }
 
-  // Sorting ids will preserve the natural order, because we
+  const result: TestGroup[] = [];
-  // replaced hashes with ordinals according to the natural ordering.
+  for (const withWorkerHash of groups.values()) {
-  const ids = Array.from(groupById.keys()).sort();
+    for (const withRequireFile of withWorkerHash.values()) {
-  return ids.map(id => groupById.get(id)!);
+      if (withRequireFile.general.tests.length)
+        result.push(withRequireFile.general);
+      result.push(...withRequireFile.parallel);
+    }
+  }
+  return result;
 }
 
 class ListModeReporter implements Reporter {

src/test/test.ts

@@ -56,7 +56,7 @@ export class Suite extends Base implements reporterTypes.Suite {
   _timeout: number | undefined;
   _annotations: Annotations = [];
   _modifiers: Modifier[] = [];
-  _serial = false;
+  _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
 
   _addTest(test: TestCase) {
     test.parent = this;
@@ -110,7 +110,7 @@ export class Suite extends Base implements reporterTypes.Suite {
     suite._annotations = this._annotations.slice();
     suite._modifiers = this._modifiers.slice();
     suite._isDescribe = this._isDescribe;
-    suite._serial = this._serial;
+    suite._parallelMode = this._parallelMode;
     return suite;
   }
 }

src/test/testType.ts

@@ -40,6 +40,8 @@ export class TestTypeImpl {
     test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
     test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
     test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
+    test.describe.parallel = wrapFunctionWithLocation(this._describe.bind(this, 'parallel'));
+    test.describe.parallel.only = wrapFunctionWithLocation(this._describe.bind(this, 'parallel.only'));
     test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial'));
     test.describe.serial.only = wrapFunctionWithLocation(this._describe.bind(this, 'serial.only'));
     test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach'));
@@ -77,7 +79,7 @@ export class TestTypeImpl {
       test.expectedStatus = 'skipped';
   }
 
-  private _describe(type: 'default' | 'only' | 'serial' | 'serial.only', location: Location, title: string, fn: Function) {
+  private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only', location: Location, title: string, fn: Function) {
     throwIfRunningInsideJest();
     const suite = currentlyLoadingFileSuite();
     if (!suite)
@@ -98,10 +100,19 @@ export class TestTypeImpl {
     child.location = location;
     suite._addSuite(child);
 
-    if (type === 'only' || type === 'serial.only')
+    if (type === 'only' || type === 'serial.only' || type === 'parallel.only')
       child._only = true;
     if (type === 'serial' || type === 'serial.only')
-      child._serial =  true;
+      child._parallelMode = 'serial';
+    if (type === 'parallel' || type === 'parallel.only')
+      child._parallelMode = 'parallel';
+
+    for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {
+      if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')
+        throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial');
+      if (parent._parallelMode === 'parallel' && child._parallelMode === 'serial')
+        throw errorWithLocation(location, 'describe.serial cannot be nested inside describe.parallel');
+    }
 
     setCurrentlyLoadingFileSuite(child);
     fn();

tests/playwright-test/test-parallel.spec.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('test.describe.parallel should throw inside test.describe.serial', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.describe.serial('serial suite', () => {
+        test.describe.parallel('parallel suite', () => {
+        });
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('a.test.ts:7:23: describe.parallel cannot be nested inside describe.serial');
+});
+
+test('test.describe.parallel should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.describe.parallel('parallel suite', () => {
+        test('test1', async ({}, testInfo) => {
+          console.log('\\n%% worker=' + testInfo.workerIndex);
+          await new Promise(f => setTimeout(f, 1000));
+        });
+        test('test2', async ({}, testInfo) => {
+          console.log('\\n%% worker=' + testInfo.workerIndex);
+          await new Promise(f => setTimeout(f, 1000));
+        });
+        test.describe('inner suite', () => {
+          test('test3', async ({}, testInfo) => {
+            console.log('\\n%% worker=' + testInfo.workerIndex);
+            await new Promise(f => setTimeout(f, 1000));
+          });
+        });
+      });
+    `,
+  }, { workers: 3 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output).toContain('%% worker=0');
+  expect(result.output).toContain('%% worker=1');
+  expect(result.output).toContain('%% worker=2');
+});

tests/playwright-test/test-serial.spec.ts

@@ -210,3 +210,17 @@ test('test.describe.serial should work with test.fail and retries', async ({ run
     '%%three',
   ]);
 });
+
+test('test.describe.serial should throw inside test.describe.parallel', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.describe.parallel('parallel suite', () => {
+        test.describe.serial('serial suite', () => {
+        });
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('a.test.ts:7:23: describe.serial cannot be nested inside describe.parallel');
+});

types/test.d.ts

@@ -1663,6 +1663,68 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * @param callback A callback that is run immediately when calling [test.describe.serial.only(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-serial-only). Any
    * tests added in this callback will belong to the group.
    */
+  only: SuiteFunction;
+    };
+    /**
+   * Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another,
+   * but using [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel)
+   * allows them to run in parallel.
+   *
+   * ```js js-flavor=js
+   * test.describe.parallel('group', () => {
+   *   test('runs in parallel 1', async ({ page }) => {
+   *   });
+   *   test('runs in parallel 2', async ({ page }) => {
+   *   });
+   * });
+   * ```
+   *
+   * ```js js-flavor=ts
+   * test.describe.parallel('group', () => {
+   *   test('runs in parallel 1', async ({ page }) => {
+   *   });
+   *   test('runs in parallel 2', async ({ page }) => {
+   *   });
+   * });
+   * ```
+   *
+   * Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the
+   * parallel tests executes all relevant hooks.
+   * @param title Group title.
+   * @param callback A callback that is run immediately when calling [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel). Any tests
+   * added in this callback will belong to the group.
+   */
+  parallel: SuiteFunction & {
+      /**
+   * Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after
+   * another, but using
+   * [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel) allows them
+   * to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else.
+   *
+   * ```js js-flavor=js
+   * test.describe.parallel.only('group', () => {
+   *   test('runs in parallel 1', async ({ page }) => {
+   *   });
+   *   test('runs in parallel 2', async ({ page }) => {
+   *   });
+   * });
+   * ```
+   *
+   * ```js js-flavor=ts
+   * test.describe.parallel.only('group', () => {
+   *   test('runs in parallel 1', async ({ page }) => {
+   *   });
+   *   test('runs in parallel 2', async ({ page }) => {
+   *   });
+   * });
+   * ```
+   *
+   * Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the
+   * parallel tests executes all relevant hooks.
+   * @param title Group title.
+   * @param callback A callback that is run immediately when calling [test.describe.parallel.only(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel-only).
+   * Any tests added in this callback will belong to the group.
+   */
   only: SuiteFunction;
     };
   };

utils/generate_types/overrides-test.d.ts

@@ -226,6 +226,9 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
     serial: SuiteFunction & {
       only: SuiteFunction;
     };
+    parallel: SuiteFunction & {
+      only: SuiteFunction;
+    };
   };
   skip(title: string, testFunction: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void): void;
   skip(): void;