feat(shard): introduce mode: 'default' (#23023)

This mode allows a suite to opt-out from parallelism. Useful to setup
multiple suites running in parallel, with each suite not being sharded.

References #22891.

docs/src/test-api/class-test.md

@@ -284,9 +284,27 @@ Learn more about the execution modes [here](../test-parallel.md).
   test('runs second', async ({ page }) => {});
   ```
 
+* Run multiple describes in parallel, but tests inside each describe in order.
+
+  ```js
+  test.describe.configure({ mode: 'parallel' });
+
+  test.describe('A, runs in parallel with B', () => {
+    test.describe.configure({ mode: 'default' });
+    test('in order A1', async ({ page }) => {});
+    test('in order A2', async ({ page }) => {});
+  });
+
+  test.describe('B, runs in parallel with A', () => {
+    test.describe.configure({ mode: 'default' });
+    test('in order B1', async ({ page }) => {});
+    test('in order B2', async ({ page }) => {});
+  });
+  ```
+
 ### option: Test.describe.configure.mode
 * since: v1.10
-- `mode` <[TestMode]<"parallel"|"serial">>
+- `mode` <[TestMode]<"default"|"parallel"|"serial">>
 
 Execution mode. Learn more about the execution modes [here](../test-parallel.md).
 

packages/playwright-test/src/common/test.ts

@@ -50,7 +50,7 @@ export class Suite extends Base implements SuitePrivate {
   _retries: number | undefined;
   _staticAnnotations: Annotation[] = [];
   _modifiers: Modifier[] = [];
-  _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
+  _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
   _fullProject: FullProjectInternal | undefined;
   _fileId: string | undefined;
   readonly _type: 'root' | 'project' | 'file' | 'describe';

packages/playwright-test/src/common/testType.ts

@@ -124,6 +124,8 @@ export class TestTypeImpl {
     for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {
       if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')
         throw new Error('describe.parallel cannot be nested inside describe.serial');
+      if (parent._parallelMode === 'default' && child._parallelMode === 'parallel')
+        throw new Error('describe.parallel cannot be nested inside describe with default mode');
     }
 
     setCurrentlyLoadingFileSuite(child);
@@ -138,7 +140,7 @@ export class TestTypeImpl {
     suite._hooks.push({ type: name, fn, location });
   }
 
-  private _configure(location: Location, options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) {
+  private _configure(location: Location, options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) {
     throwIfRunningInsideJest();
     const suite = this._currentSuite(location, `test.describe.configure()`);
     if (!suite)
@@ -151,12 +153,14 @@ export class TestTypeImpl {
       suite._retries = options.retries;
 
     if (options.mode !== undefined) {
-      if (suite._parallelMode !== 'default')
+      if (suite._parallelMode !== 'none')
-        throw new Error('Parallel mode is already assigned for the enclosing scope.');
+        throw new Error(`"${suite._parallelMode}" mode is already assigned for the enclosing scope.`);
       suite._parallelMode = options.mode;
       for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) {
         if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel')
-          throw new Error('describe.parallel cannot be nested inside describe.serial');
+          throw new Error('describe with parallel mode cannot be nested inside describe with serial mode');
+        if (parent._parallelMode === 'default' && suite._parallelMode === 'parallel')
+          throw new Error('describe with parallel mode cannot be nested inside describe with default mode');
       }
     }
   }

packages/playwright-test/src/isomorphic/teleReceiver.ts

@@ -61,7 +61,7 @@ export type JsonSuite = {
   suites: JsonSuite[];
   tests: JsonTestCase[];
   fileId: string | undefined;
-  parallelMode: 'default' | 'serial' | 'parallel';
+  parallelMode: 'none' | 'default' | 'serial' | 'parallel';
 };
 
 export type JsonTestCase = {
@@ -383,7 +383,7 @@ export class TeleSuite implements SuitePrivate {
   _timeout: number | undefined;
   _retries: number | undefined;
   _fileId: string | undefined;
-  _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
+  _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
   readonly _type: 'root' | 'project' | 'file' | 'describe';
 
   constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') {

packages/playwright-test/src/runner/testGroups.ts

@@ -79,20 +79,20 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou
 
     // Note that a parallel suite cannot be inside a serial suite. This is enforced in TestType.
     let insideParallel = false;
-    let outerMostSerialSuite: Suite | undefined;
+    let outerMostSequentialSuite: Suite | undefined;
     let hasAllHooks = false;
     for (let parent: Suite | undefined = test.parent; parent; parent = parent.parent) {
-      if (parent._parallelMode === 'serial')
+      if (parent._parallelMode === 'serial' || parent._parallelMode === 'default')
-        outerMostSerialSuite = parent;
+        outerMostSequentialSuite = parent;
       insideParallel = insideParallel || parent._parallelMode === 'parallel';
       hasAllHooks = hasAllHooks || parent._hooks.some(hook => hook.type === 'beforeAll' || hook.type === 'afterAll');
     }
 
     if (insideParallel) {
-      if (hasAllHooks && !outerMostSerialSuite) {
+      if (hasAllHooks && !outerMostSequentialSuite) {
         withRequireFile.parallelWithHooks.tests.push(test);
       } else {
-        const key = outerMostSerialSuite || test;
+        const key = outerMostSequentialSuite || test;
         let group = withRequireFile.parallel.get(key);
         if (!group) {
           group = createGroup(test);

packages/playwright-test/types/reporterPrivate.ts

@@ -18,5 +18,5 @@ import type { Suite } from './testReporter';
 
 export interface SuitePrivate extends Suite {
   _fileId: string | undefined;
-  _parallelMode: 'default' | 'serial' | 'parallel';
+  _parallelMode: 'none' | 'default' | 'serial' | 'parallel';
 }

packages/playwright-test/types/test.d.ts

@@ -2626,9 +2626,27 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    *   test('runs second', async ({ page }) => {});
    *   ```
    *
+   * - Run multiple describes in parallel, but tests inside each describe in order.
+   *
+   *   ```js
+   *   test.describe.configure({ mode: 'parallel' });
+   *
+   *   test.describe('A, runs in parallel with B', () => {
+   *     test.describe.configure({ mode: 'default' });
+   *     test('in order A1', async ({ page }) => {});
+   *     test('in order A2', async ({ page }) => {});
+   *   });
+   *
+   *   test.describe('B, runs in parallel with A', () => {
+   *     test.describe.configure({ mode: 'default' });
+   *     test('in order B1', async ({ page }) => {});
+   *     test('in order B2', async ({ page }) => {});
+   *   });
+   *   ```
+   *
    * @param options
    */
-  configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
+  configure: (options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
   };
   /**
    * Declares a skipped test, similarly to

tests/playwright-test/shard.spec.ts

@@ -225,3 +225,62 @@ test('should skip dependency when project is sharded out', async ({ runInlineTes
     'test in tests2',
   ]);
 });
+
+test('should not shard mode:default suites', async ({ runInlineTest }) => {
+  test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22891' });
+
+  const tests = {
+    'a1.spec.ts': `
+      import { test } from '@playwright/test';
+      test('test0', async ({ }) => {
+        console.log('\\n%%test0');
+      });
+      test('test1', async ({ }) => {
+        console.log('\\n%%test1');
+      });
+    `,
+    'a2.spec.ts': `
+      import { test } from '@playwright/test';
+      test.describe.configure({ mode: 'parallel' });
+
+      test.describe(() => {
+        test.describe.configure({ mode: 'default' });
+        test.beforeAll(() => {
+          console.log('\\n%%beforeAll1');
+        });
+        test('test2', async ({ }) => {
+          console.log('\\n%%test2');
+        });
+        test('test3', async ({ }) => {
+          console.log('\\n%%test3');
+        });
+      });
+
+      test.describe(() => {
+        test.describe.configure({ mode: 'default' });
+        test.beforeAll(() => {
+          console.log('\\n%%beforeAll2');
+        });
+        test('test4', async ({ }) => {
+          console.log('\\n%%test4');
+        });
+        test('test5', async ({ }) => {
+          console.log('\\n%%test5');
+        });
+      });
+  `,
+  };
+
+  {
+    const result = await runInlineTest(tests, { shard: '2/3', workers: 1 });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(2);
+    expect(result.outputLines).toEqual(['beforeAll1', 'test2', 'test3']);
+  }
+  {
+    const result = await runInlineTest(tests, { shard: '3/3', workers: 1 });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(2);
+    expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
+  }
+});

utils/generate_types/overrides-test.d.ts

@@ -131,7 +131,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
     parallel: SuiteFunction & {
       only: SuiteFunction;
     };
-    configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
+    configure: (options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) => void;
   };
   skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
   skip(): void;