feat(test runner): shuffle order of tests with sharding seed (#30817)

docs/src/test-api/class-testconfig.md

@@ -476,6 +476,27 @@ export default defineConfig({
 ```
 
 
+## property: TestConfig.shardingSeed
+
+* since: v1.45
+- type: ?<[string]>
+
+Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests in a deterministic way can help to distribute the load more evenly.
+
+The sharding seed is a string that is used to initialize a random number generator.
+
+Learn more about [parallelism and sharding](../test-parallel.md) with Playwright Test.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  shardingSeed: 'string value'
+});
+```
+
 ## property: TestConfig.testDir
 * since: v1.10
 - type: ?<[string]>

docs/src/test-sharding-js.md

@@ -22,6 +22,12 @@ Now, if you run these shards in parallel on different computers, your test suite
 
 Note that Playwright can only shard tests that can be run in parallel. By default, this means Playwright will shard test files. Learn about other options in the [parallelism guide](./test-parallel.md).
 
+## Randomizing test order in a deterministic way
+
+By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. For example, if the first half of your tests are slower than the rest of your tests and you are using 4 shards it means that shard 1 and 2 will take significantly more time then shard 3 and 4.
+
+To aid with this problem you can pass `--sharding-seed=string-value` to randomize the order of tests in a deterministic way, which could yield better distribution of slow and fast tests across all shards.
+
 ## Merging reports from multiple shards
 
 In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.

packages/playwright/src/common/config.ts

@@ -55,6 +55,7 @@ export class FullConfigInternal {
   cliFailOnFlakyTests?: boolean;
   testIdMatcher?: Matcher;
   defineConfigWasUsed = false;
+  shardingSeed: string | null;
 
   constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) {
     if (configCLIOverrides.projects && userConfig.projects)
@@ -92,6 +93,7 @@ export class FullConfigInternal {
       workers: 0,
       webServer: null,
     };
+    this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null);
     for (const key in userConfig) {
       if (key.startsWith('@'))
         (this.config as any)[key] = (userConfig as any)[key];

packages/playwright/src/common/ipc.ts

@@ -32,6 +32,7 @@ export type ConfigCLIOverrides = {
   reporter?: ReporterDescription[];
   additionalReporters?: ReporterDescription[];
   shard?: { current: number, total: number };
+  shardingSeed?: string;
   timeout?: number;
   ignoreSnapshots?: boolean;
   updateSnapshots?: 'all'|'none'|'missing';

packages/playwright/src/program.ts

@@ -281,6 +281,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     retries: options.retries ? parseInt(options.retries, 10) : undefined,
     reporter: resolveReporterOption(options.reporter),
     shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
+    shardingSeed: options.shardingSeed ? options.shardingSeed : undefined,
     timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
     ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
     updateSnapshots: options.updateSnapshots ? 'all' as const : undefined,
@@ -358,6 +359,7 @@ const testOptions: [string, string][] = [
   ['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],
   ['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`],
   ['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`],
+  ['--sharding-seed <seed>', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`],
   ['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`],
   ['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`],
   ['--ui', `Run tests in interactive UI mode`],

packages/playwright/src/runner/loadUtils.ts

@@ -31,6 +31,7 @@ import { createTestGroups, filterForShard, type TestGroup } from './testGroups';
 import { dependenciesForTestFile } from '../transform/compilationCache';
 import { sourceMapSupport } from '../utilsBundle';
 import type { RawSourceMap } from 'source-map';
+import { shuffleWithSeed } from './shuffle';
 
 export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) {
   const config = testRun.config;
@@ -179,6 +180,9 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     for (const projectSuite of rootSuite.suites)
       testGroups.push(...createTestGroups(projectSuite, config.config.workers));
 
+    if (config.shardingSeed)
+      shuffleWithSeed(testGroups, config.shardingSeed);
+
     // Shard test groups.
     const testGroupsInThisShard = filterForShard(config.config.shard, testGroups);
     const testsInThisShard = new Set<TestCase>();

packages/playwright/src/runner/shuffle.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Shuffles the given array of items using the given seed.
+ *
+ * @param items The array of items to shuffle.
+ * @param seed The seed to use for shuffling.
+ */
+export function shuffleWithSeed(items: any[], seed: string): void {
+  const random = rng(cyrb32(seed));
+  for (let i = items.length - 1; i > 0; i--) {
+    const j = Math.floor(random() * (i + 1));
+    [items[i], items[j]] = [items[j], items[i]];
+  }
+}
+
+/**
+ * Returns a random number generator seeded with the given seed.
+ *
+ * @param seed The seed for the random number generator.
+ * @returns The random number generator.
+ */
+function rng(seed: number) {
+  const m = 2 ** 35 - 31;
+  const a = 185852;
+  let s = seed % m;
+  return function() {
+    return (s = s * a % m) / m;
+  };
+}
+
+/**
+ * Return a 32-bit hash from a string.
+ *
+ * @param str The string to hash.
+ * @returns The 32-bit hash.
+ */
+function cyrb32(str: string) {
+  let h = 0x2323;
+  for (let i = 0; i < str.length; i++) {
+    h = h ^ str.charCodeAt(i);
+    h = Math.imul(h, 2654435761);
+  }
+  return h >>> 0;
+}

packages/playwright/types/test.d.ts

@@ -1407,6 +1407,29 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
     total: number;
   };
 
+  /**
+   * Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is
+   * mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests
+   * in a deterministic way can help to distribute the load more evenly.
+   *
+   * The sharding seed is a string that is used to initialize a random number generator.
+   *
+   * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   shardingSeed: 'string value'
+   * });
+   * ```
+   *
+   */
+  shardingSeed?: string;
+
   /**
    * **NOTE** Use
    * [testConfig.snapshotPathTemplate](https://playwright.dev/docs/api/class-testconfig#test-config-snapshot-path-template)