mirror of
https://github.com/microsoft/playwright.git
synced 2024-10-27 13:50:25 +03:00
feat(test runner): describe.parallel (#8662)
This commit is contained in:
parent
947ff6755d
commit
e691b649de
@ -245,6 +245,78 @@ A callback that is run immediately when calling [`method: Test.describe.only`].
|
||||
|
||||
|
||||
|
||||
## method: Test.describe.parallel
|
||||
|
||||
Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel.
|
||||
|
||||
```js js-flavor=js
|
||||
test.describe.parallel('group', () => {
|
||||
test('runs in parallel 1', async ({ page }) => {
|
||||
});
|
||||
test('runs in parallel 2', async ({ page }) => {
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
```js js-flavor=ts
|
||||
test.describe.parallel('group', () => {
|
||||
test('runs in parallel 1', async ({ page }) => {
|
||||
});
|
||||
test('runs in parallel 2', async ({ page }) => {
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks.
|
||||
|
||||
### param: Test.describe.parallel.title
|
||||
- `title` <[string]>
|
||||
|
||||
Group title.
|
||||
|
||||
### param: Test.describe.parallel.callback
|
||||
- `callback` <[function]>
|
||||
|
||||
A callback that is run immediately when calling [`method: Test.describe.parallel`]. Any tests added in this callback will belong to the group.
|
||||
|
||||
|
||||
|
||||
## method: Test.describe.parallel.only
|
||||
|
||||
Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else.
|
||||
|
||||
```js js-flavor=js
|
||||
test.describe.parallel.only('group', () => {
|
||||
test('runs in parallel 1', async ({ page }) => {
|
||||
});
|
||||
test('runs in parallel 2', async ({ page }) => {
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
```js js-flavor=ts
|
||||
test.describe.parallel.only('group', () => {
|
||||
test('runs in parallel 1', async ({ page }) => {
|
||||
});
|
||||
test('runs in parallel 2', async ({ page }) => {
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the parallel tests executes all relevant hooks.
|
||||
|
||||
### param: Test.describe.parallel.only.title
|
||||
- `title` <[string]>
|
||||
|
||||
Group title.
|
||||
|
||||
### param: Test.describe.parallel.only.callback
|
||||
- `callback` <[function]>
|
||||
|
||||
A callback that is run immediately when calling [`method: Test.describe.parallel.only`]. Any tests added in this callback will belong to the group.
|
||||
|
||||
|
||||
|
||||
## method: Test.describe.serial
|
||||
|
||||
Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together.
|
||||
|
@ -161,7 +161,7 @@ export class Dispatcher {
|
||||
|
||||
let outermostSerialSuite: Suite | undefined;
|
||||
for (let parent = this._testById.get(params.failedTestId)!.test.parent; parent; parent = parent.parent) {
|
||||
if (parent._serial)
|
||||
if (parent._parallelMode === 'serial')
|
||||
outermostSerialSuite = parent;
|
||||
}
|
||||
|
||||
|
@ -472,47 +472,61 @@ function createTestGroups(rootSuite: Suite): TestGroup[] {
|
||||
// - They have a different repeatEachIndex - requires different workers.
|
||||
// - They have a different set of worker fixtures in the pool - requires different workers.
|
||||
// - They have a different requireFile - reuses the worker, but runs each requireFile separately.
|
||||
// - They belong to a parallel suite.
|
||||
|
||||
// We try to preserve the order of tests when they require different workers
|
||||
// by ordering different worker hashes sequentially.
|
||||
const workerHashToOrdinal = new Map<string, number>();
|
||||
const requireFileToOrdinal = new Map<string, number>();
|
||||
// Using the map "workerHash -> requireFile -> group" makes us preserve the natural order
|
||||
// of worker hashes and require files for the simple cases.
|
||||
const groups = new Map<string, Map<string, { general: TestGroup, parallel: TestGroup[] }>>();
|
||||
|
||||
const createGroup = (test: TestCase): TestGroup => {
|
||||
return {
|
||||
workerHash: test._workerHash,
|
||||
requireFile: test._requireFile,
|
||||
repeatEachIndex: test._repeatEachIndex,
|
||||
projectIndex: test._projectIndex,
|
||||
tests: [],
|
||||
};
|
||||
};
|
||||
|
||||
const groupById = new Map<number, TestGroup>();
|
||||
for (const projectSuite of rootSuite.suites) {
|
||||
for (const test of projectSuite.allTests()) {
|
||||
let workerHashOrdinal = workerHashToOrdinal.get(test._workerHash);
|
||||
if (!workerHashOrdinal) {
|
||||
workerHashOrdinal = workerHashToOrdinal.size + 1;
|
||||
workerHashToOrdinal.set(test._workerHash, workerHashOrdinal);
|
||||
let withWorkerHash = groups.get(test._workerHash);
|
||||
if (!withWorkerHash) {
|
||||
withWorkerHash = new Map();
|
||||
groups.set(test._workerHash, withWorkerHash);
|
||||
}
|
||||
|
||||
let requireFileOrdinal = requireFileToOrdinal.get(test._requireFile);
|
||||
if (!requireFileOrdinal) {
|
||||
requireFileOrdinal = requireFileToOrdinal.size + 1;
|
||||
requireFileToOrdinal.set(test._requireFile, requireFileOrdinal);
|
||||
}
|
||||
|
||||
const id = workerHashOrdinal * 10000 + requireFileOrdinal;
|
||||
let group = groupById.get(id);
|
||||
if (!group) {
|
||||
group = {
|
||||
workerHash: test._workerHash,
|
||||
requireFile: test._requireFile,
|
||||
repeatEachIndex: test._repeatEachIndex,
|
||||
projectIndex: test._projectIndex,
|
||||
tests: [],
|
||||
let withRequireFile = withWorkerHash.get(test._requireFile);
|
||||
if (!withRequireFile) {
|
||||
withRequireFile = {
|
||||
general: createGroup(test),
|
||||
parallel: [],
|
||||
};
|
||||
groupById.set(id, group);
|
||||
withWorkerHash.set(test._requireFile, withRequireFile);
|
||||
}
|
||||
|
||||
let insideParallel = false;
|
||||
for (let parent = test.parent; parent; parent = parent.parent)
|
||||
insideParallel = insideParallel || parent._parallelMode === 'parallel';
|
||||
|
||||
if (insideParallel) {
|
||||
const group = createGroup(test);
|
||||
group.tests.push(test);
|
||||
withRequireFile.parallel.push(group);
|
||||
} else {
|
||||
withRequireFile.general.tests.push(test);
|
||||
}
|
||||
group.tests.push(test);
|
||||
}
|
||||
}
|
||||
|
||||
// Sorting ids will preserve the natural order, because we
|
||||
// replaced hashes with ordinals according to the natural ordering.
|
||||
const ids = Array.from(groupById.keys()).sort();
|
||||
return ids.map(id => groupById.get(id)!);
|
||||
const result: TestGroup[] = [];
|
||||
for (const withWorkerHash of groups.values()) {
|
||||
for (const withRequireFile of withWorkerHash.values()) {
|
||||
if (withRequireFile.general.tests.length)
|
||||
result.push(withRequireFile.general);
|
||||
result.push(...withRequireFile.parallel);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
class ListModeReporter implements Reporter {
|
||||
|
@ -56,7 +56,7 @@ export class Suite extends Base implements reporterTypes.Suite {
|
||||
_timeout: number | undefined;
|
||||
_annotations: Annotations = [];
|
||||
_modifiers: Modifier[] = [];
|
||||
_serial = false;
|
||||
_parallelMode: 'default' | 'serial' | 'parallel' = 'default';
|
||||
|
||||
_addTest(test: TestCase) {
|
||||
test.parent = this;
|
||||
@ -110,7 +110,7 @@ export class Suite extends Base implements reporterTypes.Suite {
|
||||
suite._annotations = this._annotations.slice();
|
||||
suite._modifiers = this._modifiers.slice();
|
||||
suite._isDescribe = this._isDescribe;
|
||||
suite._serial = this._serial;
|
||||
suite._parallelMode = this._parallelMode;
|
||||
return suite;
|
||||
}
|
||||
}
|
||||
|
@ -40,6 +40,8 @@ export class TestTypeImpl {
|
||||
test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
|
||||
test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
|
||||
test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
|
||||
test.describe.parallel = wrapFunctionWithLocation(this._describe.bind(this, 'parallel'));
|
||||
test.describe.parallel.only = wrapFunctionWithLocation(this._describe.bind(this, 'parallel.only'));
|
||||
test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial'));
|
||||
test.describe.serial.only = wrapFunctionWithLocation(this._describe.bind(this, 'serial.only'));
|
||||
test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach'));
|
||||
@ -77,7 +79,7 @@ export class TestTypeImpl {
|
||||
test.expectedStatus = 'skipped';
|
||||
}
|
||||
|
||||
private _describe(type: 'default' | 'only' | 'serial' | 'serial.only', location: Location, title: string, fn: Function) {
|
||||
private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only', location: Location, title: string, fn: Function) {
|
||||
throwIfRunningInsideJest();
|
||||
const suite = currentlyLoadingFileSuite();
|
||||
if (!suite)
|
||||
@ -98,10 +100,19 @@ export class TestTypeImpl {
|
||||
child.location = location;
|
||||
suite._addSuite(child);
|
||||
|
||||
if (type === 'only' || type === 'serial.only')
|
||||
if (type === 'only' || type === 'serial.only' || type === 'parallel.only')
|
||||
child._only = true;
|
||||
if (type === 'serial' || type === 'serial.only')
|
||||
child._serial = true;
|
||||
child._parallelMode = 'serial';
|
||||
if (type === 'parallel' || type === 'parallel.only')
|
||||
child._parallelMode = 'parallel';
|
||||
|
||||
for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {
|
||||
if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')
|
||||
throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial');
|
||||
if (parent._parallelMode === 'parallel' && child._parallelMode === 'serial')
|
||||
throw errorWithLocation(location, 'describe.serial cannot be nested inside describe.parallel');
|
||||
}
|
||||
|
||||
setCurrentlyLoadingFileSuite(child);
|
||||
fn();
|
||||
|
60
tests/playwright-test/test-parallel.spec.ts
Normal file
60
tests/playwright-test/test-parallel.spec.ts
Normal file
@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Copyright Microsoft Corporation. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
|
||||
test('test.describe.parallel should throw inside test.describe.serial', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test.describe.serial('serial suite', () => {
|
||||
test.describe.parallel('parallel suite', () => {
|
||||
});
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain('a.test.ts:7:23: describe.parallel cannot be nested inside describe.serial');
|
||||
});
|
||||
|
||||
test('test.describe.parallel should work', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test.describe.parallel('parallel suite', () => {
|
||||
test('test1', async ({}, testInfo) => {
|
||||
console.log('\\n%% worker=' + testInfo.workerIndex);
|
||||
await new Promise(f => setTimeout(f, 1000));
|
||||
});
|
||||
test('test2', async ({}, testInfo) => {
|
||||
console.log('\\n%% worker=' + testInfo.workerIndex);
|
||||
await new Promise(f => setTimeout(f, 1000));
|
||||
});
|
||||
test.describe('inner suite', () => {
|
||||
test('test3', async ({}, testInfo) => {
|
||||
console.log('\\n%% worker=' + testInfo.workerIndex);
|
||||
await new Promise(f => setTimeout(f, 1000));
|
||||
});
|
||||
});
|
||||
});
|
||||
`,
|
||||
}, { workers: 3 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.output).toContain('%% worker=0');
|
||||
expect(result.output).toContain('%% worker=1');
|
||||
expect(result.output).toContain('%% worker=2');
|
||||
});
|
@ -210,3 +210,17 @@ test('test.describe.serial should work with test.fail and retries', async ({ run
|
||||
'%%three',
|
||||
]);
|
||||
});
|
||||
|
||||
test('test.describe.serial should throw inside test.describe.parallel', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test.describe.parallel('parallel suite', () => {
|
||||
test.describe.serial('serial suite', () => {
|
||||
});
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.output).toContain('a.test.ts:7:23: describe.serial cannot be nested inside describe.parallel');
|
||||
});
|
||||
|
62
types/test.d.ts
vendored
62
types/test.d.ts
vendored
@ -1663,6 +1663,68 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
|
||||
* @param callback A callback that is run immediately when calling [test.describe.serial.only(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-serial-only). Any
|
||||
* tests added in this callback will belong to the group.
|
||||
*/
|
||||
only: SuiteFunction;
|
||||
};
|
||||
/**
|
||||
* Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another,
|
||||
* but using [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel)
|
||||
* allows them to run in parallel.
|
||||
*
|
||||
* ```js js-flavor=js
|
||||
* test.describe.parallel('group', () => {
|
||||
* test('runs in parallel 1', async ({ page }) => {
|
||||
* });
|
||||
* test('runs in parallel 2', async ({ page }) => {
|
||||
* });
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* ```js js-flavor=ts
|
||||
* test.describe.parallel('group', () => {
|
||||
* test('runs in parallel 1', async ({ page }) => {
|
||||
* });
|
||||
* test('runs in parallel 2', async ({ page }) => {
|
||||
* });
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the
|
||||
* parallel tests executes all relevant hooks.
|
||||
* @param title Group title.
|
||||
* @param callback A callback that is run immediately when calling [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel). Any tests
|
||||
* added in this callback will belong to the group.
|
||||
*/
|
||||
parallel: SuiteFunction & {
|
||||
/**
|
||||
* Declares a focused group of tests that could be run in parallel. By default, tests in a single test file run one after
|
||||
* another, but using
|
||||
* [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel) allows them
|
||||
* to run in parallel. If there are some focused tests or suites, all of them will be run but nothing else.
|
||||
*
|
||||
* ```js js-flavor=js
|
||||
* test.describe.parallel.only('group', () => {
|
||||
* test('runs in parallel 1', async ({ page }) => {
|
||||
* });
|
||||
* test('runs in parallel 2', async ({ page }) => {
|
||||
* });
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* ```js js-flavor=ts
|
||||
* test.describe.parallel.only('group', () => {
|
||||
* test('runs in parallel 1', async ({ page }) => {
|
||||
* });
|
||||
* test('runs in parallel 2', async ({ page }) => {
|
||||
* });
|
||||
* });
|
||||
* ```
|
||||
*
|
||||
* Note that parallel tests are executed in separate processes and cannot share any state or global variables. Each of the
|
||||
* parallel tests executes all relevant hooks.
|
||||
* @param title Group title.
|
||||
* @param callback A callback that is run immediately when calling [test.describe.parallel.only(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel-only).
|
||||
* Any tests added in this callback will belong to the group.
|
||||
*/
|
||||
only: SuiteFunction;
|
||||
};
|
||||
};
|
||||
|
3
utils/generate_types/overrides-test.d.ts
vendored
3
utils/generate_types/overrides-test.d.ts
vendored
@ -226,6 +226,9 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
|
||||
serial: SuiteFunction & {
|
||||
only: SuiteFunction;
|
||||
};
|
||||
parallel: SuiteFunction & {
|
||||
only: SuiteFunction;
|
||||
};
|
||||
};
|
||||
skip(title: string, testFunction: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void): void;
|
||||
skip(): void;
|
||||
|
Loading…
Reference in New Issue
Block a user