From 27f4ef566e7b200fedb8813f98f34f81348ed742 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Wed, 8 Sep 2021 07:44:26 -0700 Subject: [PATCH] docs: test parallel/failures docs (#8706) --- docs/src/test-api/class-testconfig.md | 2 +- docs/src/test-api/class-testinfo.md | 2 +- docs/src/test-api/class-testproject.md | 2 +- docs/src/test-cli-js.md | 2 +- docs/src/test-parallel-js.md | 144 ++------------ docs/src/test-reporter-api/class-testcase.md | 2 +- .../src/test-reporter-api/class-testresult.md | 2 +- docs/src/test-retries-js.md | 181 +++++++++++++++++- types/test.d.ts | 6 +- types/testReporter.d.ts | 4 +- 10 files changed, 203 insertions(+), 144 deletions(-) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 6f50da901d..27bef1ceee 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -219,7 +219,7 @@ Tests that took more than `threshold` milliseconds are considered slow, and the ## property: TestConfig.retries - type: <[int]> -The maximum number of retry attempts given to failed tests. Learn more about [test retries](./test-retries.md). +The maximum number of retry attempts given to failed tests. Learn more about [test retries](./test-retries.md#retries). ## property: TestConfig.shard - type: <[Object]> diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 2a1a23ece4..c0cc611ebb 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -213,7 +213,7 @@ Specifies a unique repeat index when running in "repeat each" mode. This mode is ## property: TestInfo.retry - type: <[int]> -Specifies the retry number when the test is retried after a failure. The first test run has [`property: TestInfo.retry`] equal to zero, the first retry has it equal to one, and so on. Learn more about [retries](./test-retries.md). +Specifies the retry number when the test is retried after a failure. The first test run has [`property: TestInfo.retry`] equal to zero, the first retry has it equal to one, and so on. Learn more about [retries](./test-retries.md#retries). ## method: TestInfo.setTimeout diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 9a60a0d768..a6feb19d4a 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -159,7 +159,7 @@ The number of times to repeat each test, useful for debugging flaky tests. ## property: TestProject.retries - type: <[int]> -The maximum number of retry attempts given to failed tests. Learn more about [test retries](./test-retries.md). +The maximum number of retry attempts given to failed tests. Learn more about [test retries](./test-retries.md#retries). ## property: TestProject.testDir - type: <[string]> diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 35152ec217..8f83014435 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -112,7 +112,7 @@ Complete set of Playwright Test options is available in the [configuration file] - `--reporter `: Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. -- `--retries `: The maximum number of [retries](./test-retries.md) for flaky tests, defaults to zero (no retries). +- `--retries `: The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). - `--shard `: [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`. diff --git a/docs/src/test-parallel-js.md b/docs/src/test-parallel-js.md index b0d7794109..dc115dd58a 100644 --- a/docs/src/test-parallel-js.md +++ b/docs/src/test-parallel-js.md @@ -5,9 +5,13 @@ title: "Parallelism and sharding" Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time. -By default, test files are run in parallel. Tests in a single file are run in order, in the same worker process. You can control the number of [parallel worker processes](#limit-workers) or [disable parallelism](#disable-parallelism) altogether. +- By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process. +- Group tests with [`test.describe.parallel`](#parallelize-tests-in-a-single-file) to run **tests in a single file** in parallel. +- To **disable** parallelism limit the number of [workers to one](#disable-parallelism). -You can [limit the number of failures](#limit-failures-and-fail-fast) in the whole test suite to avoid wasting resources and "fail fast". +You can control the number of [parallel worker processes](#limit-workers) and [limit the number of failures](#limit-failures-and-fail-fast) in the whole test suite for efficiency. + + ## Worker processes @@ -15,6 +19,8 @@ All tests run in worker processes. These processes are OS processes, running ind You can't communicate between the workers. Playwright Test reuses a single worker as much as it can to make testing faster, so multiple test files are usually run in a single worker one after another. +Workers are always shutdown after a [test failure](./test-retries.md#failures) to guarantee pristine environment for following tests. + ## Limit workers You can control the maximum number of parallel worker processes via [command line](./test-cli.md) or in the [configuration file](./test-configuration.md). @@ -57,143 +63,27 @@ You can disable any parallelism by allowing just a single worker at any time. Ei npx playwright test --workers=1 ``` -## Failed tests, retries and serial mode +## Parallelize tests in a single file -Should any test fail, Playwright Test will discard entire worker process along with the browsers used and will start a new one. Testing will continue in the new worker process, starting with retrying the failed test, or from the next test if retries are disabled. +By default, tests in a single file are run in order. If you have many independent tests in a single file, you might want to run them in parallel with [`method: Test.describe.parallel`]. -This scheme works perfectly for independent tests and guarantees that failing tests can't affect healthy ones. Consider the following snippet: +Note that parallel tests are executed in separate worker processes and cannot share any state or global variables. Each test executes all relevant hooks just for itself, including `beforeAll` and `afterAll`. ```js js-flavor=js const { test } = require('@playwright/test'); -test.describe('suite', () => { - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); +test.describe.parallel('suite', () => { + test('runs in parallel 1', async ({ page }) => { /* ... */ }); + test('runs in parallel 2', async ({ page }) => { /* ... */ }); }); ``` ```js js-flavor=ts import { test } from '@playwright/test'; -test.describe('suite', () => { - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); -}); -``` - -Tests will run in the following way: -* Worker process #1: - * `first good` passes - * `second flaky` fails -* Worker process #2: - * If [retries](./test-retries.md) are enabled, `second flaky` is retried and passes - * `third good` passes - -### Serial mode - -Use [`method: Test.describe.serial`] to group dependent tests to ensure they will always run together and in order. If one of the tests fails, all subsequent tests are skipped. All tests in the group are retried together. - -This is useful for dependent tests that cannot be run in isolation. For example, restarting after the failure from the second test might not work. - -Consider the following snippet that uses `test.describe.serial`: - -```js js-flavor=js -const { test } = require('@playwright/test'); - -test.describe.serial('suite', () => { - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); -}); -``` - -```js js-flavor=ts -import { test } from '@playwright/test'; - -test.describe.serial('suite', () => { - test('first good', async ({ page }) => { /* ... */ }); - test('second flaky', async ({ page }) => { /* ... */ }); - test('third good', async ({ page }) => { /* ... */ }); -}); -``` - -When running without [retries](./test-retries.md), all tests after the failure are skipped: -* Worker process #1: - * `first good` passes - * `second flaky` fails - * `third good` is skipped entirely - -When running with [retries](./test-retries.md), all tests are retried together: -* Worker process #1: - * `first good` passes - * `second flaky` fails - * `third good` is skipped -* Worker process #2: - * `first good` passes again - * `second flaky` passes - * `third good` passes - -:::note -It is usually better to make your tests isolated, so they can be efficiently run and retried independently. -::: - -### Reusing single page between tests - -Playwright Test creates an isolated [Page] object for each test. However, if you'd like to reuse a single [Page] object between multiple tests, you can create your own in the [`method: Test.beforeAll`] and close it in [`method: Test.afterAll`]. - -```js js-flavor=js -// example.spec.js -// @ts-check - -const { test } = require('@playwright/test'); - -test.describe.serial('use the same page', () => { - /** @type {import('@playwright/test').Page} */ - let page; - - test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - }); - - test.afterAll(async () => { - await page.close(); - }); - - test('runs first', async () => { - await page.goto('https://playwright.dev/'); - }); - - test('runs second', async () => { - await page.click('text=Get Started'); - }); -}); -``` - -```js js-flavor=ts -// example.spec.ts - -import { test, Page } from '@playwright/test'; - -test.describe.serial('use the same page', () => { - let page: Page; - - test.beforeAll(async ({ browser }) => { - page = await browser.newPage(); - }); - - test.afterAll(async () => { - await page.close(); - }); - - test('runs first', async () => { - await page.goto('https://playwright.dev/'); - }); - - test('runs second', async () => { - await page.click('text=Get Started'); - }); +test.describe.parallel('suite', () => { + test('runs in parallel 1', async ({ page }) => { /* ... */ }); + test('runs in parallel 2', async ({ page }) => { /* ... */ }); }); ``` diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 783ec06701..cd0e3d192c 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -51,7 +51,7 @@ Results for each run of this test. The maximum number of retries given to this test in the configuration. -Learn more about [test retries](./test-retries.md). +Learn more about [test retries](./test-retries.md#retries). ## property: TestCase.timeout - type: <[float]> diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index e0d4f870ac..7a96d057e0 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -27,7 +27,7 @@ An error thrown during the test execution, if any. When test is retries multiple times, each retry attempt is given a sequential number. -Learn more about [test retries](./test-retries.md). +Learn more about [test retries](./test-retries.md#retries). ## property: TestResult.startTime - type: <[Date]> diff --git a/docs/src/test-retries-js.md b/docs/src/test-retries-js.md index 62da755f6b..5c22faa629 100644 --- a/docs/src/test-retries-js.md +++ b/docs/src/test-retries-js.md @@ -3,9 +3,63 @@ id: test-retries title: "Test retry" --- -Playwright Test will retry tests if they failed. Pass the maximum number of retries when running the tests, or set them in the [configuration file](./test-configuration.md). + + +## Failures + +Playwright Test runs tests in worker processes. These processes are OS processes, running independently, orchestrated by the test runner. All workers have identical environments and each starts its own browser. + +Consider the following snippet: + +```js js-flavor=js +const { test } = require('@playwright/test'); + +test.describe('suite', () => { + test('first good', async ({ page }) => { /* ... */ }); + test('second flaky', async ({ page }) => { /* ... */ }); + test('third good', async ({ page }) => { /* ... */ }); +}); +``` + +```js js-flavor=ts +import { test } from '@playwright/test'; + +test.describe('suite', () => { + test('first good', async ({ page }) => { /* ... */ }); + test('second flaky', async ({ page }) => { /* ... */ }); + test('third good', async ({ page }) => { /* ... */ }); +}); +``` + +When **all tests pass**, they will run in order in the same worker process. +* Worker process starts + * `first good` passes + * `second flaky` passes + * `third good` passes + +Should **any test fail**, Playwright Test will discard the entire worker process along with the browser and will start a new one. Testing will continue in the new worker process starting with the next test. +* Worker process #1 starts + * `first good` passes + * `second flaky` fails +* Worker process #2 starts + * `third good` passes + +If you **enable [retries](#retries)**, second worker process will start by retrying the failed test and continue from there. +* Worker process #1 starts + * `first good` passes + * `second flaky` fails +* Worker process #2 starts + * `second flaky` is retried and passes + * `third good` passes + +This scheme works perfectly for independent tests and guarantees that failing tests can't affect healthy ones. + +## Retries + +Playwright Test supports **test retries**. When enabled, failing tests will be retried multiple times until they pass, or until the maximum number of retries is reached. ```bash +# Give failing tests 3 retry attempts npx playwright test --retries=3 ``` @@ -15,6 +69,7 @@ npx playwright test --retries=3 /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { + // Give failing tests 3 retry attempts retries: 3, }; @@ -26,16 +81,130 @@ module.exports = config; import { PlaywrightTestConfig } from '@playwright/test'; const config: PlaywrightTestConfig = { + // Give failing tests 3 retry attempts retries: 3, }; export default config; ``` -Failing tests will be retried multiple times until they pass, or until the maximum number of retries is reached. Playwright Test will report all tests that failed at least once. +Playwright Test will categorize tests as follows: +- "passed" - tests that passed on the first run; +- "flaky" - tests that failed on the first run, but passed when retried; +- "failed" - tests that failed on the first run and failed all retries. ```bash -Running 1 test using 1 worker -××± -1 flaky - 1) my.test.js:1:1 +Running 3 tests using 1 worker + + ✓ example.spec.ts:4:2 › first passes (438ms) + x example.spec.ts:5:2 › second flaky (691ms) + ✓ example.spec.ts:5:2 › second flaky (522ms) + ✓ example.spec.ts:6:2 › third passes (932ms) + + 1 flaky + example.spec.ts:5:2 › second flaky + 2 passed (4s) +``` + +## Serial mode + +Use [`method: Test.describe.serial`] to group dependent tests to ensure they will always run together and in order. If one of the tests fails, all subsequent tests are skipped. All tests in the group are retried together. + +Consider the following snippet that uses `test.describe.serial`: + +```js js-flavor=js +const { test } = require('@playwright/test'); + +test.describe.serial('suite', () => { + test('first good', async ({ page }) => { /* ... */ }); + test('second flaky', async ({ page }) => { /* ... */ }); + test('third good', async ({ page }) => { /* ... */ }); +}); +``` + +```js js-flavor=ts +import { test } from '@playwright/test'; + +test.describe.serial('suite', () => { + test('first good', async ({ page }) => { /* ... */ }); + test('second flaky', async ({ page }) => { /* ... */ }); + test('third good', async ({ page }) => { /* ... */ }); +}); +``` + +When running without [retries](#retries), all tests after the failure are skipped: +* Worker process #1: + * `first good` passes + * `second flaky` fails + * `third good` is skipped entirely + +When running with [retries](#retries), all tests are retried together: +* Worker process #1: + * `first good` passes + * `second flaky` fails + * `third good` is skipped +* Worker process #2: + * `first good` passes again + * `second flaky` passes + * `third good` passes + +:::note +It is usually better to make your tests isolated, so they can be efficiently run and retried independently. +::: + +## Reuse single page between tests + +Playwright Test creates an isolated [Page] object for each test. However, if you'd like to reuse a single [Page] object between multiple tests, you can create your own in [`method: Test.beforeAll`] and close it in [`method: Test.afterAll`]. + +```js js-flavor=js +// example.spec.js +// @ts-check + +const { test } = require('@playwright/test'); + +test.describe.serial('use the same page', () => { + /** @type {import('@playwright/test').Page} */ + let page; + + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + }); + + test.afterAll(async () => { + await page.close(); + }); + + test('runs first', async () => { + await page.goto('https://playwright.dev/'); + }); + + test('runs second', async () => { + await page.click('text=Get Started'); + }); +}); +``` + +```js js-flavor=ts +// example.spec.ts + +import { test, Page } from '@playwright/test'; + +test.describe.serial('use the same page', () => { + let page: Page; + + test.beforeAll(async ({ browser }) => { + page = await browser.newPage(); + }); + + test.afterAll(async () => { + await page.close(); + }); + + test('runs first', async () => { + await page.goto('https://playwright.dev/'); + }); + + test('runs second', async () => { + await page.click('text=Get Started'); + }); +}); ``` diff --git a/types/test.d.ts b/types/test.d.ts index 140d13fac3..8f4a23ebd2 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -147,7 +147,7 @@ interface TestProject { */ repeatEach?: number; /** - * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries). + * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). */ retries?: number; /** @@ -535,7 +535,7 @@ interface TestConfig { */ repeatEach?: number; /** - * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries). + * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). */ retries?: number; /** @@ -1050,7 +1050,7 @@ export interface TestInfo { /** * Specifies the retry number when the test is retried after a failure. The first test run has * [testInfo.retry](https://playwright.dev/docs/api/class-testinfo#test-info-retry) equal to zero, the first retry has it - * equal to one, and so on. Learn more about [retries](https://playwright.dev/docs/test-retries). + * equal to one, and so on. Learn more about [retries](https://playwright.dev/docs/test-retries#retries). */ retry: number; /** diff --git a/types/testReporter.d.ts b/types/testReporter.d.ts index 6a3755870c..8f2b6c9344 100644 --- a/types/testReporter.d.ts +++ b/types/testReporter.d.ts @@ -146,7 +146,7 @@ export interface TestCase { /** * The maximum number of retries given to this test in the configuration. * - * Learn more about [test retries](https://playwright.dev/docs/test-retries). + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). */ retries: number; /** @@ -173,7 +173,7 @@ export interface TestResult { /** * When test is retries multiple times, each retry attempt is given a sequential number. * - * Learn more about [test retries](https://playwright.dev/docs/test-retries). + * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). */ retry: number; /**