From 3331a40647688b3f63b62754e34189eda560b3e1 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Wed, 7 Feb 2024 16:31:25 -0800 Subject: [PATCH] feat(test runner): tags/annotations (#29248) API changes: - `test(title, details, body)` where details contain `tag` and `annotation`. - similar `details` property added to `test.skip`, `test.fail`, `test.fixme`, `test.only`, `test.describe` and other `test.describe.*` variations. - `TestProject.tagFilter`/`TestConfig.tagFilter` that supports logical tag expressions with `(`, `)`, `and`, `or` and `not`. - `--tag` CLI option to filter by tags. - New annotations are available in `TestInfo.annotations` and `TestCase.annotations`. - New tags are available in `TestCase.tags`. Reporter changes: - `json` reporter includes new tags in addition to old `@smoke`-style tags. **Breaking**: tags are now listed with the leading `@` symbol. - `html` reporter filters by old and new tags with the same `@smoke` token. Fixes #29229, fixes #23180. --- docs/src/test-annotations-js.md | 121 ++- docs/src/test-api/class-test.md | 245 ++++- docs/src/test-api/class-testconfig.md | 25 + docs/src/test-api/class-testproject.md | 29 + docs/src/test-cli-js.md | 1 + docs/src/test-reporter-api/class-testcase.md | 13 +- packages/html-reporter/src/filter.ts | 6 +- packages/html-reporter/src/labelUtils.tsx | 23 +- .../html-reporter/src/testCaseView.spec.tsx | 1 + packages/html-reporter/src/testCaseView.tsx | 2 +- packages/html-reporter/src/testFileView.tsx | 5 +- packages/html-reporter/src/types.ts | 1 + packages/playwright/src/common/config.ts | 3 + packages/playwright/src/common/suiteUtils.ts | 5 +- packages/playwright/src/common/test.ts | 6 + packages/playwright/src/common/testType.ts | 63 +- .../playwright/src/isomorphic/teleReceiver.ts | 3 + packages/playwright/src/program.ts | 2 + packages/playwright/src/reporters/html.ts | 2 + packages/playwright/src/reporters/json.ts | 9 +- .../playwright/src/reporters/teleEmitter.ts | 1 + packages/playwright/src/runner/loadUtils.ts | 32 +- packages/playwright/src/util.ts | 50 + packages/playwright/types/test.d.ts | 939 ++++++++++++++++-- packages/playwright/types/testReporter.d.ts | 55 +- tests/playwright-test/reporter-html.spec.ts | 14 +- tests/playwright-test/reporter-json.spec.ts | 22 +- tests/playwright-test/reporter.spec.ts | 93 ++ tests/playwright-test/test-modifiers.spec.ts | 6 +- tests/playwright-test/test-tag.spec.ts | 245 +++++ tests/playwright-test/types-2.spec.ts | 19 + utils/generate_types/index.js | 5 + utils/generate_types/overrides-test.d.ts | 23 +- 33 files changed, 1849 insertions(+), 220 deletions(-) create mode 100644 tests/playwright-test/test-tag.spec.ts diff --git a/docs/src/test-annotations-js.md b/docs/src/test-annotations-js.md index 15a7f0529b..768f930255 100644 --- a/docs/src/test-annotations-js.md +++ b/docs/src/test-annotations-js.md @@ -5,13 +5,17 @@ title: "Annotations" ## Introduction -Playwright Test supports test annotations to deal with failures, flakiness, skip, focus and tag tests: -- [`method: Test.skip`] marks the test as irrelevant. Playwright Test does not run such a test. Use this annotation when the test is not applicable in some configuration. -- [`method: Test.fail`] marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain. -- [`method: Test.fixme`] marks the test as failing. Playwright Test will not run this test, as opposed to the `fail` annotation. Use `fixme` when running the test is slow or crashes. +Playwright supports tags and annotations that are displayed in the test report. + +You can add your own tags and annotations at any moment, but Playwright comes with a few built-in ones: +- [`method: Test.skip`] marks the test as irrelevant. Playwright does not run such a test. Use this annotation when the test is not applicable in some configuration. +- [`method: Test.fail`] marks the test as failing. Playwright will run this test and ensure it does indeed fail. If the test does not fail, Playwright will complain. +- [`method: Test.fixme`] marks the test as failing. Playwright will not run this test, as opposed to the `fail` annotation. Use `fixme` when running the test is slow or crashes. - [`method: Test.slow`] marks the test as slow and triples the test timeout. -Annotations can be used on a single test or a group of tests. Annotations can be conditional, in which case they apply when the condition is truthy. Annotations may depend on test fixtures. There could be multiple annotations on the same test, possibly in different configurations. +Annotations can be used on a single test or a group of tests. + +Built-in annotations can be conditional, in which case they apply when the condition is truthy, and may depend on test fixtures. There could be multiple annotations on the same test, possibly in different configurations. ## Focus a test @@ -63,66 +67,129 @@ test.describe('two tests', () => { ## Tag tests -Sometimes you want to tag your tests as `@fast` or `@slow` and only run the tests that have the certain tag. We recommend that you use the `--grep` and `--grep-invert` command line flags for that: +Sometimes you want to tag your tests as `@fast` or `@slow`, and then filter by tag in the test report. Or you might want to only run tests that have a certain tag. To do this, provide additional details when declaring a test. ```js import { test, expect } from '@playwright/test'; -test('Test login page @fast', async ({ page }) => { +test('test login page', { + tag: '@fast', +}, async ({ page }) => { // ... }); -test('Test full report @slow', async ({ page }) => { +test('test full report', { + tag: '@slow', +}, async ({ page }) => { // ... }); ``` -You will then be able to run only that test: +You can also tag all tests in a group or provide multiple tags: + +```js +import { test, expect } from '@playwright/test'; + +test.describe('group', { + tag: '@report', +}, () => { + test('test report header', async ({ page }) => { + // ... + }); + + test('test full report', { + tag: ['@slow', '@vrt'], + }, async ({ page }) => { + // ... + }); +}); +``` + +You can now run tests that have a particular tag. ```bash tab=bash-bash -npx playwright test --grep @fast +npx playwright test --tag @fast ``` ```powershell tab=bash-powershell -npx playwright test --grep "@fast" +npx playwright test --tag "@fast" ``` ```batch tab=bash-batch -npx playwright test --grep @fast +npx playwright test --tag @fast ``` Or if you want the opposite, you can skip the tests with a certain tag: ```bash tab=bash-bash -npx playwright test --grep-invert @fast +npx playwright test --tag "not @fast" ``` ```powershell tab=bash-powershell -npx playwright test --grep-invert "@fast" +npx playwright test --tag "not @fast" ``` ```batch tab=bash-batch -npx playwright test --grep-invert @fast +npx playwright test --tag "not @fast" ``` -To run tests containing either tag (logical `OR` operator): +The `--tag` option supports logical tag expressions. You can use `and`, `or` and `not` operators, as well as group with parenthesis. For example, to run `@smoke` tests that are either `@slow` or `@very-slow`: ```bash tab=bash-bash -npx playwright test --grep "@fast|@slow" +npx playwright test --tag "@smoke and (@slow or @very-slow)" ``` ```powershell tab=bash-powershell -npx playwright test --grep --% "@fast^|@slow" +npx playwright test --tag "@smoke and (@slow or @very-slow)" ``` ```batch tab=bash-batch -npx playwright test --grep "@fast^|@slow" +npx playwright test --tag "@smoke and (@slow or @very-slow)" ``` -Or run tests containing both tags (logical `AND` operator) using regex lookaheads: +You can also filter tests in the configuration file via [`property: TestConfig.tagFilter`] and [`property: TestProject.tagFilter`]. -```bash -npx playwright test --grep "(?=.*@fast)(?=.*@slow)" + +## Annotate tests + +If you would like to annotate your tests with something more substantial than a tag, you can do that when declaring a test. Annotations have a `type` and a `description` for more context, and will be visible in the test report. + +For example, to annotate a test with an issue url: + +```js +import { test, expect } from '@playwright/test'; + +test('test login page', { + annotation: { + type: 'issue', + description: 'https://github.com/microsoft/playwright/issues/23180', + }, +}, async ({ page }) => { + // ... +}); +``` + +You can also annotate all tests in a group or provide multiple annotations: + +```js +import { test, expect } from '@playwright/test'; + +test.describe('report tests', { + annotation: { type: 'category', description: 'report' }, +}, () => { + test('test report header', async ({ page }) => { + // ... + }); + + test('test full report', { + annotation: [ + { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/23180' }, + { type: 'performance', description: 'very slow test!' }, + ], + }, async ({ page }) => { + // ... + }); +}); ``` ## Conditionally skip a group of tests @@ -166,17 +233,17 @@ test('user profile', async ({ page }) => { }); ``` -## Custom annotations +## Dynamic annotations -It's also possible to add custom metadata in the form of annotations to your tests. Annotations are key/value pairs accessible via [`test.info().annotations`](./api/class-testinfo#test-info-annotations). Many reporters show annotations, for example `'html'`. +While the test is running, you can add dynamic annotations to [`test.info().annotations`](./api/class-testinfo#test-info-annotations). ```js title="example.spec.ts" -test('user profile', async ({ page }) => { +test('example test', async ({ page, browser }) => { test.info().annotations.push({ - type: 'issue', - description: 'https://github.com/microsoft/playwright/issues/', + type: 'browser version', + description: browser.version(), }); // ... }); diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 583373fac9..91976673b9 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -19,6 +19,9 @@ test('basic test', async ({ page }) => { Declares a test. +* `test(title, body)` +* `test(title, details, body)` + **Usage** ```js @@ -30,12 +33,69 @@ test('basic test', async ({ page }) => { }); ``` +**Tags** + +You can tag tests by providing additional test details. Note that each tag must start with `@` symbol. + +```js +import { test, expect } from '@playwright/test'; + +test('basic test', { + tag: '@smoke', +}, async ({ page }) => { + await page.goto('https://playwright.dev/'); + // ... +}); +``` + +Test tags are displayed in the test report, and are available to a custom reporter via `TestCase.tags` property. + +You can also filter tests by their tags during test execution: +* in the [command line](../test-cli.md#reference); +* in the config with [`property: TestConfig.tagFilter`] and [`property: TestProject.tagFilter`]; + +Learn more about [tagging](../test-annotations.md#tag-tests). + +**Annotations** + +You can annotate tests by providing additional test details. + +```js +import { test, expect } from '@playwright/test'; + +test('basic test', { + annotation: { + type: 'issue', + description: 'https://github.com/microsoft/playwright/issues/23180', + }, +}, async ({ page }) => { + await page.goto('https://playwright.dev/'); + // ... +}); +``` + +Test annotations are displayed in the test report, and are available to a custom reporter via `TestCase.annotations` property. + +You can also add dynamic annotations by manipulating [`property: TestInfo.annotations`]. + +Learn more about [test annotations](../test-annotations.md). + ### param: Test.(call).title * since: v1.10 - `title` <[string]> Test title. +### param: Test.(call).details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> Annotation type, for example `'issue'`. + - `description` ?<[string]> Optional annotation description, for example an issue url. + +Additional test details. + ### param: Test.(call).body * since: v1.10 - `body` <[function]\([Fixtures], [TestInfo]\)> @@ -279,6 +339,7 @@ Declares a group of tests. * `test.describe(title, callback)` * `test.describe(callback)` +* `test.describe(title, details, callback)` **Usage** @@ -296,7 +357,9 @@ test.describe('two tests', () => { }); ``` -Without a title, this method declares an **anonymous** group of tests. This is convenient to give a group of tests a common option with [`method: Test.use`]. +**Anonymous group** + +You can also declare a test group without a title. This is convenient to give a group of tests a common option with [`method: Test.use`]. ```js test.describe(() => { @@ -312,12 +375,69 @@ test.describe(() => { }); ``` +**Tags** + +You can tag all tests in a group by providing additional details. Note that each tag must start with `@` symbol. + +```js +import { test, expect } from '@playwright/test'; + +test.describe('two tagged tests', { + tag: '@smoke', +}, () => { + test('one', async ({ page }) => { + // ... + }); + + test('two', async ({ page }) => { + // ... + }); +}); +``` + +Learn more about [tagging](../test-annotations.md#tag-tests). + +**Annotations** + +You can annotate all tests in a group by providing additional details. + +```js +import { test, expect } from '@playwright/test'; + +test.describe('two annotated tests', { + annotation: { + type: 'issue', + description: 'https://github.com/microsoft/playwright/issues/23180', + }, +}, () => { + test('one', async ({ page }) => { + // ... + }); + + test('two', async ({ page }) => { + // ... + }); +}); +``` + +Learn more about [test annotations](../test-annotations.md). + ### param: Test.describe.title * since: v1.10 - `title` ?<[string]> Group title. +### param: Test.describe.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +Additional details for all tests in the group. + ### param: Test.describe.callback * since: v1.10 - `callback` <[function]> @@ -410,6 +530,7 @@ Declares a test group similarly to [`method: Test.describe`]. Tests in this grou * `test.describe.fixme(title, callback)` * `test.describe.fixme(callback)` +* `test.describe.fixme(title, details, callback)` **Usage** @@ -435,6 +556,16 @@ test.describe.fixme(() => { Group title. +### param: Test.describe.fixme.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.fixme.callback * since: v1.25 - `callback` <[function]> @@ -450,6 +581,7 @@ Declares a focused group of tests. If there are some focused tests or suites, al * `test.describe.only(title, callback)` * `test.describe.only(callback)` +* `test.describe.only(title, details, callback)` **Usage** @@ -479,6 +611,16 @@ test.describe.only(() => { Group title. +### param: Test.describe.only.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.only.callback * since: v1.10 - `callback` <[function]> @@ -495,6 +637,7 @@ Declares a group of tests that could be run in parallel. By default, tests in a * `test.describe.parallel(title, callback)` * `test.describe.parallel(callback)` +* `test.describe.parallel(title, details, callback)` **Usage** @@ -521,6 +664,16 @@ test.describe.parallel(() => { Group title. +### param: Test.describe.parallel.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.parallel.callback * since: v1.10 - `callback` <[function]> @@ -537,6 +690,7 @@ Declares a focused group of tests that could be run in parallel. This is similar * `test.describe.parallel.only(title, callback)` * `test.describe.parallel.only(callback)` +* `test.describe.parallel.only(title, details, callback)` **Usage** @@ -561,6 +715,16 @@ test.describe.parallel.only(() => { Group title. +### param: Test.describe.parallel.only.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.parallel.only.callback * since: v1.10 - `callback` <[function]> @@ -581,6 +745,7 @@ Using serial is not recommended. It is usually better to make your tests isolate * `test.describe.serial(title, callback)` * `test.describe.serial(title)` +* `test.describe.serial(title, details, callback)` **Usage** @@ -605,6 +770,16 @@ test.describe.serial(() => { Group title. +### param: Test.describe.serial.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.serial.callback * since: v1.10 - `callback` <[function]> @@ -625,6 +800,7 @@ Using serial is not recommended. It is usually better to make your tests isolate * `test.describe.serial.only(title, callback)` * `test.describe.serial.only(title)` +* `test.describe.serial.only(title, details, callback)` **Usage** @@ -651,6 +827,16 @@ test.describe.serial.only(() => { Group title. +### param: Test.describe.serial.only.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.serial.only.callback * since: v1.10 - `callback` <[function]> @@ -667,6 +853,7 @@ Declares a skipped test group, similarly to [`method: Test.describe`]. Tests in * `test.describe.skip(title, callback)` * `test.describe.skip(title)` +* `test.describe.skip(title, details, callback)` **Usage** @@ -692,6 +879,16 @@ test.describe.skip(() => { Group title. +### param: Test.describe.skip.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for details description. + ### param: Test.describe.skip.callback * since: v1.10 - `callback` <[function]> @@ -837,6 +1034,7 @@ An object containing fixtures and/or options. Learn more about [fixtures format] Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. * `test.fail(title, body)` +* `test.fail(title, details, body)` * `test.fail()` * `test.fail(condition, description)` * `test.fail(callback, description)` @@ -896,6 +1094,16 @@ test('less readable', async ({ page }) => { Test title. +### param: Test.fail.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.(call)`] for test details description. + ### param: Test.fail.body * since: v1.42 - `body` ?<[function]\([Fixtures], [TestInfo]\)> @@ -928,6 +1136,7 @@ Optional description that will be reflected in a test report. Mark a test as "fixme", with the intention to fix it. Playwright will not run the test past the `test.fixme()` call. * `test.fixme(title, body)` +* `test.fixme(title, details, body)` * `test.fixme()` * `test.fixme(condition, description)` * `test.fixme(callback, description)` @@ -987,6 +1196,16 @@ test('less readable', async ({ page }) => { Test title. +### param: Test.fixme.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.(call)`] for test details description. + ### param: Test.fixme.body * since: v1.10 - `body` ?<[function]\([Fixtures], [TestInfo]\)> @@ -1037,6 +1256,9 @@ test('example test', async ({ page }) => { Declares a focused test. If there are some focused tests or suites, all of them will be run but nothing else. +* `test.only(title, body)` +* `test.only(title, details, body)` + **Usage** ```js @@ -1051,6 +1273,16 @@ test.only('focus this test', async ({ page }) => { Test title. +### param: Test.only.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.(call)`] for test details description. + ### param: Test.only.body * since: v1.10 - `body` <[function]\([Fixtures], [TestInfo]\)> @@ -1123,6 +1355,7 @@ Skip a test. Playwright will not run the test past the `test.skip()` call. Skipped tests are not supposed to be ever run. If you intent to fix the test, use [`method: Test.fixme`] instead. * `test.skip(title, body)` +* `test.skip(title, details, body)` * `test.skip()` * `test.skip(condition, description)` * `test.skip(callback, description)` @@ -1182,6 +1415,16 @@ test('less readable', async ({ page }) => { Test title. +### param: Test.skip.details +* since: v1.42 +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.(call)`] for test details description. + ### param: Test.skip.body * since: v1.10 - `body` ?<[function]\([Fixtures], [TestInfo]\)> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 926b3eca6d..bd184faa4c 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -474,6 +474,31 @@ export default defineConfig({ }); ``` + +## property: TestConfig.tagFilter +* since: v1.42 +- type: ?<[string]> + +Filter to only run tests with or without particular tag(s). This property must be a logical expression containing tags, parenthesis `(` and `)`, and operators `and`, `or` and `not`. + +Learn more about [tagging](../test-annotations.md#tag-tests). + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + tagFilter: '@smoke', +}); +``` + +Tag expression examples: +* `tagFilter: '@smoke'` - run only "smoke" tests; +* `tagFilter: '@smoke and not @production'` - run only "smoke" tests that are not tagged as "production"; +* `tagFilter: '(@smoke or @fast) and @v2'` - run "v2" tests that are tagged as "smoke", "fast" or both. + + ## property: TestConfig.testDir * since: v1.10 - type: ?<[string]> diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 61a1cc5b5c..a01fde3275 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -199,6 +199,35 @@ Use [`method: Test.describe.configure`] to change the number of retries for a sp Use [`property: TestConfig.retries`] to change this option for all projects. + +## property: TestProject.tagFilter +* since: v1.42 +- type: ?<[string]> + +Filter to only run tests with or without particular tag(s). This property must be a logical expression containing tags, parenthesis `(` and `)`, and operators `and`, `or` and `not`. + +Learn more about [tagging](../test-annotations.md#tag-tests). + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + projects: [ + { + name: 'tests', + tagFilter: '@smoke', + }, + ], +}); +``` + +Tag expression examples: +* `tagFilter: '@smoke'` - run only "smoke" tests; +* `tagFilter: '@smoke and not @production'` - run only "smoke" tests that are not tagged as "production"; +* `tagFilter: '(@smoke or @fast) and @v2'` - run "v2" tests that are tagged as "smoke", "fast" or both. + ## property: TestProject.teardown * since: v1.34 - type: ?<[string]> diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 6eeec4abe2..61e204ab14 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -98,6 +98,7 @@ Complete set of Playwright Test options is available in the [configuration file] | `--reporter ` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. | | `--retries ` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). | | `--shard ` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.| +| `--tag ` | Only run tests with a tag matching this tag expression. Learn more about [tagging](./test-annotations.md#tag-tests). | | `--timeout ` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).| | `--trace ` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` | | `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. | diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index df721ff404..a0b26f6ebd 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -10,7 +10,10 @@ - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - `description` ?<[string]> Optional description. -The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file. +The list of annotations applicable to the current test. Includes: +* annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`]; +* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`]; +* annotations appended to [`property: TestInfo.annotations`] during the test execution. Annotations are available during test execution through [`property: TestInfo.annotations`]. @@ -79,6 +82,14 @@ The maximum number of retries given to this test in the configuration. Learn more about [test retries](../test-retries.md#retries). +## property: TestCase.tags +* since: v1.42 +- type: <[Array]<[string]>> + +The list of tags defined on the test or suite via [`method: Test.(call)`] or [`method: Test.describe`]. + +Learn more about [test tags](../test-annotations.md#tag-tests). + ## property: TestCase.timeout * since: v1.10 - type: <[float]> diff --git a/packages/html-reporter/src/filter.ts b/packages/html-reporter/src/filter.ts index 0a49c1dc61..6f7ef14525 100644 --- a/packages/html-reporter/src/filter.ts +++ b/packages/html-reporter/src/filter.ts @@ -14,7 +14,7 @@ limitations under the License. */ -import { escapeRegExp } from './labelUtils'; +import { testCaseLabels } from './labelUtils'; import type { TestCaseSummary } from './types'; export class Filter { @@ -114,6 +114,7 @@ export class Filter { file: test.location.file, line: String(test.location.line), column: String(test.location.column), + labels: testCaseLabels(test).map(label => label.toLowerCase()), }; (test as any).searchValues = searchValues; } @@ -140,7 +141,7 @@ export class Filter { } } if (this.labels.length) { - const matches = this.labels.every(l => searchValues.text?.match(new RegExp(`(\\s|^)${escapeRegExp(l)}(\\s|$)`, 'g'))); + const matches = this.labels.every(l => searchValues.labels.includes(l)); if (!matches) return false; } @@ -156,5 +157,6 @@ type SearchValues = { file: string; line: string; column: string; + labels: string[]; }; diff --git a/packages/html-reporter/src/labelUtils.tsx b/packages/html-reporter/src/labelUtils.tsx index 52d587db2e..f0d5909494 100644 --- a/packages/html-reporter/src/labelUtils.tsx +++ b/packages/html-reporter/src/labelUtils.tsx @@ -16,20 +16,19 @@ import type { TestCaseSummary } from './types'; -export function escapeRegExp(string: string) { - const reRegExpChar = /[\\^$.*+?()[\]{}|]/g; - const reHasRegExpChar = RegExp(reRegExpChar.source); - - return (string && reHasRegExpChar.test(string)) - ? string.replace(reRegExpChar, '\\$&') - : (string || ''); -} +const labelsSymbol = Symbol('labels'); +// Note: all labels start with "@" export function testCaseLabels(test: TestCaseSummary): string[] { - const tags = matchTags(test.path.join(' ') + ' ' + test.title).sort((a, b) => a.localeCompare(b)); - if (test.botName) - tags.unshift(test.botName); - return tags; + if (!(test as any)[labelsSymbol]) { + const labels: string[] = []; + if (test.botName) + labels.push('@' + test.botName); + labels.push(...test.tags); + labels.push(...matchTags(test.path.join(' ') + ' ' + test.title).sort((a, b) => a.localeCompare(b))); + (test as any)[labelsSymbol] = labels; + } + return (test as any)[labelsSymbol]; } // match all tags in test title diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx index f67a3f2cc5..28fe8247f5 100644 --- a/packages/html-reporter/src/testCaseView.spec.tsx +++ b/packages/html-reporter/src/testCaseView.spec.tsx @@ -55,6 +55,7 @@ const testCase: TestCase = { { type: 'annotation', description: 'Annotation text' }, { type: 'annotation', description: 'Another annotation text' }, ], + tags: [], outcome: 'expected', duration: 10, ok: true, diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsx index 4fcd63baf7..4b79908edf 100644 --- a/packages/html-reporter/src/testCaseView.tsx +++ b/packages/html-reporter/src/testCaseView.tsx @@ -95,7 +95,7 @@ const LabelsLinkView: React.FC ( - {label.startsWith('@') ? label.slice(1) : label} + {label.slice(1)} ))} diff --git a/packages/html-reporter/src/testFileView.tsx b/packages/html-reporter/src/testFileView.tsx index 4d6560afb2..619d6263d2 100644 --- a/packages/html-reporter/src/testFileView.tsx +++ b/packages/html-reporter/src/testFileView.tsx @@ -98,8 +98,7 @@ const LabelsClickView: React.FC {labels.map(label => ( onClickHandle(e, label)}> - {label.startsWith('@') ? label.slice(1) : label} + {label.slice(1)} ))} diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index 3a1f14f56e..d7ada9cbe6 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -69,6 +69,7 @@ export type TestCaseSummary = { botName?: string; location: Location; annotations: TestCaseAnnotation[]; + tags: string[]; outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; duration: number; ok: boolean; diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index b3a124e47c..fe411577fc 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -47,6 +47,7 @@ export class FullConfigInternal { cliArgs: string[] = []; cliGrep: string | undefined; cliGrepInvert: string | undefined; + cliTagFilter: string | undefined; cliProjectFilter?: string[]; cliListOnly = false; cliPassWithNoTests?: boolean; @@ -160,6 +161,7 @@ export class FullProjectInternal { id = ''; deps: FullProjectInternal[] = []; teardown: FullProjectInternal | undefined; + tagFilter: string | undefined; constructor(configDir: string, config: Config, fullConfig: FullConfigInternal, projectConfig: Project, configCLIOverrides: ConfigCLIOverrides, throwawayArtifactsPath: string) { this.fullConfig = fullConfig; @@ -188,6 +190,7 @@ export class FullProjectInternal { }; this.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined); this.expect = takeFirst(projectConfig.expect, config.expect, {}); + this.tagFilter = takeFirst(projectConfig.tagFilter, config.tagFilter, undefined); if (this.expect.toHaveScreenshot?.stylePath) { const stylePaths = Array.isArray(this.expect.toHaveScreenshot.stylePath) ? this.expect.toHaveScreenshot.stylePath : [this.expect.toHaveScreenshot.stylePath]; this.expect.toHaveScreenshot.stylePath = stylePaths.map(stylePath => path.resolve(configDir, stylePath)); diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts index 6c3015772f..69d8e7b086 100644 --- a/packages/playwright/src/common/suiteUtils.ts +++ b/packages/playwright/src/common/suiteUtils.ts @@ -62,7 +62,10 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit let inheritedRetries: number | undefined; let inheritedTimeout: number | undefined; for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) { - test._staticAnnotations.push(...parentSuite._staticAnnotations); + if (parentSuite._staticAnnotations.length) + test._staticAnnotations = [...parentSuite._staticAnnotations, ...test._staticAnnotations]; + if (parentSuite._tags.length) + test.tags = [...parentSuite._tags, ...test.tags]; if (inheritedRetries === undefined && parentSuite._retries !== undefined) inheritedRetries = parentSuite._retries; if (inheritedTimeout === undefined && parentSuite._timeout !== undefined) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 578a1574f4..f798ed8932 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -49,6 +49,7 @@ export class Suite extends Base implements SuitePrivate { _timeout: number | undefined; _retries: number | undefined; _staticAnnotations: Annotation[] = []; + _tags: string[] = []; _modifiers: Modifier[] = []; _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; @@ -187,6 +188,7 @@ export class Suite extends Base implements SuitePrivate { timeout: this._timeout, retries: this._retries, staticAnnotations: this._staticAnnotations.slice(), + tags: this._tags.slice(), modifiers: this._modifiers.slice(), parallelMode: this._parallelMode, hooks: this._hooks.map(h => ({ type: h.type, location: h.location, title: h.title })), @@ -202,6 +204,7 @@ export class Suite extends Base implements SuitePrivate { suite._timeout = data.timeout; suite._retries = data.retries; suite._staticAnnotations = data.staticAnnotations; + suite._tags = data.tags; suite._modifiers = data.modifiers; suite._parallelMode = data.parallelMode; suite._hooks = data.hooks.map((h: any) => ({ type: h.type, location: h.location, title: h.title, fn: () => { } })); @@ -234,6 +237,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { annotations: Annotation[] = []; retries = 0; repeatEachIndex = 0; + tags: string[] = []; _testType: TestTypeImpl; id = ''; @@ -295,6 +299,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { workerHash: this._workerHash, staticAnnotations: this._staticAnnotations.slice(), annotations: this.annotations.slice(), + tags: this.tags.slice(), projectId: this._projectId, }; } @@ -311,6 +316,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { test._workerHash = data.workerHash; test._staticAnnotations = data.staticAnnotations; test.annotations = data.annotations; + test.tags = data.tags; test._projectId = data.projectId; return test; } diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index aba682f6dc..27949c49d5 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -19,7 +19,7 @@ import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuit import { TestCase, Suite } from './test'; import { wrapFunctionWithLocation } from '../transform/transform'; import type { FixturesWithLocation } from './config'; -import type { Fixtures, TestType } from '../../types/test'; +import type { Fixtures, TestType, TestDetails } from '../../types/test'; import type { Location } from '../../types/testReporter'; import { getPackageManagerExecCommand } from 'playwright-core/lib/utils'; @@ -87,13 +87,27 @@ export class TestTypeImpl { return suite; } - private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail', location: Location, title: string, fn: Function) { + private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail', location: Location, title: string, fnOrDetails: Function | TestDetails, fn?: Function) { throwIfRunningInsideJest(); const suite = this._currentSuite('test()'); if (!suite) return; - const test = new TestCase(title, fn, this, location); + + let details: TestDetails; + let body: Function; + if (typeof fnOrDetails === 'function') { + body = fnOrDetails; + details = {}; + } else { + body = fn!; + details = fnOrDetails; + } + + const validatedDetails = validateTestDetails(details); + const test = new TestCase(title, body, this, location); test._requireFile = suite._requireFile; + test._staticAnnotations.push(...validatedDetails.annotations); + test.tags.push(...validatedDetails.tags); suite._addTest(test); if (type === 'only') @@ -102,20 +116,36 @@ export class TestTypeImpl { test._staticAnnotations.push({ type }); } - private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, title: string | Function, fn?: Function) { + private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) { throwIfRunningInsideJest(); const suite = this._currentSuite('test.describe()'); if (!suite) return; - if (typeof title === 'function') { - fn = title; + let title: string; + let body: Function; + let details: TestDetails; + + if (typeof titleOrFn === 'function') { title = ''; + details = {}; + body = titleOrFn; + } else if (typeof fnOrDetails === 'function') { + title = titleOrFn; + details = {}; + body = fnOrDetails; + } else { + title = titleOrFn; + details = fnOrDetails!; + body = fn!; } + const validatedDetails = validateTestDetails(details); const child = new Suite(title, 'describe', this); child._requireFile = suite._requireFile; child.location = location; + child._staticAnnotations.push(...validatedDetails.annotations); + child._tags.push(...validatedDetails.tags); suite._addSuite(child); if (type === 'only' || type === 'serial.only' || type === 'parallel.only') @@ -135,7 +165,7 @@ export class TestTypeImpl { } setCurrentlyLoadingFileSuite(child); - fn!(); + body(); setCurrentlyLoadingFileSuite(suite); } @@ -176,14 +206,19 @@ export class TestTypeImpl { } } - private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: [arg?: any | Function, description?: string]) { + private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: any[]) { const suite = currentlyLoadingFileSuite(); if (suite) { if (typeof modifierArgs[0] === 'string' && typeof modifierArgs[1] === 'function' && (type === 'skip' || type === 'fixme' || type === 'fail')) { - // Support for test.{skip,fixme}('title', () => {}) + // Support for test.{skip,fixme,fail}(title, body) this._createTest(type, location, modifierArgs[0], modifierArgs[1]); return; } + if (typeof modifierArgs[0] === 'string' && typeof modifierArgs[1] === 'object' && typeof modifierArgs[2] === 'function' && (type === 'skip' || type === 'fixme' || type === 'fail')) { + // Support for test.{skip,fixme,fail}(title, details, body) + this._createTest(type, location, modifierArgs[0], modifierArgs[1], modifierArgs[2]); + return; + } if (typeof modifierArgs[0] === 'function') { suite._modifiers.push({ type, fn: modifierArgs[0], location, description: modifierArgs[1] }); @@ -253,6 +288,16 @@ function throwIfRunningInsideJest() { } } +function validateTestDetails(details: TestDetails) { + const annotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []); + const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []); + for (const tag of tags) { + if (tag[0] !== '@') + throw new Error(`Tag must start with "@" symbol, got "${tag}" instead.`); + } + return { annotations, tags }; +} + export const rootTestType = new TestTypeImpl([]); export function mergeTests(...tests: TestType[]) { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index a14523da1a..ec41cb3da2 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -73,6 +73,7 @@ export type JsonTestCase = { title: string; location: JsonLocation; retries: number; + tags: string[]; }; export type JsonTestEnd = { @@ -394,6 +395,7 @@ export class TeleReporterReceiver { test.id = payload.testId; test.location = this._absoluteLocation(payload.location); test.retries = payload.retries; + test.tags = payload.tags; return test; } @@ -475,6 +477,7 @@ export class TeleTestCase implements reporterTypes.TestCase { timeout = 0; annotations: Annotation[] = []; retries = 0; + tags: string[] = []; repeatEachIndex = 0; id: string; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index f3d3fc26c4..8b167910b7 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -133,6 +133,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { config.cliArgs = args; config.cliGrep = opts.grep as string | undefined; config.cliGrepInvert = opts.grepInvert as string | undefined; + config.cliTagFilter = opts.tag; config.cliListOnly = !!opts.list; config.cliProjectFilter = opts.project || undefined; config.cliPassWithNoTests = !!opts.passWithNoTests; @@ -333,6 +334,7 @@ const testOptions: [string, string][] = [ ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], ['--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`], ['--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`], + ['--tag ', `Only run tests with a tag(s) matching the specified expression (default: no filtering)`], ['--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`], ['--trace ', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`], ['--ui', `Run tests in interactive UI mode`], diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index a660b2b671..3f21de3cde 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -363,6 +363,7 @@ class HtmlBuilder { duration, // Annotations can be pushed directly, with a wrong type. annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })), + tags: test.tags, outcome: test.outcome(), path, results, @@ -377,6 +378,7 @@ class HtmlBuilder { duration, // Annotations can be pushed directly, with a wrong type. annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })), + tags: test.tags, outcome: test.outcome(), path, ok: test.outcome() === 'expected' || test.outcome() === 'flaky', diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index 41c26b653c..c62fd7781c 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -169,7 +169,7 @@ class JSONReporter extends EmptyReporter { return { title: test.title, ok: test.ok(), - tags: (test.title.match(/@[\S]+/g) || []).map(t => t.substring(1)), + tags: extractTags(test), tests: [this._serializeTest(projectId, projectName, test)], id: test.id, ...this._relativeLocation(test.location), @@ -256,6 +256,13 @@ function reportOutputNameFromEnv(): string | undefined { return undefined; } +function extractTags(test: TestCase) { + return [ + ...test.tags, + ...(test.title.match(/@[\S]+/g) || []), + ]; +} + export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] { if (!Array.isArray(patterns)) patterns = [patterns]; diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 113856ce62..7f4caf6ad5 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -202,6 +202,7 @@ export class TeleReporterEmitter implements ReporterV2 { title: test.title, location: this._relativeLocation(test.location), retries: test.retries, + tags: test.tags, }; } diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index f75aca8fb8..5dc50c74a8 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -21,8 +21,8 @@ import { Suite } from '../common/test'; import type { TestCase } from '../common/test'; import type { FullProjectInternal } from '../common/config'; import type { FullConfigInternal } from '../common/config'; -import { createFileMatcherFromArguments, createFileFiltersFromArguments, createTitleMatcher, errorWithFile, forceRegExp } from '../util'; -import type { Matcher, TestFileFilter } from '../util'; +import { createFileMatcherFromArguments, createFileFiltersFromArguments, createTitleMatcher, errorWithFile, forceRegExp, createTagMatcher } from '../util'; +import type { Matcher, TagMatcher, TestFileFilter } from '../util'; import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils'; import type { TestRun } from './tasks'; import { requireOrImport } from '../transform/transform'; @@ -135,12 +135,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho const grepMatcher = config.cliGrep ? createTitleMatcher(forceRegExp(config.cliGrep)) : () => true; const grepInvertMatcher = config.cliGrepInvert ? createTitleMatcher(forceRegExp(config.cliGrepInvert)) : () => false; const cliTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title); + const cliTagMatcher = config.cliTagFilter ? createTagMatcher(config.cliTagFilter) : undefined; // Filter file suites for all projects. for (const [project, fileSuites] of testRun.projectSuites) { const projectSuite = createProjectSuite(project, fileSuites); projectSuites.set(project, projectSuite); - const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testIdMatcher: config.testIdMatcher }); + const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, cliTagMatcher, testIdMatcher: config.testIdMatcher }); filteredProjectSuites.set(project, filteredProjectSuite); } } @@ -217,21 +218,21 @@ function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]): const grepMatcher = createTitleMatcher(project.project.grep); const grepInvertMatcher = project.project.grepInvert ? createTitleMatcher(project.project.grepInvert) : null; - - const titleMatcher = (test: TestCase) => { + const tagMatcher = project.tagFilter ? createTagMatcher(project.tagFilter) : undefined; + filterTestsRemoveEmptySuites(projectSuite, (test: TestCase) => { + if (tagMatcher && !tagMatcher(test.tags)) + return false; const grepTitle = test.titlePath().join(' '); if (grepInvertMatcher?.(grepTitle)) return false; return grepMatcher(grepTitle); - }; - - filterTestsRemoveEmptySuites(projectSuite, titleMatcher); + }); return projectSuite; } -function filterProjectSuite(projectSuite: Suite, options: { cliFileFilters: TestFileFilter[], cliTitleMatcher?: Matcher, testIdMatcher?: Matcher }): Suite { +function filterProjectSuite(projectSuite: Suite, options: { cliFileFilters: TestFileFilter[], cliTitleMatcher?: Matcher, cliTagMatcher?: TagMatcher, testIdMatcher?: Matcher }): Suite { // Fast path. - if (!options.cliFileFilters.length && !options.cliTitleMatcher && !options.testIdMatcher) + if (!options.cliFileFilters.length && !options.cliTitleMatcher && !options.testIdMatcher && !options.cliTagMatcher) return projectSuite; const result = projectSuite._deepClone(); @@ -239,10 +240,13 @@ function filterProjectSuite(projectSuite: Suite, options: { cliFileFilters: Test filterByFocusedLine(result, options.cliFileFilters); if (options.testIdMatcher) filterByTestIds(result, options.testIdMatcher); - const titleMatcher = (test: TestCase) => { - return !options.cliTitleMatcher || options.cliTitleMatcher(test.titlePath().join(' ')); - }; - filterTestsRemoveEmptySuites(result, titleMatcher); + filterTestsRemoveEmptySuites(result, (test: TestCase) => { + if (options.cliTagMatcher && !options.cliTagMatcher(test.tags)) + return false; + if (options.cliTitleMatcher && !options.cliTitleMatcher(test.titlePath().join(' '))) + return false; + return true; + }); return result; } diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.ts index 17e406d514..38bb94ee7f 100644 --- a/packages/playwright/src/util.ts +++ b/packages/playwright/src/util.ts @@ -71,6 +71,7 @@ export function serializeError(error: Error | any): TestInfoError { } export type Matcher = (value: string) => boolean; +export type TagMatcher = (tags: string[]) => boolean; export type TestFileFilter = { re?: RegExp; @@ -144,6 +145,55 @@ export function createTitleMatcher(patterns: RegExp | RegExp[]): Matcher { }; } +export function createTagMatcher(tagFilter: string): TagMatcher { + const tokens: (string | null)[] = tagFilter.split(/([()]|\s+)/).filter(s => !!s.trim()); + tokens.push(null); // eof + let pos = 0; + + function parseSingle(): TagMatcher { + if (tokens[pos] === '(') { + pos++; + const result = parseOr(); + if (tokens[pos] !== ')') + throw new Error(`Expected matching ")" when parsing tag expression: ${tagFilter}`); + pos++; + return result; + } + if (tokens[pos] === 'not') { + pos++; + const inner = parseSingle(); + return (tags: string[]) => !inner(tags); + } + const tag = tokens[pos++]; + if (tag === null) + throw new Error(`Unexpected end of tag expression: ${tagFilter}`); + return (tags: string[]) => tags.includes(tag); + } + + function parseAnd(): TagMatcher { + const singles = [parseSingle()]; + while (tokens[pos] === 'and') { + pos++; + singles.push(parseSingle()); + } + return (tags: string[]) => singles.every(s => s(tags)); + } + + function parseOr(): TagMatcher { + const ands = [parseAnd()]; + while (tokens[pos] === 'or') { + pos++; + ands.push(parseAnd()); + } + return (tags: string[]) => ands.some(a => a(tags)); + } + + const result = parseOr(); + if (tokens[pos] !== null) + throw new Error(`Unexpected extra tokens in the tag expression: ${tagFilter}`); + return result; +} + export function mergeObjects(a: A | undefined | void, b: B | undefined | void, c: B | undefined | void): A & B & C { const result = { ...a } as any; for (const x of [b, c].filter(Boolean)) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 46bc239cb9..ce285e40af 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -707,9 +707,9 @@ interface TestConfig { /** * Whether to exit with an error if any tests or groups are marked as - * [test.only(title, body)](https://playwright.dev/docs/api/class-test#test-only) or - * [test.describe.only([title, callback])](https://playwright.dev/docs/api/class-test#test-describe-only). Useful on - * CI. + * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or + * [test.describe.only([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe-only). + * Useful on CI. * * **Usage** * @@ -1244,6 +1244,30 @@ interface TestConfig { */ snapshotPathTemplate?: string; + /** + * Filter to only run tests with or without particular tag(s). This property must be a logical expression containing + * tags, parenthesis `(` and `)`, and operators `and`, `or` and `not`. + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * tagFilter: '@smoke', + * }); + * ``` + * + * Tag expression examples: + * - `tagFilter: '@smoke'` - run only "smoke" tests; + * - `tagFilter: '@smoke and not @production'` - run only "smoke" tests that are not tagged as "production"; + * - `tagFilter: '(@smoke or @fast) and @v2'` - run "v2" tests that are tagged as "smoke", "fast" or both. + */ + tagFilter?: string; + /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * @@ -1464,9 +1488,9 @@ export type Metadata = { [key: string]: any }; export interface FullConfig { /** * Whether to exit with an error if any tests or groups are marked as - * [test.only(title, body)](https://playwright.dev/docs/api/class-test#test-only) or - * [test.describe.only([title, callback])](https://playwright.dev/docs/api/class-test#test-describe-only). Useful on - * CI. + * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or + * [test.describe.only([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe-only). + * Useful on CI. * * **Usage** * @@ -2001,13 +2025,13 @@ export interface TestInfo { * Marks the currently running test as "should fail". Playwright Test runs this test and ensures that it is actually * failing. This is useful for documentation purposes to acknowledge that some functionality is broken until it is * fixed. This is similar to - * [test.fail([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). */ fail(): void; /** * Conditionally mark the currently running test as "should fail" with an optional description. This is similar to - * [test.fail([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). * @param condition Test is marked as "should fail" when the condition is `true`. * @param description Optional description that will be reflected in a test report. */ @@ -2015,13 +2039,13 @@ export interface TestInfo { /** * Mark a test as "fixme", with the intention to fix it. Test is immediately aborted. This is similar to - * [test.fixme([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme). + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme). */ fixme(): void; /** * Conditionally mark the currently running test as "fixme" with an optional description. This is similar to - * [test.fixme([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme). + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme). * @param condition Test is marked as "fixme" when the condition is `true`. * @param description Optional description that will be reflected in a test report. */ @@ -2073,13 +2097,13 @@ export interface TestInfo { /** * Unconditionally skip the currently running test. Test is immediately aborted. This is similar to - * [test.skip([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip). + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip). */ skip(): void; /** * Conditionally skips the currently running test with an optional description. This is similar to - * [test.skip([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip). + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip). * @param condition A skip condition. Test is skipped when the condition is `true`. * @param description Optional description that will be reflected in a test report. */ @@ -2115,8 +2139,8 @@ export interface TestInfo { /** * The list of annotations applicable to the current test. Includes annotations from the test, annotations from all - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups the test - * belongs to and file-level annotations for the test file. + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups the + * test belongs to and file-level annotations for the test file. * * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). */ @@ -2188,9 +2212,9 @@ export interface TestInfo { /** * Expected status for the currently running test. This is usually `'passed'`, except for a few cases: * - `'skipped'` for skipped tests, e.g. with - * [test.skip([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip); + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip); * - `'failed'` for tests marked as failed with - * [test.fail([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail). * * Expected status is usually compared with the actual * [testInfo.status](https://playwright.dev/docs/api/class-testinfo#test-info-status): @@ -2355,11 +2379,22 @@ export interface TestInfo { workerIndex: number; } +type TestDetailsAnnotation = { + type: string; + description?: string; +}; + +export type TestDetails = { + tag?: string | string[]; + annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; +} + interface SuiteFunction { /** * Declares a group of tests. * - `test.describe(title, callback)` * - `test.describe(callback)` + * - `test.describe(title, details, callback)` * * **Usage** * @@ -2378,8 +2413,10 @@ interface SuiteFunction { * }); * ``` * - * Without a title, this method declares an **anonymous** group of tests. This is convenient to give a group of tests - * a common option with [test.use(options)](https://playwright.dev/docs/api/class-test#test-use). + * **Anonymous group** + * + * You can also declare a test group without a title. This is convenient to give a group of tests a common option with + * [test.use(options)](https://playwright.dev/docs/api/class-test#test-use). * * ```js * test.describe(() => { @@ -2395,16 +2432,64 @@ interface SuiteFunction { * }); * ``` * + * **Tags** + * + * You can tag all tests in a group by providing additional details. Note that each tag must start with `@` symbol. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two tagged tests', { + * tag: '@smoke', + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate all tests in a group by providing additional details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two annotated tests', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). * @param title Group title. + * @param details Additional details for all tests in the group. * @param callback A callback that is run immediately when calling - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests declared in - * this callback will belong to the group. + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests + * declared in this callback will belong to the group. */ (title: string, callback: () => void): void; /** * Declares a group of tests. * - `test.describe(title, callback)` * - `test.describe(callback)` + * - `test.describe(title, details, callback)` * * **Usage** * @@ -2423,8 +2508,10 @@ interface SuiteFunction { * }); * ``` * - * Without a title, this method declares an **anonymous** group of tests. This is convenient to give a group of tests - * a common option with [test.use(options)](https://playwright.dev/docs/api/class-test#test-use). + * **Anonymous group** + * + * You can also declare a test group without a title. This is convenient to give a group of tests a common option with + * [test.use(options)](https://playwright.dev/docs/api/class-test#test-use). * * ```js * test.describe(() => { @@ -2440,16 +2527,299 @@ interface SuiteFunction { * }); * ``` * + * **Tags** + * + * You can tag all tests in a group by providing additional details. Note that each tag must start with `@` symbol. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two tagged tests', { + * tag: '@smoke', + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate all tests in a group by providing additional details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two annotated tests', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). * @param title Group title. + * @param details Additional details for all tests in the group. * @param callback A callback that is run immediately when calling - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests declared in - * this callback will belong to the group. + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests + * declared in this callback will belong to the group. */ (callback: () => void): void; + /** + * Declares a group of tests. + * - `test.describe(title, callback)` + * - `test.describe(callback)` + * - `test.describe(title, details, callback)` + * + * **Usage** + * + * You can declare a group of tests with a title. The title will be visible in the test report as a part of each + * test's title. + * + * ```js + * test.describe('two tests', () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * **Anonymous group** + * + * You can also declare a test group without a title. This is convenient to give a group of tests a common option with + * [test.use(options)](https://playwright.dev/docs/api/class-test#test-use). + * + * ```js + * test.describe(() => { + * test.use({ colorScheme: 'dark' }); + * + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * **Tags** + * + * You can tag all tests in a group by providing additional details. Note that each tag must start with `@` symbol. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two tagged tests', { + * tag: '@smoke', + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate all tests in a group by providing additional details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two annotated tests', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + * @param title Group title. + * @param details Additional details for all tests in the group. + * @param callback A callback that is run immediately when calling + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests + * declared in this callback will belong to the group. + */ + (title: string, details: TestDetails, callback: () => void): void; } interface TestFunction { - (title: string, testFunction: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; + /** + * Declares a test. + * - `test(title, body)` + * - `test(title, details, body)` + * + * **Usage** + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * **Tags** + * + * You can tag tests by providing additional test details. Note that each tag must start with `@` symbol. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', { + * tag: '@smoke', + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * Test tags are displayed in the test report, and are available to a custom reporter via `TestCase.tags` property. + * + * You can also filter tests by their tags during test execution: + * - in the [command line](https://playwright.dev/docs/test-cli#reference); + * - in the config with + * [testConfig.tagFilter](https://playwright.dev/docs/api/class-testconfig#test-config-tag-filter) and + * [testProject.tagFilter](https://playwright.dev/docs/api/class-testproject#test-project-tag-filter); + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate tests by providing additional test details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * Test annotations are displayed in the test report, and are available to a custom reporter via + * `TestCase.annotations` property. + * + * You can also add dynamic annotations by manipulating + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + * @param title Test title. + * @param details Additional test details. + * @param body Test body that takes one or two arguments: an object with fixtures and optional {@link TestInfo}. + */ + (title: string, body: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; + /** + * Declares a test. + * - `test(title, body)` + * - `test(title, details, body)` + * + * **Usage** + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * **Tags** + * + * You can tag tests by providing additional test details. Note that each tag must start with `@` symbol. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', { + * tag: '@smoke', + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * Test tags are displayed in the test report, and are available to a custom reporter via `TestCase.tags` property. + * + * You can also filter tests by their tags during test execution: + * - in the [command line](https://playwright.dev/docs/test-cli#reference); + * - in the config with + * [testConfig.tagFilter](https://playwright.dev/docs/api/class-testconfig#test-config-tag-filter) and + * [testProject.tagFilter](https://playwright.dev/docs/api/class-testproject#test-project-tag-filter); + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate tests by providing additional test details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, async ({ page }) => { + * await page.goto('https://playwright.dev/'); + * // ... + * }); + * ``` + * + * Test annotations are displayed in the test report, and are available to a custom reporter via + * `TestCase.annotations` property. + * + * You can also add dynamic annotations by manipulating + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + * @param title Test title. + * @param details Additional test details. + * @param body Test body that takes one or two arguments: an object with fixtures and optional {@link TestInfo}. + */ + (title: string, details: TestDetails, body: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; } /** @@ -2469,6 +2839,8 @@ interface TestFunction { export interface TestType extends TestFunction { /** * Declares a focused test. If there are some focused tests or suites, all of them will be run but nothing else. + * - `test.only(title, body)` + * - `test.only(title, details, body)` * * **Usage** * @@ -2479,6 +2851,8 @@ export interface TestType; @@ -2486,6 +2860,7 @@ export interface TestType { @@ -2521,10 +2898,57 @@ export interface TestType { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Annotations** + * + * You can annotate all tests in a group by providing additional details. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.describe('two annotated tests', { + * annotation: { + * type: 'issue', + * description: 'https://github.com/microsoft/playwright/issues/23180', + * }, + * }, () => { + * test('one', async ({ page }) => { + * // ... + * }); + * + * test('two', async ({ page }) => { + * // ... + * }); + * }); + * ``` + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). * @param title Group title. + * @param details Additional details for all tests in the group. * @param callback A callback that is run immediately when calling - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests declared in - * this callback will belong to the group. + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe). Any tests + * declared in this callback will belong to the group. */ describe: SuiteFunction & { /** @@ -2532,6 +2956,7 @@ export interface TestType Promise | void): void; + skip(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; /** * Skip a test. Playwright will not run the test past the `test.skip()` call. * * Skipped tests are not supposed to be ever run. If you intent to fix the test, use - * [test.fixme([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) * instead. * - `test.skip(title, body)` + * - `test.skip(title, details, body)` * - `test.skip()` * - `test.skip(condition, description)` * - `test.skip(callback, description)` @@ -2936,8 +3385,8 @@ export interface TestType Promise | void): void; + /** + * Skip a test. Playwright will not run the test past the `test.skip()` call. + * + * Skipped tests are not supposed to be ever run. If you intent to fix the test, use + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * instead. + * - `test.skip(title, body)` + * - `test.skip(title, details, body)` + * - `test.skip()` + * - `test.skip(condition, description)` + * - `test.skip(callback, description)` + * + * **Usage** + * + * You can declare a skipped test, and Playwright will not run it. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.skip('never run', async ({ page }) => { + * // ... + * }); + * ``` + * + * If your test should be skipped in some configurations, but not all, you can skip the test inside the test body + * based on some condition. We recommend passing a `description` argument in this case. Playwright will run the test, + * but abort it immediately after the `test.skip` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('Safari-only test', async ({ page, browserName }) => { + * test.skip(browserName !== 'webkit', 'This feature is Safari-only'); + * // ... + * }); + * ``` + * + * You can skip all tests in a file or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group based + * on some condition with a single `test.skip(callback, description)` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.skip(({ browserName }) => browserName !== 'webkit', 'Safari-only'); + * + * test('Safari-only test 1', async ({ page }) => { + * // ... + * }); + * test('Safari-only test 2', async ({ page }) => { + * // ... + * }); + * ``` + * + * You can also call `test.skip()` without arguments inside the test body to always mark the test as failed. We + * recommend using `test.skip(title, body)` instead. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('less readable', async ({ page }) => { + * test.skip(); + * // ... + * }); + * ``` + * + * @param title Test title. + * @param details See [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) for test details + * description. * @param body Test body that takes one or two arguments: an object with fixtures and optional {@link TestInfo}. * @param condition Test is marked as "should fail" when the condition is `true`. * @param callback A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as @@ -2976,9 +3503,10 @@ export interface TestType Promise | void): void; + fixme(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; /** * Mark a test as "fixme", with the intention to fix it. Playwright will not run the test past the `test.fixme()` * call. * - `test.fixme(title, body)` + * - `test.fixme(title, details, body)` * - `test.fixme()` * - `test.fixme(condition, description)` * - `test.fixme(callback, description)` @@ -3222,8 +3759,8 @@ export interface TestType Promise | void): void; + /** + * Mark a test as "fixme", with the intention to fix it. Playwright will not run the test past the `test.fixme()` + * call. + * - `test.fixme(title, body)` + * - `test.fixme(title, details, body)` + * - `test.fixme()` + * - `test.fixme(condition, description)` + * - `test.fixme(callback, description)` + * + * **Usage** + * + * You can declare a test as to be fixed, and Playwright will not run it. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fixme('to be fixed', async ({ page }) => { + * // ... + * }); + * ``` + * + * If your test should be fixed in some configurations, but not all, you can mark the test as "fixme" inside the test + * body based on some condition. We recommend passing a `description` argument in this case. Playwright will run the + * test, but abort it immediately after the `test.fixme` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('to be fixed in Safari', async ({ page, browserName }) => { + * test.fixme(browserName === 'webkit', 'This feature breaks in Safari for some reason'); + * // ... + * }); + * ``` + * + * You can mark all tests in a file or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group as + * "fixme" based on some condition with a single `test.fixme(callback, description)` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fixme(({ browserName }) => browserName === 'webkit', 'Should figure out the issue'); + * + * test('to be fixed in Safari 1', async ({ page }) => { + * // ... + * }); + * test('to be fixed in Safari 2', async ({ page }) => { + * // ... + * }); + * ``` + * + * You can also call `test.fixme()` without arguments inside the test body to always mark the test as failed. We + * recommend using `test.fixme(title, body)` instead. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('less readable', async ({ page }) => { + * test.fixme(); + * // ... + * }); + * ``` + * + * @param title Test title. + * @param details See [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) for test details + * description. * @param body Test body that takes one or two arguments: an object with fixtures and optional {@link TestInfo}. * @param condition Test is marked as "should fail" when the condition is `true`. * @param callback A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as @@ -3262,6 +3874,7 @@ export interface TestType Promise | void): void; + fail(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; /** * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful * for documentation purposes to acknowledge that some functionality is broken until it is fixed. * - `test.fail(title, body)` + * - `test.fail(title, details, body)` * - `test.fail()` * - `test.fail(condition, description)` * - `test.fail(callback, description)` @@ -3500,8 +4122,8 @@ export interface TestType Promise | void): void; + /** + * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful + * for documentation purposes to acknowledge that some functionality is broken until it is fixed. + * - `test.fail(title, body)` + * - `test.fail(title, details, body)` + * - `test.fail()` + * - `test.fail(condition, description)` + * - `test.fail(callback, description)` + * + * **Usage** + * + * You can declare a test as failing, so that Playwright ensures it actually fails. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fail('not yet ready', async ({ page }) => { + * // ... + * }); + * ``` + * + * If your test fails in some configurations, but not all, you can mark the test as failing inside the test body based + * on some condition. We recommend passing a `description` argument in this case. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('fail in WebKit', async ({ page, browserName }) => { + * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); + * // ... + * }); + * ``` + * + * You can mark all tests in a file or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group as + * "should fail" based on some condition with a single `test.fail(callback, description)` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fail(({ browserName }) => browserName === 'webkit', 'not implemented yet'); + * + * test('fail in WebKit 1', async ({ page }) => { + * // ... + * }); + * test('fail in WebKit 2', async ({ page }) => { + * // ... + * }); + * ``` + * + * You can also call `test.fail()` without arguments inside the test body to always mark the test as failed. We + * recommend declaring a failing test with `test.fail(title, body)` instead. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('less readable', async ({ page }) => { + * test.fail(); + * // ... + * }); + * ``` + * + * @param title Test title. + * @param details See [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) for test details + * description. * @param body Test body that takes one or two arguments: an object with fixtures and optional {@link TestInfo}. * @param condition Test is marked as "should fail" when the condition is `true`. * @param callback A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as @@ -3540,6 +4236,7 @@ export interface TestType { @@ -3905,8 +4607,8 @@ export interface TestType Promise | any): void; /** * Specifies options or fixtures to use in a single test file or a - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) group. Most useful to - * set an option, for example set `locale` to configure `context` fixture. + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group. Most + * useful to set an option, for example set `locale` to configure `context` fixture. * * **Usage** * @@ -8023,6 +8725,35 @@ interface TestProject { */ snapshotPathTemplate?: string; + /** + * Filter to only run tests with or without particular tag(s). This property must be a logical expression containing + * tags, parenthesis `(` and `)`, and operators `and`, `or` and `not`. + * + * Learn more about [tagging](https://playwright.dev/docs/test-annotations#tag-tests). + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * projects: [ + * { + * name: 'tests', + * tagFilter: '@smoke', + * }, + * ], + * }); + * ``` + * + * Tag expression examples: + * - `tagFilter: '@smoke'` - run only "smoke" tests; + * - `tagFilter: '@smoke and not @production'` - run only "smoke" tests that are not tagged as "production"; + * - `tagFilter: '(@smoke or @fast) and @v2'` - run "v2" tests that are tagged as "smoke", "fast" or both. + */ + tagFilter?: string; + /** * Name of a project that needs to run after this and all dependent projects have finished. Teardown is useful to * cleanup any resources acquired by this project. diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 19735e972b..01ec9e4fdf 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -26,7 +26,8 @@ export type { FullConfig, TestStatus } from './test'; * - {@link TestCase} #1 * - {@link TestCase} #2 * - Suite corresponding to a - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) group + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) + * group * - {@link TestCase} #1 in a group * - {@link TestCase} #2 in a group * - < more test cases ... > @@ -71,8 +72,9 @@ export interface Suite { /** * Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases - * defined in nested [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) - * groups are listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). + * defined in nested + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups are + * listed in the child [suite.suites](https://playwright.dev/docs/api/class-suite#suite-suites). */ tests: Array; @@ -81,28 +83,30 @@ export interface Suite { * - Empty for root suite. * - Project name for project suite. * - File path for file suite. - * - Title passed to [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) - * for a group suite. + * - Title passed to + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for a + * group suite. */ title: string; } /** - * `TestCase` corresponds to every [test.(call)(title, body)](https://playwright.dev/docs/api/class-test#test-call) - * call in a test file. When a single [test.(call)(title, body)](https://playwright.dev/docs/api/class-test#test-call) - * is running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in - * corresponding projects' suites. + * `TestCase` corresponds to every + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call in a test file. + * When a single [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) is + * running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding + * projects' suites. */ export interface TestCase { /** * Expected test status. * - Tests marked as - * [test.skip([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip) + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip) * or - * [test.fixme([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) * are expected to be `'skipped'`. * - Tests marked as - * [test.fail([title, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) * are expected to be `'failed'`. * - Other tests are expected to be `'passed'`. * @@ -129,9 +133,18 @@ export interface TestCase { titlePath(): Array; /** - * The list of annotations applicable to the current test. Includes annotations from the test, annotations from all - * [test.describe([title, callback])](https://playwright.dev/docs/api/class-test#test-describe) groups the test - * belongs to and file-level annotations for the test file. + * The list of annotations applicable to the current test. Includes: + * - annotations defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) and + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe); + * - annotations implicitly added by methods + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * and + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail); + * - annotations appended to + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test + * execution. * * Annotations are available during test execution through * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). @@ -184,6 +197,15 @@ export interface TestCase { */ retries: number; + /** + * The list of tags defined on the test or suite via + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe). + * + * Learn more about [test tags](https://playwright.dev/docs/test-annotations#tag-tests). + */ + tags: Array; + /** * The timeout given to the test. Affected by * [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout), @@ -195,7 +217,8 @@ export interface TestCase { timeout: number; /** - * Test title as passed to the [test.(call)(title, body)](https://playwright.dev/docs/api/class-test#test-call) call. + * Test title as passed to the + * [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) call. */ title: string; } diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 0225b6c0e2..5f38b0e212 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -1126,7 +1126,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { `, 'c.test.js': ` const { expect, test } = require('@playwright/test'); - test('@regression @failed failed', async ({}) => { + test('@regression @failed failed', { tag: '@foo' }, async ({}) => { expect(1).toBe(2); }); test('@regression @flaky flaky', async ({}, testInfo) => { @@ -1135,7 +1135,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { else expect(1).toBe(2); }); - test.skip('@regression skipped', async ({}) => { + test.skip('@regression skipped', { tag: ['@foo', '@bar'] }, async ({}) => { expect(1).toBe(2); }); `, @@ -1147,15 +1147,17 @@ for (const useIntermediateMergeReport of [false, true] as const) { await showReport(); - await expect(page.locator('.test-file-test .label')).toHaveCount(42); await expect(page.locator('.test-file-test', { has: page.getByText('@regression @failed failed', { exact: true }) }).locator('.label')).toHaveText([ 'chromium', + 'foo', 'failed', 'regression', 'firefox', + 'foo', 'failed', 'regression', 'webkit', + 'foo', 'failed', 'regression' ]); @@ -1172,10 +1174,16 @@ for (const useIntermediateMergeReport of [false, true] as const) { ]); await expect(page.locator('.test-file-test', { has: page.getByText('@regression skipped', { exact: true }) }).locator('.label')).toHaveText([ 'chromium', + 'foo', + 'bar', 'regression', 'firefox', + 'foo', + 'bar', 'regression', 'webkit', + 'foo', + 'bar', 'regression', ]); await expect(page.locator('.test-file-test', { has: page.getByText('@smoke @passed passed', { exact: true }) }).locator('.label')).toHaveText([ diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index f86f22d4fd..e9353da055 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -160,29 +160,15 @@ test('should display tags separately from title', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.js': ` import { test, expect } from '@playwright/test'; - test('math works! @USR-MATH-001 @USR-MATH-002', async ({}) => { - expect(1 + 1).toBe(2); - await test.step('math works in a step', async () => { - expect(2 + 2).toBe(4); - await test.step('nested step', async () => { - expect(2 + 2).toBe(4); - await test.step('deeply nested step', async () => { - expect(2 + 2).toBe(4); - }); - }) - }) + test('math works! @USR-MATH-001 @USR-MATH-002', { tag: '@foo' }, async ({}) => { + test.info().annotations.push({ type: 'issue', description: 'issue-link' }); + test.info().annotations.push({ type: 'novalue' }); }); ` }); expect(result.exitCode).toBe(0); - expect(result.report.suites.length).toBe(1); - expect(result.report.suites[0].specs.length).toBe(1); - // Ensure the length is as expected - expect(result.report.suites[0].specs[0].tags.length).toBe(2); - // Ensure that the '@' value is stripped - expect(result.report.suites[0].specs[0].tags[0]).toBe('USR-MATH-001'); - expect(result.report.suites[0].specs[0].tags[1]).toBe('USR-MATH-002'); + expect(result.report.suites[0].specs[0].tags).toEqual(['@foo', '@USR-MATH-001', '@USR-MATH-002']); }); test('should have relative always-posix paths', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 653728fd78..aaa24f7fbd 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -824,3 +824,96 @@ test('should report a stable test.id', async ({ runInlineTest }) => { 'testbegin-20289bcdad95a5e18c38-8b63c3695b9c8bd62d98', ]); }); + +test('should report annotations from test declaration', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + export default class Reporter { + onBegin(config, suite) { + const visit = suite => { + for (const test of suite.tests || []) { + const annotations = test.annotations.map(a => { + return a.description ? a.type + '=' + a.description : a.type; + }); + console.log('\\n%%title=' + test.title + ', annotations=' + annotations.join(',')); + } + for (const child of suite.suites || []) + visit(child); + }; + visit(suite); + } + onError(error) { + console.log(error); + } + } + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('none', () => { + expect(test.info().annotations).toEqual([]); + }); + test('foo', { annotation: { type: 'foo' } }, () => { + expect(test.info().annotations).toEqual([{ type: 'foo' }]); + }); + test('foo-bar', { + annotation: [ + { type: 'foo', description: 'desc' }, + { type: 'bar' }, + ], + }, () => { + expect(test.info().annotations).toEqual([ + { type: 'foo', description: 'desc' }, + { type: 'bar' }, + ]); + }); + test.skip('skip-foo', { annotation: { type: 'foo' } }, () => { + }); + test.fixme('fixme-bar', { annotation: { type: 'bar' } }, () => { + }); + test.fail('fail-foo-bar', { + annotation: [ + { type: 'foo' }, + { type: 'bar', description: 'desc' }, + ], + }, () => { + expect(1).toBe(2); + }); + test.describe('suite', { annotation: { type: 'foo' } }, () => { + test('foo-suite', () => { + expect(test.info().annotations).toEqual([{ type: 'foo' }]); + }); + test.describe('inner', { annotation: { type: 'bar' } }, () => { + test('foo-bar-suite', () => { + expect(test.info().annotations).toEqual([{ type: 'foo' }, { type: 'bar' }]); + }); + }); + }); + test.describe.skip('skip-foo-suite', { annotation: { type: 'foo' } }, () => { + test('skip-foo-suite', () => { + }); + }); + test.describe.fixme('fixme-bar-suite', { annotation: { type: 'bar' } }, () => { + test('fixme-bar-suite', () => { + }); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `title=none, annotations=`, + `title=foo, annotations=foo`, + `title=foo-bar, annotations=foo=desc,bar`, + `title=skip-foo, annotations=foo,skip`, + `title=fixme-bar, annotations=bar,fixme`, + `title=fail-foo-bar, annotations=foo,bar=desc,fail`, + `title=foo-suite, annotations=foo`, + `title=foo-bar-suite, annotations=foo,bar`, + `title=skip-foo-suite, annotations=foo,skip`, + `title=fixme-bar-suite, annotations=bar,fixme`, + ]); +}); diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts index 1d7f5af83f..0b94769107 100644 --- a/tests/playwright-test/test-modifiers.spec.ts +++ b/tests/playwright-test/test-modifiers.spec.ts @@ -189,7 +189,7 @@ test.describe('test modifier annotations', () => { expect(result.passed).toBe(0); expect(result.skipped).toBe(6); expectTest('no marker', 'skipped', 'skipped', ['fixme']); - expectTest('skip wrap', 'skipped', 'skipped', ['skip', 'fixme']); + expectTest('skip wrap', 'skipped', 'skipped', ['fixme', 'skip']); expectTest('skip inner', 'skipped', 'skipped', ['fixme']); expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'fixme']); expectTest('fixme inner', 'skipped', 'skipped', ['fixme']); @@ -220,7 +220,7 @@ test.describe('test modifier annotations', () => { expectTest('no marker', 'skipped', 'skipped', ['skip']); expectTest('skip wrap', 'skipped', 'skipped', ['skip', 'skip']); expectTest('skip inner', 'skipped', 'skipped', ['skip']); - expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'skip']); + expectTest('fixme wrap', 'skipped', 'skipped', ['skip', 'fixme']); expectTest('fixme inner', 'skipped', 'skipped', ['skip']); expectTest('example', 'passed', 'expected', []); }); @@ -251,7 +251,7 @@ test.describe('test modifier annotations', () => { expect(result.exitCode).toBe(0); expect(result.passed).toBe(0); expect(result.skipped).toBe(2); - expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'fixme', 'skip', 'skip', 'fixme']); + expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'skip', 'skip', 'fixme', 'fixme']); expectTest('fixme inner', 'skipped', 'skipped', ['fixme', 'skip', 'skip', 'fixme']); }); diff --git a/tests/playwright-test/test-tag.spec.ts b/tests/playwright-test/test-tag.spec.ts new file mode 100644 index 0000000000..b1120b0c41 --- /dev/null +++ b/tests/playwright-test/test-tag.spec.ts @@ -0,0 +1,245 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should have correct tags', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + export default class Reporter { + onBegin(config, suite) { + const visit = suite => { + for (const test of suite.tests || []) + console.log('\\n%%title=' + test.title + ', tags=' + test.tags.join(',')); + for (const child of suite.suites || []) + visit(child); + }; + visit(suite); + } + onError(error) { + console.log(error); + } + } + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('no-tags', () => { + expect(test.info()._test.tags).toEqual([]); + }); + test('foo-tag', { tag: '@foo' }, () => { + expect(test.info()._test.tags).toEqual(['@foo']); + }); + test('foo-bar-tags', { tag: ['@foo', '@bar'] }, () => { + expect(test.info()._test.tags).toEqual(['@foo', '@bar']); + }); + test.skip('skip-foo-tag', { tag: '@foo' }, () => { + }); + test.fixme('fixme-bar-tag', { tag: '@bar' }, () => { + }); + test.fail('fail-foo-bar-tags', { tag: ['@foo', '@bar'] }, () => { + expect(1).toBe(2); + }); + test.describe('suite', { tag: '@foo' }, () => { + test('foo-suite', () => { + expect(test.info()._test.tags).toEqual(['@foo']); + }); + test.describe('inner', { tag: '@bar' }, () => { + test('foo-bar-suite', () => { + expect(test.info()._test.tags).toEqual(['@foo', '@bar']); + }); + }); + }); + test.describe.skip('skip-foo-suite', { tag: '@foo' }, () => { + test('skip-foo-suite', () => { + }); + }); + test.describe.fixme('fixme-bar-suite', { tag: '@bar' }, () => { + test('fixme-bar-suite', () => { + }); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `title=no-tags, tags=`, + `title=foo-tag, tags=@foo`, + `title=foo-bar-tags, tags=@foo,@bar`, + `title=skip-foo-tag, tags=@foo`, + `title=fixme-bar-tag, tags=@bar`, + `title=fail-foo-bar-tags, tags=@foo,@bar`, + `title=foo-suite, tags=@foo`, + `title=foo-bar-suite, tags=@foo,@bar`, + `title=skip-foo-suite, tags=@foo`, + `title=fixme-bar-suite, tags=@bar`, + ]); +}); + +test('config.tagFilter should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { tagFilter: '@tag1' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@tag1' }, async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual(['test1']); +}); + +test('config.project.tag should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'p1' }, + { name: 'p2', tagFilter: '@tag1' } + ] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@tag1' }, async () => { console.log('\\n%% test1-' + test.info().project.name); }); + test('test2', async () => { console.log('\\n%% test2-' + test.info().project.name); }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.outputLines).toEqual(['test1-p1', 'test2-p1', 'test1-p2']); +}); + +test('--tag should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@tag1' }, async () => { console.log('\\n%% test1'); }); + test('test2', async () => { console.log('\\n%% test2'); }); + `, + }, { tag: '@tag1' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual(['test1']); +}); + +test('should parse tag expressions', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'p1', tagFilter: '@foo' }, + { name: 'p2', tagFilter: 'not @foo' }, + { name: 'p3', tagFilter: ' @foo and @bar' }, + { name: 'p4', tagFilter: '@bar or not @foo' }, + { name: 'p5', tagFilter: '@bar and (@foo or not @foo)' }, + { name: 'p6', tagFilter: '@qux or @foo and @bar' }, + { name: 'p7', tagFilter: '@qux and (@foo or @bar)' }, + { name: 'p8', tagFilter: 'not not not @foo' }, + ] + }; + `, + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@foo' }, () => { + console.log('\\n%% foo-' + test.info().project.name); + }); + test('test2', { tag: '@bar' }, () => { + console.log('\\n%% bar-' + test.info().project.name); + }); + test('test3', { tag: ['@foo', '@bar'] }, () => { + console.log('\\n%% foobar-' + test.info().project.name); + }); + ` + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `foo-p1`, + `foobar-p1`, + `bar-p2`, + `foobar-p3`, + `bar-p4`, + `foobar-p4`, + `bar-p5`, + `foobar-p5`, + `foobar-p6`, + `bar-p8`, + ]); +}); + +test('should enforce @ symbol', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: 'foo' }, () => { + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Error: Tag must start with "@" symbol, got "foo" instead.`); +}); + +test('should report tag expression error 1', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@foo' }, () => { + }); + ` + }, { tag: '(@foo' }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Error: Expected matching ")" when parsing tag expression: (@foo`); +}); + +test('should report tag expression error 2', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@foo' }, () => { + }); + ` + }, { tag: '(@foo)@bar' }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Error: Unexpected extra tokens in the tag expression: (@foo)@bar`); +}); + +test('should report tag expression error 3', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@foo' }, () => { + }); + ` + }, { tag: '@foo and' }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Error: Unexpected end of tag expression: @foo and`); +}); + +test('should report tag expression error 4', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'stdio.spec.js': ` + import { test, expect } from '@playwright/test'; + test('test1', { tag: '@foo' }, () => { + }); + ` + }, { tag: '@foo @bar' }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Error: Unexpected extra tokens in the tag expression: @foo @bar`); +}); diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index 84b55f1974..e61d4870ed 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -47,6 +47,25 @@ test('basics should work', async ({ runTSC }) => { test.foo(); test.describe.configure({ mode: 'parallel' }); test.describe.configure({ retries: 3, timeout: 123 }); + test('title', { tag: '@foo' }, () => {}); + test('title', { tag: ['@foo', '@bar'] }, () => {}); + test('title', { annotation: { type: 'issue' } }, () => {}); + test('title', { annotation: [{ type: 'issue' }, { type: 'foo', description: 'bar' }] }, () => {}); + test('title', { + tag: '@foo', + annotation: { type: 'issue' }, + }, () => {}); + test.skip('title', { tag: '@foo' }, () => {}); + test.fixme('title', { tag: '@foo' }, () => {}); + test.only('title', { tag: '@foo' }, () => {}); + test.fail('title', { tag: '@foo' }, () => {}); + test.describe('title', { tag: '@foo' }, () => {}); + test.describe('title', { annotation: { type: 'issue' } }, () => {}); + // @ts-expect-error + test.describe({ tag: '@foo' }, () => {}); + test.describe.skip('title', { tag: '@foo' }, () => {}); + test.describe.fixme('title', { tag: '@foo' }, () => {}); + test.describe.only('title', { tag: '@foo' }, () => {}); ` }); expect(result.exitCode).toBe(0); diff --git a/utils/generate_types/index.js b/utils/generate_types/index.js index d41750e822..e16f6df772 100644 --- a/utils/generate_types/index.js +++ b/utils/generate_types/index.js @@ -101,6 +101,11 @@ class TypesGenerator { const method = cls.membersArray.find(m => m.alias === 'describe'); return this.memberJSDOC(method, ' ').trimLeft(); } + if (className === 'TestFunction' && methodName === '__call') { + const cls = this.documentation.classes.get('Test'); + const method = cls.membersArray.find(m => m.alias === '(call)'); + return this.memberJSDOC(method, ' ').trimLeft(); + } const docClass = this.docClassForName(className); let method; diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 6ef214bc54..fd88e84dca 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -111,13 +111,25 @@ export interface TestInfo { project: FullProject; } +type TestDetailsAnnotation = { + type: string; + description?: string; +}; + +export type TestDetails = { + tag?: string | string[]; + annotation?: TestDetailsAnnotation | TestDetailsAnnotation[]; +} + interface SuiteFunction { (title: string, callback: () => void): void; (callback: () => void): void; + (title: string, details: TestDetails, callback: () => void): void; } interface TestFunction { - (title: string, testFunction: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; + (title: string, body: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; + (title: string, details: TestDetails, body: (args: TestArgs, testInfo: TestInfo) => Promise | void): void; } export interface TestType extends TestFunction { @@ -134,15 +146,18 @@ export interface TestType void; }; - skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + skip(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + skip(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; skip(): void; skip(condition: boolean, description?: string): void; skip(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; - fixme(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + fixme(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + fixme(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; fixme(): void; fixme(condition: boolean, description?: string): void; fixme(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; - fail(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + fail(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + fail(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; fail(condition: boolean, description?: string): void; fail(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; fail(): void;