From a0aeaeb929d0615156c86253e43633d17dd60232 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 3 Jan 2022 17:29:54 -0800 Subject: [PATCH] test: expose repeatEachIndex (#11158) --- .github/workflows/tests_primary.yml | 1 - docs/src/test-reporter-api/class-testcase.md | 5 ++++ packages/playwright-test/src/project.ts | 4 ++-- packages/playwright-test/src/reporters/raw.ts | 4 ++-- packages/playwright-test/src/runner.ts | 2 +- packages/playwright-test/src/test.ts | 2 +- .../playwright-test/types/testReporter.d.ts | 5 ++++ tests/playwright-test/reporter-html.spec.ts | 23 +++++++++++++++++++ .../overrides-testReporter.d.ts | 1 + 9 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 4aa2e57c3b..764194b4f4 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -9,7 +9,6 @@ on: paths-ignore: - 'browser_patches/**' - 'docs/**' - types: [ labeled ] branches: - main - release-* diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 0ca5a2dbba..1423ad4c65 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -46,6 +46,11 @@ Testing outcome for this test. Note that outcome is not the same as [`property: Suite this test case belongs to. +## property: TestCase.repeatEachIndex +- type: <[int]> + +Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the [command line](./test-cli.md). + ## property: TestCase.results - type: <[Array]<[TestResult]>> diff --git a/packages/playwright-test/src/project.ts b/packages/playwright-test/src/project.ts index 0a4a7ac536..50df921218 100644 --- a/packages/playwright-test/src/project.ts +++ b/packages/playwright-test/src/project.ts @@ -79,7 +79,7 @@ export class ProjectImpl { const test = entry._clone(); test.retries = this.config.retries; test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`; - test._repeatEachIndex = repeatEachIndex; + test.repeatEachIndex = repeatEachIndex; test._projectIndex = this.index; to._addTest(test); if (!filter(test)) { @@ -100,7 +100,7 @@ export class ProjectImpl { clone._pool = this.buildPool(hook); clone._projectIndex = this.index; clone._id = `${hook._ordinalInFile}@${hook._requireFile}#run${this.index}-repeat${repeatEachIndex}`; - clone._repeatEachIndex = repeatEachIndex; + clone.repeatEachIndex = repeatEachIndex; to._addAllHook(clone); } return true; diff --git a/packages/playwright-test/src/reporters/raw.ts b/packages/playwright-test/src/reporters/raw.ts index d1df6fbe52..d62c51c220 100644 --- a/packages/playwright-test/src/reporters/raw.ts +++ b/packages/playwright-test/src/reporters/raw.ts @@ -181,8 +181,8 @@ class RawReporter { } private _serializeSuite(suite: Suite): JsonSuite { - const fileId = calculateSha1(suite.location!.file.split(path.sep).join('/')); const location = this._relativeLocation(suite.location); + const fileId = calculateSha1(location!.file.split(path.sep).join('/')); return { title: suite.title, fileId, @@ -195,7 +195,7 @@ class RawReporter { private _serializeTest(test: TestCase, fileId: string): JsonTestCase { const [, projectName, , ...titles] = test.titlePath(); - const testIdExpression = `project:${projectName}|path:${titles.join('>')}`; + const testIdExpression = `project:${projectName}|path:${titles.join('>')}|repeat:${test.repeatEachIndex}`; const testId = fileId + '-' + calculateSha1(testIdExpression); return { testId, diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index cb479d05c3..6d3ba0e793 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -475,7 +475,7 @@ function createTestGroups(rootSuite: Suite): TestGroup[] { return { workerHash: test._workerHash, requireFile: test._requireFile, - repeatEachIndex: test._repeatEachIndex, + repeatEachIndex: test.repeatEachIndex, projectIndex: test._projectIndex, tests: [], }; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index 06c707367b..a2cff97b63 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -131,6 +131,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { timeout = 0; annotations: Annotations = []; retries = 0; + repeatEachIndex = 0; _type: TestCaseType; _ordinalInFile: number; @@ -138,7 +139,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { _id = ''; _workerHash = ''; _pool: FixturePool | undefined; - _repeatEachIndex = 0; _projectIndex = 0; constructor(type: TestCaseType, title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) { diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index 7c02b629bb..a6d328e08e 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -166,6 +166,11 @@ export interface TestCase { * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). */ retries: number; + /** + * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the + * [command line](https://playwright.dev/docs/test-cli). + */ + repeatEachIndex: number; /** * Results for each run of this test. */ diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 6bf898ec4b..adbeede312 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -424,3 +424,26 @@ test('should strikethough textual diff with commonalities', async ({ runInlineTe const stricken = await page.locator('css=strike').innerText(); expect(stricken).toBe('old'); }); + +test('should differentiate repeat-each test cases', async ({ runInlineTest, showReport, page }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/10859' }); + const result = await runInlineTest({ + 'a.spec.js': ` + const { test } = pwt; + test('sample', async ({}, testInfo) => { + if (testInfo.repeatEachIndex === 2) + throw new Error('ouch'); + }); + ` + }, { 'reporter': 'dot,html', 'repeat-each': 3 }); + expect(result.exitCode).toBe(1); + await showReport(); + + await page.locator('text=sample').first().click(); + await expect(page.locator('text=ouch')).toBeVisible(); + await page.locator('text=All').first().click(); + + await page.locator('text=sample').nth(1).click(); + await expect(page.locator('text=Before Hooks')).toBeVisible(); + await expect(page.locator('text=ouch')).toBeHidden(); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index bc4a1c9a24..4f9937a130 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -44,6 +44,7 @@ export interface TestCase { timeout: number; annotations: { type: string, description?: string }[]; retries: number; + repeatEachIndex: number; results: TestResult[]; outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky'; ok(): boolean;