diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index 12a953e800..4e0ecf3be3 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -150,3 +150,27 @@ await expect.poll(async () => { timeout: 60_000 }).toBe(200); ``` + +## Retrying + +You can retry blocks of code until they are passing successfully. + +```js +await expect(async () => { + const response = await page.request.get('https://api.example.com'); + expect(response.status()).toBe(200); +}).toPass(); +``` + +You can also specify custom timeout for retry intervals: + +```js +await expect(async () => { + const response = await page.request.get('https://api.example.com'); + expect(response.status()).toBe(200); +}).toPass({ + // Probe, wait 1s, probe, wait 2s, probe, wait 10s, probe, wait 10s, probe, .... Defaults to [100, 250, 500, 1000]. + intervals: [1_000, 2_000, 10_000], + timeout: 60_000 +}); +``` diff --git a/packages/playwright-test/src/expect.ts b/packages/playwright-test/src/expect.ts index eb17f3b7d5..55f6139446 100644 --- a/packages/playwright-test/src/expect.ts +++ b/packages/playwright-test/src/expect.ts @@ -38,6 +38,7 @@ import { toHaveURL, toHaveValue, toHaveValues, + toPass } from './matchers/matchers'; import { toMatchSnapshot, toHaveScreenshot } from './matchers/toMatchSnapshot'; import type { Expect } from './types'; @@ -103,7 +104,7 @@ function createExpect(actual: unknown, messageOrOptions: ExpectMessageOrOptions, return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(messageOrOptions, isSoft, isPoll, generator)); } -export const expect: Expect = new Proxy(expectLibrary as any, { +export const expect: Expect = new Proxy(expectLibrary, { apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, messageOrOptions: ExpectMessageOrOptions]) { const [actual, messageOrOptions] = argumentsList; return createExpect(actual, messageOrOptions, false /* isSoft */, false /* isPoll */); @@ -145,6 +146,7 @@ const customMatchers = { toHaveValues, toMatchSnapshot, toHaveScreenshot, + toPass, }; type Generator = () => any; diff --git a/packages/playwright-test/src/matchers/matchers.ts b/packages/playwright-test/src/matchers/matchers.ts index 2a463be613..3d39c390fb 100644 --- a/packages/playwright-test/src/matchers/matchers.ts +++ b/packages/playwright-test/src/matchers/matchers.ts @@ -25,6 +25,8 @@ import { toEqual } from './toEqual'; import { toExpectedTextValues, toMatchText } from './toMatchText'; import type { ParsedStackTrace } from 'playwright-core/lib/utils/stackTrace'; import { isTextualMimeType } from 'playwright-core/lib/utils/mimeType'; +import { monotonicTime } from 'playwright-core/lib/utils'; +import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner'; interface LocatorEx extends Locator { _expect(customStackTrace: ParsedStackTrace, expression: string, options: Omit & { expectedValue?: any }): Promise<{ matches: boolean, received?: any, log?: string[], timedOut?: boolean }>; @@ -309,3 +311,52 @@ export async function toBeOK( const pass = response.ok(); return { message, pass }; } + +export async function toPass( + this: ReturnType, + callback: () => void, + options: { + intervals?: number[]; + timeout?: number, + } = {}, +) { + let matcherError: Error | undefined; + const startTime = monotonicTime(); + const pollIntervals = options.intervals || [100, 250, 500, 1000]; + const lastPollInterval = pollIntervals[pollIntervals.length - 1] || 1000; + const timeout = options.timeout !== undefined ? options.timeout : 0; + const isNot = this.isNot; + + while (true) { + const elapsed = monotonicTime() - startTime; + if (timeout !== 0 && elapsed > timeout) + break; + try { + const wrappedCallback = () => Promise.resolve().then(callback); + const received = timeout !== 0 ? await raceAgainstTimeout(wrappedCallback, timeout - elapsed) + : await wrappedCallback().then(() => ({ timedOut: false })); + if (received.timedOut) + break; + // The check passed, exit sucessfully. + if (isNot) + matcherError = new Error('Expected to fail, but passed'); + else + return { message: () => '', pass: true }; + } catch (e) { + if (isNot) + return { message: () => '', pass: false }; + matcherError = e; + } + await new Promise(x => setTimeout(x, pollIntervals!.shift() ?? lastPollInterval)); + } + + const timeoutMessage = `Timeout ${timeout}ms exceeded while waiting on the predicate`; + const message = () => matcherError ? [ + matcherError.message, + '', + `Call Log:`, + `- ${timeoutMessage}`, + ].join('\n') : timeoutMessage; + + return { message, pass: false }; +} diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index fa5da42c34..2ae64ac621 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -3287,14 +3287,20 @@ type MakeMatchers = BaseMatchers & { */ resolves: MakeMatchers, Awaited>; /** - * Unwraps the reason of a rejected promise so any other matcher can be chained. - * If the promise is fulfilled the assertion fails. - */ + * Unwraps the reason of a rejected promise so any other matcher can be chained. + * If the promise is fulfilled the assertion fails. + */ rejects: MakeMatchers, Awaited>; } & SnapshotAssertions & ExtraMatchers & ExtraMatchers & - ExtraMatchers; + ExtraMatchers & + ExtraMatchers; + }>; type BaseExpect = { // Removed following methods because they rely on a test-runner integration from Jest which we don't support: diff --git a/tests/playwright-test/expect-to-pass.spec.ts b/tests/playwright-test/expect-to-pass.spec.ts new file mode 100644 index 0000000000..d76cf094c1 --- /dev/null +++ b/tests/playwright-test/expect-to-pass.spec.ts @@ -0,0 +1,169 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, stripAnsi } from './playwright-test-fixtures'; + +test('should retry predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should toPass sync predicate', async () => { + let i = 0; + await test.expect(() => { + expect(++i).toBe(3); + }).toPass(); + expect(i).toBe(3); + }); + test('should toPass async predicate', async () => { + let i = 0; + await test.expect(async () => { + await new Promise(x => setTimeout(x, 50)); + expect(++i).toBe(3); + }).toPass(); + expect(i).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); +}); + +test('should respect timeout', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async () => { + await test.expect(() => { + expect(1).toBe(2); + }).toPass({ timeout: 100 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded while waiting on the predicate'); + expect(stripAnsi(result.output)).toContain('Received: 1'); + expect(stripAnsi(result.output)).toContain(` + 7 | await test.expect(() => { + `.trim()); +}); + +test('should not fail when used with web-first assertion', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async ({ page }) => { + let i = 0; + await test.expect(async () => { + if (++i < 3) + await expect(page.locator('body')).toHaveText('foo', { timeout: 1 }); + }).toPass(); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); +}); + +test('should support .not predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should pass', async ({ page }) => { + let i = 0; + await test.expect(() => { + expect(++i).toBeLessThan(3); + }).not.toPass(); + expect(i).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); +}); + +test('should respect interval', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async () => { + let probes = 0; + await test.expect(() => { + ++probes + expect(1).toBe(2); + }).toPass({ timeout: 1000, intervals: [600] }).catch(() => {}); + // Probe at 0s, at 0.6s. + expect(probes).toBe(2); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should compile', async ({ runTSC }) => { + const result = await runTSC({ + 'a.spec.ts': ` + const { test } = pwt; + test('should poll sync predicate', async ({ page }) => { + let i = 0; + test.expect(() => ++i).toPass(); + test.expect(() => ++i, 'message').toPass(); + test.expect(() => ++i, { message: 'message' }).toPass(); + test.expect(() => ++i).toPass({ timeout: 100 }); + test.expect(() => ++i, { message: 'message' }).toPass({ timeout: 100 }); + test.expect(async () => { + await new Promise(x => setTimeout(x, 50)); + return ++i; + }).toPass(); + test.expect(() => Promise.resolve(++i)).toPass(); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should use custom message', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail with custom message', async () => { + await test.expect(() => { + expect(1).toBe(3); + }, 'Custom message').toPass({ timeout: 1 }); + }); + ` + }); + expect(stripAnsi(result.output)).toContain('Error: Custom message'); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); +}); + +test('should work with soft', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should respect soft', async () => { + await test.expect.soft(() => { + expect(1).toBe(3); + }).toPass({ timeout: 1 }); + expect.soft(2).toBe(3); + }); + ` + }); + expect(stripAnsi(result.output)).toContain('Received: 1'); + expect(stripAnsi(result.output)).toContain('Received: 2'); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); +}); \ No newline at end of file diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index fe0db93be4..8f308ee474 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -293,14 +293,20 @@ type MakeMatchers = BaseMatchers & { */ resolves: MakeMatchers, Awaited>; /** - * Unwraps the reason of a rejected promise so any other matcher can be chained. - * If the promise is fulfilled the assertion fails. - */ + * Unwraps the reason of a rejected promise so any other matcher can be chained. + * If the promise is fulfilled the assertion fails. + */ rejects: MakeMatchers, Awaited>; } & SnapshotAssertions & ExtraMatchers & ExtraMatchers & - ExtraMatchers; + ExtraMatchers & + ExtraMatchers; + }>; type BaseExpect = { // Removed following methods because they rely on a test-runner integration from Jest which we don't support: