From 02edc5ad4fd347d78ac074b69a5f4f0415d27e71 Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Wed, 21 Feb 2024 11:47:44 -0600 Subject: [PATCH] Updated browser tests to be less flaky (#19701) no refs - Offers browser tests were subject to a race condition. I'm guessing this dates back to when we moved to Settings X (and React), as it seems the url for the offer is not present on the first render of the page - despite being returned in the `POST` request of the offer creation, the component does a `GET` on render to get the link. This is now awaited. - The Publishing timezone test also seemed to suffer from a race condition. This is less sure of a fix as it's a much less frequent failure. The date time picker input is now validated in the test before continuing. - Offers browser tests often timed out so the timeout has been moved to 90s for these tests. - All tests were bumped to 75s timeout as we generally would occasionally hit the timeout. --- ghost/core/playwright.config.js | 3 +- .../test/e2e-browser/admin/publishing.spec.js | 7 +- .../test/e2e-browser/portal/offers.spec.js | 28 +++-- .../e2e-browser/utils/e2e-browser-utils.js | 108 ++++++++++-------- 4 files changed, 86 insertions(+), 60 deletions(-) diff --git a/ghost/core/playwright.config.js b/ghost/core/playwright.config.js index 9852831aa7..23946f3c04 100644 --- a/ghost/core/playwright.config.js +++ b/ghost/core/playwright.config.js @@ -1,7 +1,7 @@ /** @type {import('@playwright/test').PlaywrightTestConfig} */ const config = { - timeout: 60 * 1000, + timeout: 75 * 1000, expect: { timeout: 10000 }, @@ -9,6 +9,7 @@ const config = { workers: process.env.CI ? '100%' : (process.env.PLAYWRIGHT_SLOWMO ? 1 : undefined), reporter: process.env.CI ? [['list', {printSteps: true}], ['html']] : [['list', {printSteps: true}]], use: { + // trace: 'retain-on-failure', // Use a single browser since we can't simultaneously test multiple browsers browserName: 'chromium', headless: !process.env.PLAYWRIGHT_DEBUG, diff --git a/ghost/core/test/e2e-browser/admin/publishing.spec.js b/ghost/core/test/e2e-browser/admin/publishing.spec.js index 736a00b2d3..3325b873e7 100644 --- a/ghost/core/test/e2e-browser/admin/publishing.spec.js +++ b/ghost/core/test/e2e-browser/admin/publishing.spec.js @@ -571,7 +571,12 @@ test.describe('Updating post access', () => { await page.locator('[data-test-date-time-picker-datepicker]').click(); await page.locator('.ember-power-calendar-nav-control--previous').click(); await page.locator('.ember-power-calendar-day', {hasText: '15'}).click(); - await page.locator('[data-test-date-time-picker-time-input]').fill('12:00'); + const dateTimePickerInput = await page.locator('[data-test-date-time-picker-time-input]'); + dateTimePickerInput.fill('12:00'); + await page.keyboard.press('Tab'); + + // test will not work if the field is not filled appropriately + await expect(dateTimePickerInput).toHaveValue('12:00'); await publishPost(page); await closePublishFlow(page); diff --git a/ghost/core/test/e2e-browser/portal/offers.spec.js b/ghost/core/test/e2e-browser/portal/offers.spec.js index f6519cf263..d33d637517 100644 --- a/ghost/core/test/e2e-browser/portal/offers.spec.js +++ b/ghost/core/test/e2e-browser/portal/offers.spec.js @@ -3,6 +3,7 @@ const test = require('../fixtures/ghost-test'); const {deleteAllMembers, createTier, createOffer, completeStripeSubscription} = require('../utils'); test.describe('Portal', () => { + test.setTimeout(90000); // override the default 60s in the config as these retries can run close to 60s test.describe('Offers', () => { test('Creates and uses a free-trial Offer', async ({sharedPage}) => { // reset members by deleting all existing @@ -33,9 +34,11 @@ test.describe('Portal', () => { await sharedPage.goto(offerLink); // Wait for the load state to ensure the page has loaded completely - const portalTriggerButton = await sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + let portalTriggerButton = sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + await expect(portalTriggerButton).toBeVisible(); + // Wait for the iframe to be attached to the DOM - await sharedPage.waitForSelector('[data-testid="portal-popup-frame"]', {state: 'attached'}); + await expect(sharedPage.locator('[data-testid="portal-popup-frame"]')).toBeAttached({timeout: 1000}); // Use the frameLocator to interact with elements inside the frame const portalFrameLocator = await sharedPage.frameLocator('[data-testid="portal-popup-frame"]'); @@ -113,9 +116,12 @@ test.describe('Portal', () => { // Wait for the load state to ensure the page has loaded completely await sharedPage.waitForLoadState('load'); - const portalTriggerButton = await sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + // Wait for the load state to ensure the page has loaded completely + let portalTriggerButton = sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + await expect(portalTriggerButton).toBeVisible(); + // Wait for the iframe to be attached to the DOM - await sharedPage.waitForSelector('[data-testid="portal-popup-frame"]', {state: 'attached'}); + await expect(sharedPage.locator('[data-testid="portal-popup-frame"]')).toBeAttached({timeout: 1000}); // Use the frameLocator to interact with elements inside the frame const portalFrameLocator = await sharedPage.frameLocator('[data-testid="portal-popup-frame"]'); @@ -184,9 +190,12 @@ test.describe('Portal', () => { // Wait for the load state to ensure the page has loaded completely await sharedPage.waitForLoadState('load'); - const portalTriggerButton = await sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + // Wait for the load state to ensure the page has loaded completely + let portalTriggerButton = sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + await expect(portalTriggerButton).toBeVisible(); + // Wait for the iframe to be attached to the DOM - await sharedPage.waitForSelector('[data-testid="portal-popup-frame"]', {state: 'attached'}); + await expect(sharedPage.locator('[data-testid="portal-popup-frame"]')).toBeAttached({timeout: 1000}); // Use the frameLocator to interact with elements inside the frame const portalFrameLocator = await sharedPage.frameLocator('[data-testid="portal-popup-frame"]'); @@ -256,9 +265,12 @@ test.describe('Portal', () => { // Wait for the load state to ensure the page has loaded completely await sharedPage.waitForLoadState('load'); - const portalTriggerButton = await sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + // Wait for the load state to ensure the page has loaded completely + let portalTriggerButton = sharedPage.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]'); + await expect(portalTriggerButton).toBeVisible(); + // Wait for the iframe to be attached to the DOM - await sharedPage.waitForSelector('[data-testid="portal-popup-frame"]', {state: 'attached'}); + await expect(sharedPage.locator('[data-testid="portal-popup-frame"]')).toBeAttached({timeout: 1000}); // Use the frameLocator to interact with elements inside the frame const portalFrameLocator = await sharedPage.frameLocator('[data-testid="portal-popup-frame"]'); diff --git a/ghost/core/test/e2e-browser/utils/e2e-browser-utils.js b/ghost/core/test/e2e-browser/utils/e2e-browser-utils.js index 63fafd6b9e..db82327cd0 100644 --- a/ghost/core/test/e2e-browser/utils/e2e-browser-utils.js +++ b/ghost/core/test/e2e-browser/utils/e2e-browser-utils.js @@ -250,64 +250,72 @@ const createTier = async (page, {name, monthlyPrice, yearlyPrice, trialDays}, en */ const createOffer = async (page, {name, tierName, offerType, amount, discountType = null, discountDuration = 3}) => { - await page.goto('/ghost'); - await page.locator('[data-test-nav="settings"]').click(); + let offerName; + let offerLink; + await test.step('Create an offer', async () => { + await page.goto('/ghost'); + await page.locator('[data-test-nav="settings"]').click(); - // Keep offer names unique & <= 40 characters - let offerName = `${name} (${new ObjectID().toHexString().slice(0, 40 - name.length - 3)})`; - // Tiers request can take time, so waiting until there is no connections before interacting with them - await page.waitForLoadState('networkidle'); + // Keep offer names unique & <= 40 characters + offerName = `${name} (${new ObjectID().toHexString().slice(0, 40 - name.length - 3)})`; + // Tiers request can take time, so waiting until there is no connections before interacting with them + await page.waitForLoadState('networkidle'); - const hasExistingOffers = await page.getByTestId('offers').getByRole('button', {name: 'Manage offers'}).isVisible(); - const isCTA = await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).isVisible(); - // Archive other offers to keep the list tidy - // We only need 1 offer to be active at a time - // Either the list of active offers loads, or the CTA when no offers exist - if (hasExistingOffers && !isCTA) { - await page.getByTestId('offers').getByRole('button', {name: 'Manage offers'}).click(); + const hasExistingOffers = await page.getByTestId('offers').getByRole('button', {name: 'Manage offers'}).isVisible(); + const isCTA = await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).isVisible(); + // Archive other offers to keep the list tidy + // We only need 1 offer to be active at a time + // Either the list of active offers loads, or the CTA when no offers exist + if (hasExistingOffers && !isCTA) { + await page.getByTestId('offers').getByRole('button', {name: 'Manage offers'}).click(); - // Selector for the elements with data-testid 'offer-item' - // const offerItemsSelector = '[data-testid="offer-item"]'; - await page.getByTestId('offer-item').nth(0).click(); - await page.getByRole('button', {name: 'Archive offer'}).click(); + // Selector for the elements with data-testid 'offer-item' + // const offerItemsSelector = '[data-testid="offer-item"]'; + await page.getByTestId('offer-item').nth(0).click(); + await page.getByRole('button', {name: 'Archive offer'}).click(); - const confirmModal = await page.getByTestId('confirmation-modal'); - await confirmModal.getByRole('button', {name: 'Archive'}).click(); - } - - if (isCTA) { - await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).click(); - } else { - await page.getByText('New offer').click(); - } - - // const newOfferButton = await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}) || await page.getByTestId('offers').getByRole('button', {name: 'New offer'}); - // await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).click(); - // await newOfferButton.click(); - await page.getByLabel('Offer name').fill(offerName); - - if (offerType === 'freeTrial') { - // await page.getByRole('button', {name: 'Free trial Give free access for a limited time.'}).click(); - await page.getByText('Give free access for a limited time').click(); - await page.getByLabel('Trial duration').fill(`${amount}`); - } else if (offerType === 'discount') { - await page.getByLabel('Amount off').fill(`${amount}`); - if (discountType === 'multiple-months') { - await chooseOptionInSelect(page.getByTestId('duration-select-offers'), `Multiple-months`); - await page.getByLabel('Duration in months').fill(discountDuration.toString()); - // await page.locator('[data-test-select="offer-duration"]').selectOption('repeating'); - // await page.locator('input#duration-months').fill(discountDuration.toString()); + const confirmModal = await page.getByTestId('confirmation-modal'); + await confirmModal.getByRole('button', {name: 'Archive'}).click(); } - if (discountType === 'forever') { - await chooseOptionInSelect(page.getByTestId('duration-select-offers'), `Forever`); + if (isCTA) { + await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).click(); + } else { + await page.getByText('New offer').click(); } - } - await chooseOptionInSelect(page.getByTestId('tier-cadence-select-offers'), `${tierName} - Monthly`); - await page.getByRole('button', {name: 'Publish'}).click(); - await page.waitForLoadState('networkidle'); - const offerLink = await page.locator('input[name="offer-url"]').inputValue(); + // const newOfferButton = await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}) || await page.getByTestId('offers').getByRole('button', {name: 'New offer'}); + // await page.getByTestId('offers').getByRole('button', {name: 'Add offer'}).click(); + // await newOfferButton.click(); + await page.getByLabel('Offer name').fill(offerName); + + if (offerType === 'freeTrial') { + // await page.getByRole('button', {name: 'Free trial Give free access for a limited time.'}).click(); + await page.getByText('Give free access for a limited time').click(); + await page.getByLabel('Trial duration').fill(`${amount}`); + } else if (offerType === 'discount') { + await page.getByLabel('Amount off').fill(`${amount}`); + if (discountType === 'multiple-months') { + await chooseOptionInSelect(page.getByTestId('duration-select-offers'), `Multiple-months`); + await page.getByLabel('Duration in months').fill(discountDuration.toString()); + // await page.locator('[data-test-select="offer-duration"]').selectOption('repeating'); + // await page.locator('input#duration-months').fill(discountDuration.toString()); + } + + if (discountType === 'forever') { + await chooseOptionInSelect(page.getByTestId('duration-select-offers'), `Forever`); + } + } + + await chooseOptionInSelect(page.getByTestId('tier-cadence-select-offers'), `${tierName} - Monthly`); + await page.getByRole('button', {name: 'Publish'}).click(); + await page.waitForLoadState('networkidle'); + + const offerLinkInput = await page.locator('input[name="offer-url"]'); + // sometimes offer link is not generated, and if so the rest of the test will fail + await expect(offerLinkInput).not.toBeEmpty(); + offerLink = await offerLinkInput.inputValue(); + }); return {offerName, offerLink}; };