Parallelise browser tests

refs: https://github.com/TryGhost/DevOps/issues/78

Re-introduce parallel browser tests

These were adding in a previous PR, but the difference between local
running tests and using CI introduced failures.

Added additional fixes to ensure the Stripe API key is used in the CLI when running in CI.

.github/workflows/browser-tests.yml

@@ -3,24 +3,7 @@ on:
   push:
     branches:
       - 'main'
-  workflow_dispatch:
-    inputs:
-      environment:
-        description: 'Environment to run tests against'
-        type: environment
-        required: true
-      site_url:
-        description: 'Site URL (override)'
-        required: false
-        type: string
-      owner_email:
-        description: 'Owner email (override)'
-        required: false
-        type: string
-      owner_password:
-        description: 'Owner password (override)'
-        required: false
-        type: string
+  workflow_dispatch: {}
 concurrency:
   group: ${{ github.workflow }}
 jobs:
@@ -28,12 +11,10 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'renovate/'))
-    environment: ${{ github.event.inputs.environment || 'browser-tests-local' }}
-    env:
-      ENVIRONMENT: ${{ github.event.inputs.environment || 'browser-tests-local' }}
     steps:
     - uses: actions/checkout@v3
       with:
+        ref: ${{ github.event.inputs.ref || github.ref }}
         fetch-depth: 0
         submodules: true
     - uses: actions/setup-node@v3
@@ -76,26 +57,15 @@ jobs:
       run: npx playwright install-deps
 
     - name: Build Admin
-      if: env.ENVIRONMENT == 'browser-tests-local'
       run: yarn nx run ghost-admin:build:dev
 
-    - name: Run Playwright tests on a remote site
-      if: env.ENVIRONMENT == 'browser-tests-staging'
-      working-directory: ghost/core
-      run: yarn test:browser
-      env:
-        TEST_URL: ${{ github.event.inputs.site_url || secrets.TEST_URL }}
-        TEST_OWNER_EMAIL: ${{ github.event.inputs.owner_email || secrets.TEST_OWNER_EMAIL }}
-        TEST_OWNER_PASSWORD: ${{ github.event.inputs.owner_password || secrets.TEST_OWNER_PASSWORD }}
-
     - name: Run Playwright tests locally
-      if: env.ENVIRONMENT == 'browser-tests-local'
       working-directory: ghost/core
       run: yarn test:browser
       env:
+        CI: true
         STRIPE_PUBLISHABLE_KEY: ${{ secrets.STRIPE_PUBLISHABLE_KEY }}
         STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
-        STRIPE_ACCOUNT_ID: ${{ secrets.STRIPE_ACCOUNT_ID }}
 
     - uses: tryghost/actions/actions/slack-build@main
       if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -108,5 +78,5 @@ jobs:
       if: always()
       with:
         name: browser-tests-playwright-report
-        path: ghost/core/playwright-report
+        path: ${{ github.workspace }}/ghost/core/playwright-report
         retention-days: 30

ghost/core/core/cli/record-test.js

@@ -1,30 +0,0 @@
-const {chromium} = require('@playwright/test');
-const Command = require('./command');
-const playwrightConfig = require('../../playwright.config');
-const {globalSetup} = require('../../test/e2e-browser/utils');
-
-module.exports = class RecordTest extends Command {
-    setup() {
-        this.help('Use PlayWright to record a browser-based test');
-    }
-
-    permittedEnvironments() {
-        return ['testing-browser'];
-    }
-
-    async handle() {
-        await globalSetup({
-            projects: [playwrightConfig]
-        });
-
-        const browser = await chromium.launch({headless: false});
-
-        const context = await browser.newContext(playwrightConfig.use);
-
-        // Pause the page, and start recording manually.
-        const page = await context.newPage();
-        await page.goto('/ghost');
-
-        await page.pause();
-    }
-};

ghost/core/package.json

@@ -33,12 +33,10 @@
     "test:integration": "yarn test:base './test/integration' --timeout=10000",
     "test:e2e": "yarn test:base ./test/e2e-* --timeout=15000",
     "test:regression": "yarn test:base './test/regression' --timeout=60000",
-    "test:browser": "yarn test:browser:admin && yarn test:browser:portal",
+    "test:browser": "NODE_ENV=testing-browser playwright test",
     "test:browser:admin": "NODE_ENV=testing-browser  playwright test test/e2e-browser --project=admin",
     "test:browser:portal": "NODE_ENV=testing-browser playwright test test/e2e-browser --project=portal",
-    "test:browser:single": "NODE_ENV=testing-browser playwright test",
     "test:browser:setup": "npx playwright install",
-    "test:browser:record": "NODE_ENV=testing-browser yarn start record-test",
     "test:ci:e2e": "c8 -c ./.c8rc.e2e.json -o coverage-e2e yarn test:e2e -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",
     "test:ci:regression": "yarn test:regression -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",
     "test:ci:integration": "c8 -c ./.c8rc.e2e.json -o coverage-integration --lines 57 --functions 47 --branches 77 --statements 57 yarn test:integration -b --retries=2 --reporter=./test/utils/mocha-retry-reporter.js",

ghost/core/playwright.config.js

@@ -5,15 +5,15 @@ const config = {
     expect: {
         timeout: 10000
     },
-    workers: 1,
-    reporter: [['list', {printSteps: true}]],
+    retries: process.env.CI ? 2 : 0,
+    workers: process.env.CI ? '100%' : (process.env.PLAYWRIGHT_SLOWMO ? 1 : undefined),
+    reporter: process.env.CI ? [['list', {printSteps: true}], ['html']] : [['list', {printSteps: true}]],
     use: {
         // Use a single browser since we can't simultaneously test multiple browsers
         browserName: 'chromium',
         headless: !process.env.PLAYWRIGHT_DEBUG,
-        baseURL: process.env.TEST_URL ?? 'http://127.0.0.1:2369',
-        // TODO: Where to put this
-        storageState: 'playwright-state.json'
+        // Port doesn't matter, overriden by baseURL fixture for each worker
+        baseURL: 'http://127.0.0.1:2368'
     },
     // separated tests to projects for better logging to console
     // portal tests are much more stable when running in the separate DB from admin tests
@@ -24,11 +24,10 @@ const config = {
         },
         {
             name: 'portal',
-            testDir: 'test/e2e-browser/portal'
+            testDir: 'test/e2e-browser/portal',
+            fullyParallel: true
         }
-    ],
-    globalSetup: './test/e2e-browser/utils/global-setup',
-    globalTeardown: './test/e2e-browser/utils/global-teardown'
+    ]
 };
 
 module.exports = config;

ghost/core/test/e2e-browser/admin/announcement-bar-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Announcement Bar Settings', () => {
     test('Bar hidden by default', async ({page}) => {

ghost/core/test/e2e-browser/admin/members.spec.js

@@ -1,10 +1,11 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, deleteAllMembers} = require('../utils/e2e-browser-utils');
 const fs = require('fs');
 
 test.describe('Admin', () => {
     test.describe('Members', () => {
-        test.describe.configure({retries: 1});
+        test.describe.configure({retries: 1, mode: 'serial'});
         test('A member can be created', async ({page}) => {
             await page.goto('/ghost');
             await page.locator('.gh-nav a[href="#/members/"]').click();
@@ -279,7 +280,7 @@ test.describe('Admin', () => {
         });
 
         test('A member can be granted a comp in admin', async ({page}) => {
-            page.goto('/ghost');
+            await page.goto('/ghost');
             await deleteAllMembers(page);
 
             // create a new member with a comped plan

ghost/core/test/e2e-browser/admin/membership-settings.spec.js

@@ -1,5 +1,6 @@
-const {expect, test} = require('@playwright/test');
-const {disconnectStripe, setupStripe, generateStripeIntegrationToken} = require('../utils');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
+const {disconnectStripe, setupStripe, generateStripeIntegrationToken, getStripeAccountId} = require('../utils');
 
 test.describe('Membership Settings', () => {
     test.describe('Portal settings', () => {
@@ -18,7 +19,8 @@ test.describe('Membership Settings', () => {
             await expect(modal.locator('label').filter({hasText: 'Free'}).first()).toBeVisible();
 
             // Reconnect Stripe for other tests
-            const stripeToken = await generateStripeIntegrationToken();
+            const stripeAccountId = await getStripeAccountId();
+            const stripeToken = await generateStripeIntegrationToken(stripeAccountId);
             await page.goto('/ghost');
             await setupStripe(page, stripeToken);
         });

ghost/core/test/e2e-browser/admin/portal-settings.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Portal Settings', () => {
     test.describe('Links', () => {

ghost/core/test/e2e-browser/admin/posts.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Admin', () => {
     test.describe('Posts', () => {

ghost/core/test/e2e-browser/admin/private-site.spec.js

@@ -1,4 +1,5 @@
-const {test, expect} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Site Settings', () => {
     test.describe('Privacy setting', () => {

ghost/core/test/e2e-browser/admin/publishing.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {DateTime} = require('luxon');
 const {slugify} = require('@tryghost/string');
 const {createTier, createMember, createPostDraft, impersonateMember} = require('../utils');
@@ -191,7 +192,7 @@ test.describe('Publishing', () => {
             };
 
             // Create a member to send and email to
-            await createMember(page, {email: 'example@example.com', name: 'Publishing member'});
+            await createMember(page, {email: 'test+recipient1@example.com', name: 'Publishing member'});
 
             await page.goto('/ghost');
             await createPostDraft(page, postData);
@@ -220,12 +221,13 @@ test.describe('Publishing', () => {
 
         // Post should be available on web and sent as a newsletter
         test('Email only', async ({page}) => {
-            // Note: this currently depends on 'Publish and Email' to create a member!
             const postData = {
                 title: 'Email only post',
                 body: 'This is my post body.'
             };
 
+            await createMember(page, {email: 'test+recipient2@example.com', name: 'Publishing member'});
+
             await page.goto('/ghost');
             await createPostDraft(page, postData);
             await publishPost(page, {type: 'send'});
@@ -327,13 +329,14 @@ test.describe('Publishing', () => {
     test.describe('Schedule post', () => {
         // Post should be published to web and sent as a newsletter at the scheduled time
         test('Publish and Email', async ({page}) => {
-            // Note: this currently depends on the first 'Publish and Email' to create a member!
             const postData = {
                 // This title should be unique
                 title: 'Scheduled post publish+email test',
                 body: 'This is my scheduled post body.'
             };
 
+            await createMember(page, {email: 'test+recipient3@example.com', name: 'Publishing member'});
+
             await page.goto('/ghost');
             await createPostDraft(page, postData);
 
@@ -395,6 +398,8 @@ test.describe('Publishing', () => {
                 body: 'This is my scheduled post body.'
             };
 
+            await createMember(page, {email: 'test+recipient4@example.com', name: 'Publishing member'});
+
             await page.goto('/ghost');
             await createPostDraft(page, postData);
 

ghost/core/test/e2e-browser/admin/setup.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 
 test.describe('Admin', () => {
     test.describe('Setup', () => {

ghost/core/test/e2e-browser/admin/site-settings.spec.js

@@ -1,5 +1,6 @@
-const {expect, test} = require('@playwright/test');
-const {createPostDraft, createTier, disconnectStripe, generateStripeIntegrationToken, setupStripe} = require('../utils');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
+const {createPostDraft, createTier, disconnectStripe, generateStripeIntegrationToken, setupStripe, getStripeAccountId} = require('../utils');
 
 const changeSubscriptionAccess = async (page, access) => {
     await page.locator('[data-test-nav="settings"]').click();
@@ -125,7 +126,8 @@ test.describe('Site Settings', () => {
             await changeSubscriptionAccess(page, 'all');
 
             await page.goto('/ghost');
-            const stripeToken = await generateStripeIntegrationToken();
+            const stripeAccountId = await getStripeAccountId();
+            const stripeToken = await generateStripeIntegrationToken(stripeAccountId);
             await setupStripe(page, stripeToken);
         });
     });

ghost/core/test/e2e-browser/admin/tiers.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createTier, createOffer, getUniqueName, getSlug, goToMembershipPage, openTierModal} = require('../utils');
 
 test.describe('Admin', () => {

ghost/core/test/e2e-browser/fixtures/ghost-test.js

@@ -0,0 +1,141 @@
+// express-test.js
+const base = require('@playwright/test');
+const {promisify} = require('util');
+const {spawn, exec} = require('child_process');
+const {setupGhost, setupMailgun, enableLabs, setupStripe, getStripeAccountId, generateStripeIntegrationToken} = require('../utils/e2e-browser-utils');
+const {allowStripe, mockMail} = require('../../utils/e2e-framework-mock-manager');
+const MailgunClient = require('@tryghost/mailgun-client');
+const sinon = require('sinon');
+const ObjectID = require('bson-objectid').default;
+const Stripe = require('stripe').Stripe;
+const configUtils = require('../../utils/configUtils');
+
+const startWebhookServer = (port) => {
+    const command = `stripe listen --forward-connect-to http://127.0.0.1:${port}/members/webhooks/stripe/ ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
+    const webhookServer = spawn(command.split(' ')[0], command.split(' ').slice(1));
+
+    // Adding event listeners here seems to prevent heisenbug where webhooks aren't received
+    webhookServer.stdout.on('data', () => {});
+    webhookServer.stderr.on('data', () => {});
+
+    return webhookServer;
+};
+
+const getWebhookSecret = async () => {
+    const command = `stripe listen --print-secret ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
+    const webhookSecret = (await promisify(exec)(command)).stdout;
+    return webhookSecret.toString().trim();
+};
+
+// Global promises for webhook secret / Stripe integration token
+const webhookSecretPromise = getWebhookSecret();
+
+module.exports = base.test.extend({
+    baseURL: async ({port, baseURL}, use) => {
+        // Replace the port in baseURL with the one we got from the port fixture
+        const url = new URL(baseURL);
+        url.port = port.toString();
+        await use(url.toString());
+    },
+
+    storageState: async ({ghost}, use) => {
+        await use(ghost.state);
+    },
+
+    // eslint-disable-next-line no-empty-pattern
+    port: [async ({}, use, workerInfo) => {
+        await use(2369 + workerInfo.parallelIndex);
+    }, {scope: 'worker'}],
+
+    ghost: [async ({browser, port}, use, workerInfo) => {
+        // Do not initialise database before this block
+        const currentDate = new Date();
+        const formattedDate = `${currentDate.getFullYear()}-${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}-${String(currentDate.getHours()).padStart(2, '0')}-${String(currentDate.getMinutes()).padStart(2, '0')}-${String(currentDate.getSeconds()).padStart(2, '0')}`;
+        process.env.database__connection__filename = `/tmp/ghost-playwright.${workerInfo.workerIndex}.${formattedDate}.db`;
+        configUtils.set('database:connection:filename', process.env.database__connection__filename);
+        configUtils.set('server:port', port);
+        configUtils.set('url', `http://127.0.0.1:${port}`);
+
+        const stripeAccountId = await getStripeAccountId();
+        const stripeIntegrationToken = await generateStripeIntegrationToken(stripeAccountId);
+
+        const WebhookManager = require('../../../../stripe/lib/WebhookManager');
+        const originalParseWebhook = WebhookManager.prototype.parseWebhook;
+        const sandbox = sinon.createSandbox();
+        sandbox.stub(WebhookManager.prototype, 'parseWebhook').callsFake(function (body, signature) {
+            const parsedBody = JSON.parse(body);
+            if (!('account' in parsedBody)) {
+                throw new Error('Webhook without account');
+            } else if (parsedBody.account !== stripeAccountId) {
+                throw new Error('Webhook for wrong account');
+            } else {
+                return originalParseWebhook.call(this, body, signature);
+            }
+        });
+
+        const StripeAPI = require('../../../../stripe/lib/StripeAPI');
+        const originalStripeConfigure = StripeAPI.prototype.configure;
+        sandbox.stub(StripeAPI.prototype, 'configure').callsFake(function (stripeConfig) {
+            originalStripeConfigure.call(this, stripeConfig);
+            if (stripeConfig) {
+                this._stripe = new Stripe(stripeConfig.secretKey, {
+                    apiVersion: '2020-08-27',
+                    stripeAccount: stripeAccountId
+                });
+            }
+        });
+
+        const stripeServer = startWebhookServer(port);
+
+        process.env.WEBHOOK_SECRET = await webhookSecretPromise;
+
+        sandbox.stub(MailgunClient.prototype, 'getInstance').returns({
+            // @ts-ignore
+            messages: {
+                create: async function () {
+                    return {
+                        id: `mailgun-mock-id-${ObjectID().toHexString()}`
+                    };
+                }
+            }
+        });
+
+        mockMail();
+
+        const {startGhost} = require('../../utils/e2e-framework');
+        const server = await startGhost({
+            frontend: true,
+            server: true,
+            backend: true
+        });
+
+        // StartGhost automatically disables network, so we need to re-enable it for Stripe
+        allowStripe();
+
+        const page = await browser.newPage({
+            baseURL: `http://127.0.0.1:${port}/`,
+            storageState: undefined
+        });
+
+        await setupGhost(page);
+        await setupStripe(page, stripeIntegrationToken);
+        await setupMailgun(page);
+        await enableLabs(page);
+        const state = await page.context().storageState();
+
+        await page.close();
+
+        // Use the server in the tests.
+        try {
+            await use({
+                server,
+                state
+            });
+        } finally {
+            const {stopGhost} = require('../../utils/e2e-utils');
+            await stopGhost();
+            stripeServer.kill();
+            sandbox.restore();
+        }
+    }, {scope: 'worker', auto: true}]
+});

ghost/core/test/e2e-browser/portal/donations.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, impersonateMember, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/portal/member-actions.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {createMember, impersonateMember} = require('../utils');
 
 /**
@@ -22,7 +23,9 @@ const addNewsletter = async (page) => {
 
 test.describe('Portal', () => {
     test.describe('Member actions', () => {
-        test.describe.configure({retries: 1});
+        // Use serial mode as the order of tests matters, we create newsletters during the tests
+        // TODO: Use a `before` block to create all the requisite newsletters before the tests run
+        test.describe.configure({retries: 1, mode: 'serial'});
 
         test('can log out', async ({page}) => {
             // create a new free member

ghost/core/test/e2e-browser/portal/offers.spec.js

@@ -1,15 +1,16 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {deleteAllMembers, createTier, createOffer, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {
     test.describe('Offers', () => {
         test('Creates and uses a free-trial Offer', async ({page}) => {
             // reset members by deleting all existing
-            page.goto('/ghost');
+            await page.goto('/ghost');
             await deleteAllMembers(page);
 
             // add a new tier for offers
-            const tierName = 'Portal Tier';
+            const tierName = 'Trial Tier';
             await createTier(page, {
                 name: tierName,
                 monthlyPrice: 6,
@@ -43,7 +44,7 @@ test.describe('Portal', () => {
 
             // fill member details and click start trial
             await portalFrame.locator('[data-test-input="input-name"]').fill('Testy McTesterson');
-            await portalFrame.locator('[data-test-input="input-email"]').fill('testy@example.com');
+            await portalFrame.locator('[data-test-input="input-email"]').fill('testy+trial@example.com');
             await portalFrame.getByRole('button', {name: 'Start 14-day free trial'}).click();
 
             // handle newsletter selection page if it opens and click continue
@@ -66,7 +67,7 @@ test.describe('Portal', () => {
             await page.locator('.gh-nav a[href="#/members/"]').click();
 
             // 1 member, should be Testy, on Portal Tier
-            await expect(page.getByRole('link', {name: 'Testy McTesterson testy@example.com'}), 'Should have 1 paid member').toBeVisible();
+            await expect(page.getByRole('link', {name: 'Testy McTesterson testy+trial@example.com'}), 'Should have 1 paid member').toBeVisible();
             await expect(page.getByRole('link', {name: tierName}), `Paid member should be on ${tierName}`).toBeVisible();
 
             // Ensure the offer redemption count was bumped
@@ -77,11 +78,11 @@ test.describe('Portal', () => {
 
         test('Creates and uses a one-time discount Offer', async ({page}) => {
             // reset members by deleting all existing
-            page.goto('/ghost');
+            await page.goto('/ghost');
             await deleteAllMembers(page);
 
             // add new tier
-            const tierName = 'Portal Tier';
+            const tierName = 'One-off Tier';
             await createTier(page, {
                 name: tierName,
                 monthlyPrice: 6,
@@ -114,7 +115,7 @@ test.describe('Portal', () => {
 
             // fill member details and continue
             await portalFrame.locator('#input-name').fill('Testy McTesterson');
-            await portalFrame.locator('#input-email').fill('testy@example.com');
+            await portalFrame.locator('#input-email').fill('testy+oneoff@example.com');
             await portalFrame.getByRole('button', {name: 'Continue'}).click();
 
             // check if newsletter selection screen is shown and continue
@@ -136,17 +137,17 @@ test.describe('Portal', () => {
             await page.locator('.gh-nav a[href="#/members/"]').click();
 
             // 1 member, should be Testy, on Portal Tier
-            await expect(page.getByRole('link', {name: 'Testy McTesterson testy@example.com'}), 'Should have 1 paid member').toBeVisible();
+            await expect(page.getByRole('link', {name: 'Testy McTesterson testy+oneoff@example.com'}), 'Should have 1 paid member').toBeVisible();
             await expect(page.getByRole('link', {name: tierName}), `Paid member should be on ${tierName}`).toBeVisible();
         });
 
         test('Creates and uses a multiple-months discount Offer', async ({page}) => {
             // reset members by deleting all existing
-            page.goto('/ghost');
+            await page.goto('/ghost');
             await deleteAllMembers(page);
 
             // add new tier
-            const tierName = 'Portal Tier';
+            const tierName = 'Multiple-month Tier';
             await createTier(page, {
                 name: tierName,
                 monthlyPrice: 6,
@@ -183,7 +184,7 @@ test.describe('Portal', () => {
 
             // fill member details and continue
             await portalFrame.locator('#input-name').fill('Testy McTesterson');
-            await portalFrame.locator('#input-email').fill('testy@example.com');
+            await portalFrame.locator('#input-email').fill('testy+multi@example.com');
             await portalFrame.getByRole('button', {name: 'Continue'}).click();
 
             // check newsletter selection if shown and continue
@@ -204,17 +205,17 @@ test.describe('Portal', () => {
             await page.locator('.gh-nav a[href="#/members/"]').click();
 
             // 1 member, should be Testy, on Portal Tier
-            await expect(page.getByRole('link', {name: 'Testy McTesterson testy@example.com'}), 'Should have 1 paid member').toBeVisible();
+            await expect(page.getByRole('link', {name: 'Testy McTesterson testy+multi@example.com'}), 'Should have 1 paid member').toBeVisible();
             await expect(page.getByRole('link', {name: tierName}), `Paid member should be on ${tierName}`).toBeVisible();
         });
 
         test('Creates and uses a forever discount Offer', async ({page}) => {
             // reset members by deleting all existing
-            page.goto('/ghost');
+            await page.goto('/ghost');
             await deleteAllMembers(page);
 
             // add tier
-            const tierName = 'Portal Tier';
+            const tierName = 'Forever Tier';
             await createTier(page, {
                 name: tierName,
                 monthlyPrice: 6,
@@ -250,7 +251,7 @@ test.describe('Portal', () => {
 
             // fill member details and continue
             await portalFrame.locator('#input-name').fill('Testy McTesterson');
-            await portalFrame.locator('#input-email').fill('testy@example.com');
+            await portalFrame.locator('#input-email').fill('testy+forever@example.com');
             await portalFrame.getByRole('button', {name: 'Continue'}).click();
 
             // check if newsletter selection page is shown and continue
@@ -269,15 +270,15 @@ test.describe('Portal', () => {
             await page.locator('.gh-nav a[href="#/members/"]').click();
 
             // 1 member, should be Testy, on Portal Tier
-            await expect(page.getByRole('link', {name: 'Testy McTesterson testy@example.com'}), 'Should have 1 paid member').toBeVisible();
+            await expect(page.getByRole('link', {name: 'Testy McTesterson testy+forever@example.com'}), 'Should have 1 paid member').toBeVisible();
             await expect(page.getByRole('link', {name: tierName}), `Paid member should be on ${tierName}`).toBeVisible();
         });
 
         test('Archiving an offer', async ({page}) => {
-            page.goto('/ghost');
+            await page.goto('/ghost');
 
             // Create a new tier to attach offer to
-            const tierName = 'Portal Tier';
+            const tierName = 'Archive Test Tier';
             await createTier(page, {
                 name: tierName,
                 monthlyPrice: 6,

ghost/core/test/e2e-browser/portal/tiers.spec.js

@@ -1,4 +1,5 @@
-const {expect, test} = require('@playwright/test');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
 const {deleteAllMembers, completeStripeSubscription} = require('../utils');
 
 test.describe('Portal', () => {

ghost/core/test/e2e-browser/portal/upgrade.spec.js

@@ -1,13 +1,23 @@
-const {expect, test} = require('@playwright/test');
-const {completeStripeSubscription, createMember, impersonateMember} = require('../utils');
+const {expect} = require('@playwright/test');
+const test = require('../fixtures/ghost-test');
+const {completeStripeSubscription, createMember, createTier, impersonateMember} = require('../utils');
+
+const tierName = 'Upgrade Tests';
 
 test.describe('Portal', () => {
+    test.describe('Upgrades', () => {
+        // Tier created in first test used in subsequent tests
+        test.describe.configure({mode: 'serial'});
+
         test.describe('Upgrade: Comped Member', () => {
             test('allows comped member to upgrade to paid tier', async ({page}) => {
-            const tierName = 'The Local Test';
-
                 // create a new member
                 await page.goto('/ghost');
+                await createTier(page, {
+                    name: tierName,
+                    monthlyPrice: 5,
+                    yearlyPrice: 50
+                });
                 await createMember(page, {
                     name: 'Testy McTest',
                     email: 'testy+upgradecompedportal@example.com',
@@ -28,7 +38,6 @@ test.describe('Portal', () => {
 
                 const portalTriggerButton = page.frameLocator('[data-testid="portal-trigger-frame"]').locator('[data-testid="portal-trigger-button"]');
                 const portalFrame = page.frameLocator('[data-testid="portal-popup-frame"]');
-            //await page.pause();
 
                 // open portal, go to plans and click continue to select the first plan(yearly)
                 await portalTriggerButton.click();
@@ -53,8 +62,10 @@ test.describe('Portal', () => {
         });
 
         test.describe('Upgrade: Single Tier', () => {
+            // Because memberUrl is set during first test, we need to run these tests in series
+            test.describe.configure({mode: 'serial'});
+
             let memberUrl;
-        const tierName = 'The Local Test';
             const member = {
                 name: 'Testy McTest',
                 email: 'testy+upgradeportal@example.com',
@@ -144,9 +155,10 @@ test.describe('Portal', () => {
                 await expect(portalFrame.getByText('$50.00/year')).toBeVisible();
             });
         });
+    });
 });
 
-async function choseTierByName(portalFrame, tierName) {
-    const portalTierCard = await portalFrame.locator('[data-test-tier="paid"]').filter({hasText: tierName}).first();
+async function choseTierByName(portalFrame, tier) {
+    const portalTierCard = await portalFrame.locator('[data-test-tier="paid"]').filter({hasText: tier}).first();
     await portalTierCard.locator('[data-test-button="select-tier"]').click();
 }

ghost/core/test/e2e-browser/utils/e2e-browser-utils.js

@@ -1,8 +1,7 @@
 const DataGenerator = require('../../utils/fixtures/data-generator');
 const {expect, test} = require('@playwright/test');
 const ObjectID = require('bson-objectid').default;
-const {promisify} = require('util');
-const {exec} = require('child_process');
+const Stripe = require('stripe').Stripe;
 
 /**
  * Tier
@@ -43,11 +42,6 @@ const setupGhost = async (page) => {
     // Add owner user data from usual fixture
     const ownerUser = DataGenerator.Content.users.find(user => user.id === '1');
 
-    if (process.env.CI && process.env.TEST_URL) {
-        ownerUser.email = process.env.TEST_OWNER_EMAIL;
-        ownerUser.password = process.env.TEST_OWNER_PASSWORD;
-    }
-
     if (action === actions.signin) {
         // Fill email + password
         await page.locator('#identification').fill(ownerUser.email);
@@ -469,29 +463,48 @@ const openTierModal = async (page, {slug}) => {
     });
 };
 
-const generateStripeIntegrationToken = async () => {
-    const inquirer = require('inquirer');
-    const {knex} = require('../../../core/server/data/db');
+const getStripeAccountId = async () => {
+    if (!('STRIPE_PUBLISHABLE_KEY' in process.env) || !('STRIPE_SECRET_KEY' in process.env)) {
+        throw new Error('Missing STRIPE_PUBLISHABLE_KEY or STRIPE_SECRET_KEY environment variables');
+    }
 
-    const stripeDatabaseKeys = {
-        publishableKey: 'stripe_connect_publishable_key',
-        secretKey: 'stripe_connect_secret_key',
-        liveMode: 'stripe_connect_livemode'
-    };
-    const publishableKey = process.env.STRIPE_PUBLISHABLE_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.publishableKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe publishable key (starts "pk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
-    const secretKey = process.env.STRIPE_SECRET_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.secretKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe secret key (starts "sk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
+    const parallelIndex = process.env.TEST_PARALLEL_INDEX;
+    let accountId;
+    const accountEmail = `test${parallelIndex}@example.com`;
 
-    const accountId = process.env.STRIPE_ACCOUNT_ID ?? JSON.parse((await promisify(exec)('stripe get account')).stdout).id;
+    const secretKey = process.env.STRIPE_SECRET_KEY;
+    const stripe = new Stripe(secretKey, {
+        apiVersion: '2020-08-27'
+    });
+    const accounts = await stripe.accounts.list();
+    if (accounts.data.length > 0) {
+        const account = accounts.data.find(acc => acc.email === accountEmail);
+        if (account) {
+            await stripe.accounts.del(account.id);
+        }
+    }
+    if (!accountId) {
+        const account = await stripe.accounts.create({
+            type: 'standard',
+            email: accountEmail,
+            business_type: 'company',
+            company: {
+                name: `Test Company ${parallelIndex}`
+            }
+        });
+        accountId = account.id;
+    }
+
+    return accountId;
+};
+
+const generateStripeIntegrationToken = async (accountId) => {
+    if (!('STRIPE_PUBLISHABLE_KEY' in process.env) || !('STRIPE_SECRET_KEY' in process.env)) {
+        throw new Error('Missing STRIPE_PUBLISHABLE_KEY or STRIPE_SECRET_KEY environment variables');
+    }
+
+    const publishableKey = process.env.STRIPE_PUBLISHABLE_KEY;
+    const secretKey = process.env.STRIPE_SECRET_KEY;
 
     return Buffer.from(JSON.stringify({
         a: secretKey,
@@ -506,6 +519,7 @@ module.exports = {
     setupStripe,
     disconnectStripe,
     enableLabs,
+    getStripeAccountId,
     generateStripeIntegrationToken,
     setupMailgun,
     deleteAllMembers,

ghost/core/test/e2e-browser/utils/global-setup.js

@@ -1,114 +0,0 @@
-const config = require('../../../core/shared/config');
-const {promisify} = require('util');
-const {spawn, exec} = require('child_process');
-const {knex} = require('../../../core/server/data/db');
-const {setupGhost, setupStripe, setupMailgun, enableLabs} = require('./e2e-browser-utils');
-const {chromium} = require('@playwright/test');
-const {startGhost} = require('../../utils/e2e-framework');
-const {stopGhost} = require('../../utils/e2e-utils');
-const MailgunClient = require('@tryghost/mailgun-client');
-const sinon = require('sinon');
-const ObjectID = require('bson-objectid').default;
-const {allowStripe} = require('../../utils/e2e-framework-mock-manager');
-
-const startWebhookServer = () => {
-    const command = `stripe listen --forward-to ${config.getSiteUrl()}members/webhooks/stripe/ ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
-    spawn(command.split(' ')[0], command.split(' ').slice(1));
-};
-
-const getWebhookSecret = async () => {
-    const command = `stripe listen --print-secret ${process.env.CI ? `--api-key ${process.env.STRIPE_SECRET_KEY}` : ''}`.trim();
-    const webhookSecret = (await promisify(exec)(command)).stdout;
-    return webhookSecret.toString().trim();
-};
-
-const generateStripeIntegrationToken = async () => {
-    const inquirer = require('inquirer');
-    const stripeDatabaseKeys = {
-        publishableKey: 'stripe_connect_publishable_key',
-        secretKey: 'stripe_connect_secret_key',
-        liveMode: 'stripe_connect_livemode'
-    };
-    const publishableKey = process.env.STRIPE_PUBLISHABLE_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.publishableKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe publishable key (starts "pk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
-    const secretKey = process.env.STRIPE_SECRET_KEY ?? (await knex('settings').select('value').where('key', stripeDatabaseKeys.secretKey).first())?.value
-        ?? (await inquirer.prompt([{
-            message: 'Stripe secret key (starts "sk_test_")',
-            type: 'password',
-            name: 'value'
-        }])).value;
-
-    const accountId = process.env.STRIPE_ACCOUNT_ID ?? JSON.parse((await promisify(exec)('stripe get account')).stdout).id;
-
-    return Buffer.from(JSON.stringify({
-        a: secretKey,
-        p: publishableKey,
-        l: false,
-        i: accountId
-    })).toString('base64');
-};
-
-const stubMailgun = () => {
-    // We need to stub the Mailgun client before starting Ghost
-    sinon.stub(MailgunClient.prototype, 'getInstance').returns({
-        // @ts-ignore
-        messages: {
-            create: async function () {
-                return {
-                    id: `mailgun-mock-id-${ObjectID().toHexString()}`
-                };
-            }
-        }
-    });
-};
-
-/**
- * Setup the environment
- */
-const setup = async (playwrightConfig) => {
-    const usingRemoteServer = process.env.CI && process.env.TEST_URL;
-
-    let stripeConnectIntegrationToken;
-    if (!usingRemoteServer) {
-        startWebhookServer();
-        stripeConnectIntegrationToken = await generateStripeIntegrationToken();
-
-        process.env.WEBHOOK_SECRET = await getWebhookSecret();
-
-        // Stub out NodeMailer
-        stubMailgun();
-
-        await startGhost({
-            frontend: true,
-            server: true,
-            backend: true
-        });
-
-        // StartGhost automatically disables network, so we need to re-enable it for Stripe
-        allowStripe();
-    }
-
-    const {baseURL, storageState} = playwrightConfig.projects[0].use;
-    const browser = await chromium.launch();
-    const page = await browser.newPage({
-        baseURL
-    });
-    await setupGhost(page);
-    if (!usingRemoteServer) {
-        await setupStripe(page, stripeConnectIntegrationToken);
-        await setupMailgun(page);
-    }
-    await enableLabs(page);
-    await page.context().storageState({path: storageState});
-    await browser.close();
-
-    if (!usingRemoteServer) {
-        await stopGhost();
-    }
-};
-
-module.exports = setup;

ghost/core/test/e2e-browser/utils/global-teardown.js

@@ -1,8 +0,0 @@
-/**
- * Teardown the environment
- */
-const teardown = async () => {
-    // @NOTE: local environment should probably drop the db state here
-};
-
-module.exports = teardown;

ghost/core/test/e2e-browser/utils/index.js

@@ -1,5 +1,4 @@
 module.exports = {
     ...require('./e2e-browser-utils'),
-    ...require('./helpers'),
-    globalSetup: require('./global-setup')
+    ...require('./helpers')
 };