Cleaned up members_stripe_* tables on MySQL (#12103)

refs #12100

For performance reasons we want to add foreign key and unique constraints
to the members_stripe_* tables so we can utilised cascading deletes and 
joins across the tables when querying.

In order to do this we must first ensure that:
- There are no duplicate entries in the `subscription_id` or `customer_id` columns
- There are no orphaned rows in the subscription or customers tables

If the first is not true, the unique constraint will fail, and if the second is not true,
the foreign key constraint will fail.

As we are only adding the indexes to existing MySQL databases at this point, the
cleanup migrations will also only be done for existing MySQL databases too.

The migrations for removing orphaned rows splits the deletion into a `SELECT`
followed by a `WHERE IN` to avoid the database "optimising" the query into a
`JOIN` which ends up taking much longer due to the lack of indexes.

core/server/data/migrations/versions/3.29/01-remove-duplicate-subscriptions.js

@@ -0,0 +1,50 @@
+const logging = require('../../../../../shared/logging');
+
+module.exports = {
+    config: {
+        transaction: true
+    },
+
+    async up({transacting: knex}) {
+        if (knex.client.config.client !== 'mysql') {
+            logging.warn('Skipping cleanup of duplicate subscriptions - database is not MySQL');
+            return;
+        }
+
+        const duplicates = await knex('members_stripe_customers_subscriptions')
+            .select('subscription_id')
+            .count('subscription_id as count')
+            .groupBy('subscription_id')
+            .having('count', '>', 1);
+
+        if (!duplicates.length) {
+            logging.info('No duplicate subscriptions found');
+            return;
+        }
+
+        logging.info(`Found ${duplicates.length} duplicate stripe subscriptions`);
+        for (const duplicate of duplicates) {
+            const subscriptions = await knex('members_stripe_customers_subscriptions')
+                .select()
+                .where('subscription_id', duplicate.subscription_id);
+
+            const orderedSubscriptions = subscriptions.sort((subA, subB) => {
+                return subB.updated_at - subA.updated_at;
+            });
+
+            const [newestSubscription, ...olderSubscriptions] = orderedSubscriptions;
+
+            logging.info(`Keeping newest subscription ${newestSubscription.id} - ${newestSubscription.subscription_id}, last updated at ${newestSubscription.updated_at}`);
+
+            for (const subscriptionToDelete of olderSubscriptions) {
+                logging.info(`Deleting duplicate subscription ${subscriptionToDelete.id} - ${subscriptionToDelete.subscription_id}, last updated at ${subscriptionToDelete.updated_at}`);
+                await knex('members_stripe_customers_subscriptions')
+                    .where({id: subscriptionToDelete.id})
+                    .del();
+            }
+        }
+    },
+
+    // noop for down
+    async down() {}
+};

core/server/data/migrations/versions/3.29/02-remove-duplicate-customers.js

@@ -0,0 +1,50 @@
+const logging = require('../../../../../shared/logging');
+
+module.exports = {
+    config: {
+        transaction: true
+    },
+
+    async up({transacting: knex}) {
+        if (knex.client.config.client !== 'mysql') {
+            logging.warn('Skipping cleanup of duplicate customers - database is not MySQL');
+            return;
+        }
+
+        const duplicates = await knex('members_stripe_customers')
+            .select('customer_id')
+            .count('customer_id as count')
+            .groupBy('customer_id')
+            .having('count', '>', 1);
+
+        if (!duplicates.length) {
+            logging.info('No duplicate customers found');
+            return;
+        }
+
+        logging.info(`Found ${duplicates.length} duplicate stripe customers`);
+        for (const duplicate of duplicates) {
+            const customers = await knex('members_stripe_customers')
+                .select()
+                .where('customer_id', duplicate.customer_id);
+
+            const orderedCustomers = customers.sort((subA, subB) => {
+                return subB.updated_at - subA.updated_at;
+            });
+
+            const [newestCustomer, ...olderCustomers] = orderedCustomers;
+
+            logging.info(`Keeping newest customer ${newestCustomer.id} - ${newestCustomer.customer_id}, last updated at ${newestCustomer.updated_at}`);
+
+            for (const customerToDelete of olderCustomers) {
+                logging.info(`Deleting duplicate customer ${customerToDelete.id} - ${customerToDelete.customer_id}, last updated at ${customerToDelete.updated_at}`);
+                await knex('members_stripe_customers')
+                    .where({id: customerToDelete.id})
+                    .del();
+            }
+        }
+    },
+
+    // noop for down
+    async down() {}
+};

core/server/data/migrations/versions/3.29/03-remove-orphaned-customers.js

@@ -0,0 +1,34 @@
+const logging = require('../../../../../shared/logging');
+
+module.exports = {
+    config: {
+        transaction: true
+    },
+
+    async up({transacting: knex}) {
+        if (knex.client.config.client !== 'mysql') {
+            logging.warn('Skipping cleanup of orphaned customers - database is not MySQL');
+            return;
+        }
+
+        const orphanedCustomers = await knex('members_stripe_customers')
+            .select('id')
+            .whereNotIn(
+                'member_id',
+                knex('members')
+                    .select('id')
+            );
+
+        if (!orphanedCustomers || !orphanedCustomers.length) {
+            logging.info('No orphaned customer records found');
+            return;
+        }
+
+        logging.info(`Deleting ${orphanedCustomers.length} orphaned customers`);
+        await knex('members_stripe_customers')
+            .whereIn('id', orphanedCustomers.map(customer => customer.id))
+            .del();
+    },
+
+    async down() {}
+};

core/server/data/migrations/versions/3.29/04-remove-orphaned-subscriptions.js

@@ -0,0 +1,34 @@
+const logging = require('../../../../../shared/logging');
+
+module.exports = {
+    config: {
+        transaction: true
+    },
+
+    async up({transacting: knex}) {
+        if (knex.client.config.client !== 'mysql') {
+            logging.warn('Skipping cleanup of orphaned subscriptions - database is not MySQL');
+            return;
+        }
+
+        const orphanedSubscriptions = await knex('members_stripe_customers_subscriptions')
+            .select('id')
+            .whereNotIn(
+                'customer_id',
+                knex('members_stripe_customers')
+                    .select('customer_id')
+            );
+
+        if (!orphanedSubscriptions || !orphanedSubscriptions.length) {
+            logging.info('No orphaned subscription records found');
+            return;
+        }
+
+        logging.info(`Deleting ${orphanedSubscriptions.length} orphaned subscriptions`);
+        await knex('members_stripe_customers_subscriptions')
+            .whereIn('id', orphanedSubscriptions.map(subscription => subscription.id))
+            .del();
+    },
+
+    async down() {}
+};