Updated members_status_events table (#12647)

refs https://github.com/TryGhost/Ghost/issues/12602

* Updated members_status_events table

By replacing the `status` column with a `from_status` and `to_status`
column, we are able to track the changes between multiple statuses
easier, and accumulate the data. e.g. the delta of paid members in a
given time range is the sum of the `to_status` columns set to 'paid'
minus the sum of the `from_status` columns set to 'paid' within that
time range

* Updated MEGA to handle addition of 'comped' status

With the addition of the 'comped' status, we need to ensure that MEGA
will still send emails to the correct recipients. I've opted to use an
"inverse" filter, as that is the intention of the free/paid split in
MEGA - as far as MEGA is concerned, "free" is the opposite of "paid"

* Updated customQuery for MemberStatusEvent

With the `status` column replaced with `from_status` and `to_status`
this allows us to fix and update the customQuery to correctly accumulate
the data into deltas over time, broken down by day.

* Populated members_status_events table

As the table will be used to generate deltas, we need to backfill the
data so that existing sites will be able to sum up the deltas and
calculate correct data.

The assumptions used in backfilling is that a Member's current status,
is their only status.

core/server/data/migrations/versions/4.0/10-add-members-status-events-table.js

@@ -3,6 +3,7 @@ const {addTable} = require('../../utils');
 module.exports = addTable('members_status_events', {
     id: {type: 'string', maxlength: 24, nullable: false, primary: true},
     member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
-    status: {type: 'string', maxlength: 50, nullable: false},
+    from_status: {type: 'string', maxlength: 50, nullable: true},
+    to_status: {type: 'string', maxlength: 50, nullable: true},
     created_at: {type: 'dateTime', nullable: false}
 });

core/server/data/migrations/versions/4.0/17-populate-members-status-events-table.js

@@ -0,0 +1,38 @@
+const {chunk} = require('lodash');
+const ObjectID = require('bson-objectid');
+const logging = require('../../../../../shared/logging');
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Populating members_status_events from members table');
+        await knex('members_status_events').del();
+        const allMembers = await knex.select(
+            'id as member_id',
+            'status as to_status',
+            'created_at'
+        ).from('members');
+
+        const membersStatusEvents = allMembers.map((event) => {
+            return {
+                ...event,
+                id: ObjectID.generate(),
+                from_status: null
+            };
+        });
+
+        // SQLite3 supports 999 variables max, each row uses 5 variables so ⌊999/5⌋ = 199
+        const chunkSize = 199;
+
+        const eventChunks = chunk(membersStatusEvents, chunkSize);
+
+        for (const events of eventChunks) {
+            await knex.insert(events).into('members_status_events');
+        }
+    },
+    async function down(knex) {
+        logging.info('Deleting all members_status_events');
+        return knex('members_status_events').del();
+    }
+);
+

core/server/data/schema/schema.js

@@ -379,8 +379,13 @@ module.exports = {
     members_status_events: {
         id: {type: 'string', maxlength: 24, nullable: false, primary: true},
         member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
-        status: {
-            type: 'string', maxlength: 50, nullable: false, validations: {
+        from_status: {
+            type: 'string', maxlength: 50, nullable: true, validations: {
+                isIn: [['free', 'paid', 'comped']]
+            }
+        },
+        to_status: {
+            type: 'string', maxlength: 50, nullable: true, validations: {
                 isIn: [['free', 'paid', 'comped']]
             }
         },

core/server/models/member-status-event.js

@@ -11,9 +11,21 @@ const MemberStatusEvent = ghostBookshelf.Model.extend({
             const knex = ghostBookshelf.knex;
             return qb.clear('select')
                 .select(knex.raw('DATE(created_at) as date'))
-                .select(knex.raw(`SUM(CASE WHEN status='paid' THEN 1 ELSE 0 END) as paid_delta`))
-                .select(knex.raw(`SUM(CASE WHEN status='comped' THEN 1 ELSE 0 END) as comped_delta`))
-                .select(knex.raw(`SUM(CASE WHEN status='free' THEN 1 ELSE 0 END) as free_delta`))
+                .select(knex.raw(`SUM(
+                    CASE WHEN to_status='paid' THEN 1
+                    CASE WHEN from_status='paid' THEN -1
+                    ELSE 0 END
+                ) as paid_delta`))
+                .select(knex.raw(`SUM(
+                    CASE WHEN to_status='comped' THEN 1
+                    CASE WHEN from_status='comped' THEN -1
+                    ELSE 0 END
+                ) as comped_delta`))
+                .select(knex.raw(`SUM(
+                    CASE WHEN to_status='free' THEN 1
+                    CASE WHEN from_status='free' THEN -1
+                    ELSE 0 END
+                ) as free_delta`))
                 .groupByRaw('DATE(created_at)')
                 .orderByRaw('DATE(created_at)');
         }

core/server/services/mega/mega.js

@@ -98,7 +98,7 @@ const addEmail = async (postModel, options) => {
 
     switch (emailRecipientFilter) {
     case 'paid':
-        filterOptions.filter = 'subscribed:true+status:paid';
+        filterOptions.filter = 'subscribed:true+status:-free';
         break;
     case 'free':
         filterOptions.filter = 'subscribed:true+status:free';
@@ -295,7 +295,7 @@ async function getEmailMemberRows({emailModel, options}) {
 
     switch (recipientFilter) {
     case 'paid':
-        filterOptions.filter = 'subscribed:true+status:paid';
+        filterOptions.filter = 'subscribed:true+status:-free';
         break;
     case 'free':
         filterOptions.filter = 'subscribed:true+status:free';

test/unit/data/schema/integrity_spec.js

@@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = '3f88f2c34001cc34d74d945dbf4f0bb5';
+    const currentSchemaHash = 'd23279e16028161ab337000744480111';
     const currentFixturesHash = '370d0da0ab7c45050b2ff30bce8896ba';
     const currentSettingsHash = '24453dc02be9df7284acf1748862a545';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';