Added members.email_{count,opened_count} column migrations (#12470)

refs https://github.com/TryGhost/Ghost/issues/12461

- adds `members.email_count` and `members.email_opened_count` columns to contain cached counts for faster queries when outputting member data via API
- adds migration to populate cached counts with existing data
  - tested locally on ~50k members which took ~4sec on mysql
- updates members output serializer to include the new fields in API output

core/server/api/canary/utils/serializers/output/members.js

@@ -101,6 +101,8 @@ function serializeMember(member, options) {
         stripe: json.stripe,
         avatar_image: json.avatar_image,
         comped: comped,
+        email_count: json.email_count,
+        email_opened_count: json.email_opened_count,
         email_open_rate: json.email_open_rate
     };
 }
@@ -145,6 +147,8 @@ function createSerializer(debugString, serialize) {
  * @prop {null|SerializedMemberStripeData} stripe
  * @prop {string} avatar_image
  * @prop {boolean} comped
+ * @prop {number} email_count
+ * @prop {number} email_opened_count
  * @prop {number} email_open_rate
  */
 

core/server/data/migrations/versions/3.40/02-add members-email-aggregation-columns.js

@@ -0,0 +1,16 @@
+const {createAddColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createAddColumnMigration('members', 'email_count', {
+        type: 'integer',
+        unsigned: true,
+        nullable: false,
+        defaultTo: 0
+    }),
+    createAddColumnMigration('members', 'email_opened_count', {
+        type: 'integer',
+        unsigned: true,
+        nullable: false,
+        defaultTo: 0
+    })
+);

core/server/data/migrations/versions/3.40/03-populate-members-email-counts.js

@@ -0,0 +1,15 @@
+const logging = require('../../../../../shared/logging');
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Populating email counts on members');
+        await knex('members')
+            .update({
+                email_count: knex.raw('(SELECT COUNT(id) FROM email_recipients WHERE email_recipients.member_id = members.id)'),
+                email_opened_count: knex.raw('(SELECT COUNT(id) FROM email_recipients WHERE email_recipients.member_id = members.id AND email_recipients.opened_at IS NOT NULL)')
+            });
+    },
+
+    async function down() {}
+);

core/server/data/schema/schema.js

@@ -378,6 +378,8 @@ module.exports = {
         note: {type: 'string', maxlength: 2000, nullable: true},
         geolocation: {type: 'string', maxlength: 2000, nullable: true},
         subscribed: {type: 'bool', nullable: true, defaultTo: true},
+        email_count: {type: 'integer', unsigned: true, nullable: false, defaultTo: 0},
+        email_opened_count: {type: 'integer', unsigned: true, nullable: false, defaultTo: 0},
         email_open_rate: {type: 'integer', unsigned: true, nullable: true, index: true},
         created_at: {type: 'dateTime', nullable: false},
         created_by: {type: 'string', maxlength: 24, nullable: false},

core/server/models/member.js

@@ -11,7 +11,9 @@ const Member = ghostBookshelf.Model.extend({
     defaults() {
         return {
             subscribed: true,
-            uuid: uuid.v4()
+            uuid: uuid.v4(),
+            email_count: 0,
+            email_opened_count: 0
         };
     },
 

test/unit/data/schema/integrity_spec.js

@@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = '4e31c2f48018f5a95244a78064ddf4ff';
+    const currentSchemaHash = 'c7e7b458f2c12ad193326784fefb80fe';
     const currentFixturesHash = 'd46d696c94d03e41a5903500547fea77';
     const currentSettingsHash = 'd3821715e4b34d92d6ba6ed0d4918f5c';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';