Improved subscriptions in data generator

ref PROD-244

- Added support for canceled subscriptions and different subscription statusses
- Removed generation of susbcriptions table (not used)
- Added old canceled subscriptions for free members
- Added both positive and negative MRR events

ghost/data-generator/lib/DataGenerator.js

@@ -50,11 +50,13 @@ class DataGenerator {
         // Add missing dependencies
         for (const table of this.tableList) {
             table.importer = importers[table.name];
+
             // eslint-disable-next-line no-unused-vars
             table.dependencies = Object.entries(schema[table.name]).reduce((acc, [_col, data]) => {
                 if (data.references) {
                     const referencedTable = data.references.split('.')[0];
-                    if (!acc.includes(referencedTable)) {
+                    // The ghost_subscriptions_id property has a foreign key to the subscriptions table, but we don't use that table yet atm, so don't add it as a dependency
+                    if (!acc.includes(referencedTable) && referencedTable !== 'subscriptions') {
                         acc.push(referencedTable);
                     }
                 }

ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js

@@ -3,40 +3,80 @@ const {faker} = require('@faker-js/faker');
 
 class MembersPaidSubscriptionEventsImporter extends TableImporter {
     static table = 'members_paid_subscription_events';
-    static dependencies = ['subscriptions', 'members_stripe_customers_subscriptions'];
+    static dependencies = ['members_stripe_customers_subscriptions'];
 
     constructor(knex, transaction) {
         super(MembersPaidSubscriptionEventsImporter.table, knex, transaction);
     }
 
-    async import(quantity) {
-        const subscriptions = await this.transaction.select('id', 'member_id', 'currency', 'created_at').from('subscriptions');
-        this.membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id', 'plan_id', 'mrr').from('members_stripe_customers_subscriptions');
+    async import() {
+        const subscriptions = await this.transaction.select('id', 'customer_id', 'plan_currency', 'plan_amount', 'created_at', 'plan_id', 'status', 'cancel_at_period_end', 'current_period_end').from('members_stripe_customers_subscriptions');
+        this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers');
 
-        await this.importForEach(subscriptions, quantity ? quantity / subscriptions.length : 1);
+        await this.importForEach(subscriptions, 2);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+    }
+
+    isActiveSubscriptionStatus(status) {
+        return ['active', 'trialing', 'unpaid', 'past_due'].includes(status);
+    }
+
+    getStatus(modelToCheck) {
+        const status = modelToCheck.status;
+        const canceled = modelToCheck.cancel_at_period_end;
+
+        if (status === 'canceled') {
+            return 'expired';
+        }
+
+        if (canceled) {
+            return 'canceled';
+        }
+
+        if (this.isActiveSubscriptionStatus(status)) {
+            return 'active';
+        }
+
+        return 'inactive';
     }
 
     generate() {
-        if (!this.model.currency) {
-            // Not a paid subscription
-            return null;
+        this.count += 1;
+
+        const isActive = this.isActiveSubscriptionStatus(this.model.status);
+        if (this.count > 1 && isActive) {
+            // We only need one event, because the MRR is still here
+            return;
         }
-        // TODO: Implement upgrades
-        const membersStripeCustomersSubscription = this.membersStripeCustomersSubscriptions.find((m) => {
-            return m.ghost_subscription_id === this.model.id;
-        });
+
+        if (this.model.status === 'incomplete' || this.model.status === 'incomplete_expired') {
+            // Not a paid subscription
+            return;
+        }
+
+        const memberCustomer = this.membersStripeCustomers.find(c => c.customer_id === this.model.customer_id);
+        const isMonthly = this.model.plan_interval === 'month';
+
+        // Note that we need to recalculate the MRR, because it will be zero for inactive subscrptions
+        const mrr = isMonthly ? this.model.plan_amount : Math.floor(this.model.plan_amount / 12);
+
+        // todo: implement + MRR and -MRR in case of inactive subscriptions
         return {
             id: faker.database.mongodbObjectId(),
             // TODO: Support expired / updated / cancelled events too
-            type: 'created',
-            member_id: this.model.member_id,
+            type: this.count === 1 ? 'created' : this.getStatus(this.model),
+            member_id: memberCustomer.member_id,
             subscription_id: this.model.id,
-            from_plan: null,
-            to_plan: membersStripeCustomersSubscription.plan_id,
-            currency: this.model.currency,
+            from_plan: this.count === 1 ? null : this.model.plan_id,
+            to_plan: this.count === 1 ? this.model.plan_id : null,
+            currency: this.model.plan_currency,
             source: 'stripe',
-            mrr_delta: membersStripeCustomersSubscription.mrr,
-            created_at: this.model.created_at
+            mrr_delta: this.count === 1 ? mrr : -mrr,
+            created_at: this.count === 1 ? this.model.created_at : this.model.current_period_end
         };
     }
 }

ghost/data-generator/lib/importers/MembersProductsImporter.js

@@ -13,7 +13,7 @@ class MembersProductsImporter extends TableImporter {
 
     async import(quantity) {
         const members = await this.transaction.select('id').from('members').whereNot('status', 'free');
-        this.products = await this.transaction.select('id').from('products').whereNot('name', 'Free');
+        this.products = await this.transaction.select('id').from('products').whereNot('type', 'fee');
 
         await this.importForEach(members, quantity ? quantity / members.length : 1);
     }

ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js

@@ -10,12 +10,24 @@ class MembersStripeCustomersImporter extends TableImporter {
     }
 
     async import(quantity) {
-        const members = await this.transaction.select('id', 'name', 'email', 'created_at').from('members').where('status', 'paid');
+        const members = await this.transaction.select('id', 'name', 'email', 'created_at', 'status').from('members');
 
         await this.importForEach(members, quantity ? quantity / members.length : 1);
     }
 
     generate() {
+        if (this.model.status !== 'paid') {
+            // Only 30% of free members should have a stripe customer = have had a subscription in the past or tried to subscribe
+            // The number should increase the older the member is
+
+            const daysSinceMemberCreated = Math.floor((new Date() - new Date(this.model.created_at)) / (1000 * 60 * 60 * 24));
+            const shouldHaveStripeCustomer = faker.datatype.number({min: 0, max: 100}) < Math.max(Math.min(daysSinceMemberCreated / 30, 30), 5);
+
+            if (!shouldHaveStripeCustomer) {
+                return;
+            }
+        }
+
         return {
             id: faker.database.mongodbObjectId(),
             member_id: this.model.id,

ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js

@@ -1,52 +1,227 @@
 const {faker} = require('@faker-js/faker');
 const TableImporter = require('./TableImporter');
+const dateToDatabaseString = require('../utils/database-date');
+const generateEvents = require('../utils/event-generator');
+const {luck} = require('../utils/random');
 
 class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
     static table = 'members_stripe_customers_subscriptions';
-    static dependencies = ['subscriptions', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices'];
+    static dependencies = ['members', 'members_products', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices'];
 
     constructor(knex, transaction) {
         super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction);
     }
 
     async import() {
-        const subscriptions = await this.transaction.select('id', 'member_id', 'tier_id', 'cadence', 'created_at', 'expires_at').from('subscriptions');
+        this.membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products');
+        this.members = await this.transaction.select('id', 'status', 'created_at').from('members');//.where('status', 'paid');
         this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers');
-        this.products = await this.transaction.select('id', 'name').from('products');
+        this.products = await this.transaction.select('id', 'name').from('products').whereNot('type', 'free');
         this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
         this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices');
 
-        await this.importForEach(subscriptions, 1);
+        await this.importForEach(this.members, 2);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+        this.lastSubscriptionStart = null;
     }
 
     generate() {
-        const customer = this.membersStripeCustomers.find(c => this.model.member_id === c.member_id);
-        const isMonthly = this.model.cadence === 'month';
-        const ghostProduct = this.products.find(product => product.id === this.model.tier_id);
-        const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.tier_id);
+        this.count += 1;
+
+        const member = this.model;
+        const customer = this.membersStripeCustomers.find(c => this.model.id === c.member_id);
+
+        if (!customer) {
+            // This is a requirement, so skip if we don't have a customer
+            return;
+        }
+
+        if (this.count > 1 && member.status !== 'paid') {
+            return;
+        }
+
+        const memberProduct = this.membersProducts.find(p => p.member_id === this.model.id);
+        let ghostProduct = memberProduct ? this.products.find(product => product.id === memberProduct.product_id) : null;
+
+        // Whether we should create a valid subscription or not
+        // We'll only create one valid subscription for each member if they are currently paid
+        let createValid = this.count === 1 && member.status === 'paid';
+
+        if (!ghostProduct) {
+            // Generate canceled, incomplete, incomplete_expired or unpaid subscriptions
+            // Choose a random paid product
+            ghostProduct = faker.helpers.arrayElement(this.products);
+            createValid = false;
+        }
+
+        const isMonthly = luck(70);
+        const stripeProduct = this.stripeProducts.find(product => product.product_id === ghostProduct.id);
         const stripePrice = this.stripePrices.find((price) => {
             return price.stripe_product_id === stripeProduct.stripe_product_id &&
                 (isMonthly ? price.interval === 'month' : price.interval === 'year');
         });
-        const mrr = isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12);
+        const mrr = createValid ? (isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12)) : 0;
+
+        const referenceEndDate = this.lastSubscriptionStart ?? new Date();
+
+        if (!createValid) {
+            if (isMonthly) {
+                referenceEndDate.setMonth(referenceEndDate.getMonth() - 1);
+            } else {
+                referenceEndDate.setFullYear(referenceEndDate.getFullYear() - 1);
+            }
+        }
+
+        if (referenceEndDate < member.created_at) {
+            // Not possible to create an invalid subscription here
+            return;
+        }
+
+        const [startDate] = generateEvents({
+            total: 1,
+            trend: 'negative',
+            startTime: new Date(member.created_at),
+            endTime: referenceEndDate,
+            shape: 'ease-out'
+        });
+        this.lastSubscriptionStart = startDate;
+        const endDate = new Date(startDate);
+
+        if (createValid) {
+            // End date should be in the future
+
+            if (isMonthly) {
+                endDate.setFullYear(new Date().getFullYear());
+                endDate.setMonth(new Date().getMonth());
+                if (endDate < new Date()) {
+                    endDate.setMonth(endDate.getMonth() + 1);
+                }
+            } else {
+                endDate.setFullYear(new Date().getFullYear());
+                if (endDate < new Date()) {
+                    endDate.setFullYear(endDate.getFullYear() + 1);
+                }
+            }
+        } else {
+            // End date should be in the past
+            if (isMonthly) {
+                // What is the month difference between startDate and now? Pick a random number in between
+                const monthDiff = (new Date().getFullYear() - startDate.getFullYear()) * 12 + (new Date().getMonth() - startDate.getMonth());
+                if (monthDiff === 0) {
+                    // Not possible to create an invalid subscription here
+                    return;
+                }
+
+                const randomMonthDiff = faker.datatype.number({min: 1, max: monthDiff});
+                endDate.setMonth(startDate.getMonth() + randomMonthDiff);
+            } else {
+                // What is the year difference between startDate and now? Pick a random number in between
+                const yearDiff = new Date().getFullYear() - startDate.getFullYear();
+
+                if (yearDiff === 0) {
+                    // Not possible to create an invalid subscription here
+                    return;
+                }
+                const randomYearDiff = faker.datatype.number({min: 1, max: yearDiff});
+
+                endDate.setFullYear(startDate.getFullYear() + randomYearDiff);
+            }
+        }
+
+        // Simulate some different statusses here:
+        // - active, not ending (cancel_at_period_end  = false)
+        // - active, ending (cancel_at_period_end = true)
+        // - canceled -> current_period_end can be in both past or present, cancel_at_period_end can be both true or false
+        // - incomplete_expired -> user tried to pay but 3D secure expired
+        // - incomplete -> waiting on 3D secure
+        // - trialing -> need to set trial_end_at to a date in the future
+        // - past_due -> last paymet failed, but subscription still active until tried a couple of times
+        // - unpaid -> all payment attempts failed - but keep the subscription active (special setting in Stripe)
+
+        const validStatusses = new Array(10).fill({
+            status: 'active',
+            cancel_at_period_end: false
+        });
+
+        // Trialing only possible when the startDate > 1 month ago
+        const monthAgo = new Date();
+
+        if (!isMonthly) {
+            // Year ago
+            monthAgo.setFullYear(monthAgo.getFullYear() - 1);
+        } else {
+            // Month ago
+            monthAgo.setMonth(monthAgo.getMonth() - 1);
+        }
+
+        if (startDate > monthAgo) {
+            validStatusses.push({
+                status: 'trialing',
+                cancel_at_period_end: false,
+                trial_end_at: dateToDatabaseString(endDate),
+                trial_start_at: dateToDatabaseString(startDate)
+            });
+        }
+
+        // Past due only possible if startDate < 1 month ago
+        if (startDate < monthAgo) {
+            validStatusses.push({
+                status: 'past_due',
+                cancel_at_period_end: false
+            });
+            validStatusses.push({
+                status: 'unpaid',
+                cancel_at_period_end: false
+            });
+        }
+
+        const invalidStatusses = [
+            {
+                status: 'canceled',
+                cancel_at_period_end: true
+            },
+            {
+                status: 'canceled',
+                cancel_at_period_end: false
+            },
+            {
+                status: 'incomplete_expired',
+                cancel_at_period_end: false
+            },
+            {
+                status: 'incomplete',
+                cancel_at_period_end: false
+            }
+        ];
+
+        const status = createValid ? faker.helpers.arrayElement(validStatusses) : faker.helpers.arrayElement(invalidStatusses);
+
         return {
             id: faker.database.mongodbObjectId(),
             customer_id: customer.customer_id,
-            ghost_subscription_id: this.model.id,
             subscription_id: `sub_${faker.random.alphaNumeric(14)}`,
             stripe_price_id: stripePrice.stripe_price_id,
-            status: 'active',
-            cancel_at_period_end: false,
-            current_period_end: this.model.expires_at,
-            start_date: this.model.created_at,
-            created_at: this.model.created_at,
+            start_date: dateToDatabaseString(startDate),
+            created_at: dateToDatabaseString(startDate),
             created_by: 'unused',
             mrr,
             plan_id: stripeProduct.stripe_product_id,
             plan_nickname: `${ghostProduct.name} - ${stripePrice.nickname}`,
             plan_interval: stripePrice.interval,
             plan_amount: stripePrice.amount,
-            plan_currency: stripePrice.currency
+            plan_currency: stripePrice.currency,
+
+            // Defaults
+            status: 'active',
+            cancel_at_period_end: false,
+            current_period_end: dateToDatabaseString(endDate),
+
+            // Override
+            ...status
         };
     }
 }

ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js

@@ -5,7 +5,7 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class MembersSubscribeEventsImporter extends TableImporter {
     static table = 'members_subscribe_events';
-    static dependencies = ['members', 'newsletters'/*, 'subscriptions'*/];
+    static dependencies = ['members', 'newsletters'];
 
     constructor(knex, transaction) {
         super(MembersSubscribeEventsImporter.table, knex, transaction);
@@ -14,7 +14,6 @@ class MembersSubscribeEventsImporter extends TableImporter {
     async import(quantity) {
         const members = await this.transaction.select('id', 'created_at', 'status').from('members');
         this.newsletters = await this.transaction.select('id').from('newsletters').orderBy('sort_order');
-        //this.subscriptions = await this.transaction.select('member_id', 'created_at').from('subscriptions');
 
         await this.importForEach(members, quantity ? quantity / members.length : this.newsletters.length);
     }

ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js

@@ -4,25 +4,24 @@ const {luck} = require('../utils/random');
 
 class MembersSubscriptionCreatedEventsImporter extends TableImporter {
     static table = 'members_subscription_created_events';
-    static dependencies = ['members_stripe_customers_subscriptions', 'subscriptions', 'posts'];
+    static dependencies = ['members_stripe_customers_subscriptions', 'posts'];
 
     constructor(knex, transaction) {
         super(MembersSubscriptionCreatedEventsImporter.table, knex, transaction);
     }
 
     async import(quantity) {
-        const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id').from('members_stripe_customers_subscriptions');
-        this.subscriptions = await this.transaction.select('id', 'created_at', 'member_id').from('subscriptions');
+        const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'created_at', 'customer_id').from('members_stripe_customers_subscriptions');
+        this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers');
         this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
 
         await this.importForEach(membersStripeCustomersSubscriptions, quantity ? quantity / membersStripeCustomersSubscriptions.length : 1);
     }
 
     generate() {
-        const subscription = this.subscriptions.find(s => s.id === this.model.ghost_subscription_id);
         let attribution = {};
         if (luck(10)) {
-            const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(subscription.created_at));
+            const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(this.model.created_at));
             if (post) {
                 attribution = {
                     attribution_id: post.id,
@@ -31,10 +30,12 @@ class MembersSubscriptionCreatedEventsImporter extends TableImporter {
                 };
             }
         }
+        const memberCustomer = this.membersStripeCustomers.find(c => c.customer_id === this.model.customer_id);
+
         return Object.assign({}, {
             id: faker.database.mongodbObjectId(),
-            created_at: subscription.created_at,
-            member_id: subscription.member_id,
+            created_at: this.model.created_at,
+            member_id: memberCustomer.member_id,
             subscription_id: this.model.id,
             // TODO: Implement referrers
             referrer_source: null,

ghost/data-generator/lib/importers/index.js

@@ -15,7 +15,6 @@ module.exports = [
     require('./MembersNewslettersImporter'),
     require('./StripeProductsImporter'),
     require('./StripePricesImporter'),
-    require('./SubscriptionsImporter'),
     require('./EmailsImporter'),
     require('./EmailBatchesImporter'),
     require('./EmailRecipientsImporter'),