From 70b991cc1c5a1efff6f09b664072c2eb26bf1bfe Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 14 Dec 2023 16:38:14 +0100 Subject: [PATCH] Improved subscriptions in data generator ref PROD-244 - Added support for canceled subscriptions and different subscription statusses - Removed generation of susbcriptions table (not used) - Added old canceled subscriptions for free members - Added both positive and negative MRR events --- ghost/data-generator/lib/DataGenerator.js | 4 +- .../MembersPaidSubscriptionEventsImporter.js | 78 +++++-- .../lib/importers/MembersProductsImporter.js | 2 +- .../MembersStripeCustomersImporter.js | 14 +- ...ersStripeCustomersSubscriptionsImporter.js | 207 ++++++++++++++++-- .../MembersSubscribeEventsImporter.js | 3 +- ...embersSubscriptionCreatedEventsImporter.js | 15 +- ghost/data-generator/lib/importers/index.js | 1 - 8 files changed, 276 insertions(+), 48 deletions(-) diff --git a/ghost/data-generator/lib/DataGenerator.js b/ghost/data-generator/lib/DataGenerator.js index a8fdba86b8..8140297740 100644 --- a/ghost/data-generator/lib/DataGenerator.js +++ b/ghost/data-generator/lib/DataGenerator.js @@ -50,11 +50,13 @@ class DataGenerator { // Add missing dependencies for (const table of this.tableList) { table.importer = importers[table.name]; + // eslint-disable-next-line no-unused-vars table.dependencies = Object.entries(schema[table.name]).reduce((acc, [_col, data]) => { if (data.references) { const referencedTable = data.references.split('.')[0]; - if (!acc.includes(referencedTable)) { + // The ghost_subscriptions_id property has a foreign key to the subscriptions table, but we don't use that table yet atm, so don't add it as a dependency + if (!acc.includes(referencedTable) && referencedTable !== 'subscriptions') { acc.push(referencedTable); } } diff --git a/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js b/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js index afc212f2e5..7d5625f790 100644 --- a/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js +++ b/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js @@ -3,40 +3,80 @@ const {faker} = require('@faker-js/faker'); class MembersPaidSubscriptionEventsImporter extends TableImporter { static table = 'members_paid_subscription_events'; - static dependencies = ['subscriptions', 'members_stripe_customers_subscriptions']; + static dependencies = ['members_stripe_customers_subscriptions']; constructor(knex, transaction) { super(MembersPaidSubscriptionEventsImporter.table, knex, transaction); } - async import(quantity) { - const subscriptions = await this.transaction.select('id', 'member_id', 'currency', 'created_at').from('subscriptions'); - this.membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id', 'plan_id', 'mrr').from('members_stripe_customers_subscriptions'); + async import() { + const subscriptions = await this.transaction.select('id', 'customer_id', 'plan_currency', 'plan_amount', 'created_at', 'plan_id', 'status', 'cancel_at_period_end', 'current_period_end').from('members_stripe_customers_subscriptions'); + this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers'); - await this.importForEach(subscriptions, quantity ? quantity / subscriptions.length : 1); + await this.importForEach(subscriptions, 2); + } + + setReferencedModel(model) { + this.model = model; + this.count = 0; + } + + isActiveSubscriptionStatus(status) { + return ['active', 'trialing', 'unpaid', 'past_due'].includes(status); + } + + getStatus(modelToCheck) { + const status = modelToCheck.status; + const canceled = modelToCheck.cancel_at_period_end; + + if (status === 'canceled') { + return 'expired'; + } + + if (canceled) { + return 'canceled'; + } + + if (this.isActiveSubscriptionStatus(status)) { + return 'active'; + } + + return 'inactive'; } generate() { - if (!this.model.currency) { - // Not a paid subscription - return null; + this.count += 1; + + const isActive = this.isActiveSubscriptionStatus(this.model.status); + if (this.count > 1 && isActive) { + // We only need one event, because the MRR is still here + return; } - // TODO: Implement upgrades - const membersStripeCustomersSubscription = this.membersStripeCustomersSubscriptions.find((m) => { - return m.ghost_subscription_id === this.model.id; - }); + + if (this.model.status === 'incomplete' || this.model.status === 'incomplete_expired') { + // Not a paid subscription + return; + } + + const memberCustomer = this.membersStripeCustomers.find(c => c.customer_id === this.model.customer_id); + const isMonthly = this.model.plan_interval === 'month'; + + // Note that we need to recalculate the MRR, because it will be zero for inactive subscrptions + const mrr = isMonthly ? this.model.plan_amount : Math.floor(this.model.plan_amount / 12); + + // todo: implement + MRR and -MRR in case of inactive subscriptions return { id: faker.database.mongodbObjectId(), // TODO: Support expired / updated / cancelled events too - type: 'created', - member_id: this.model.member_id, + type: this.count === 1 ? 'created' : this.getStatus(this.model), + member_id: memberCustomer.member_id, subscription_id: this.model.id, - from_plan: null, - to_plan: membersStripeCustomersSubscription.plan_id, - currency: this.model.currency, + from_plan: this.count === 1 ? null : this.model.plan_id, + to_plan: this.count === 1 ? this.model.plan_id : null, + currency: this.model.plan_currency, source: 'stripe', - mrr_delta: membersStripeCustomersSubscription.mrr, - created_at: this.model.created_at + mrr_delta: this.count === 1 ? mrr : -mrr, + created_at: this.count === 1 ? this.model.created_at : this.model.current_period_end }; } } diff --git a/ghost/data-generator/lib/importers/MembersProductsImporter.js b/ghost/data-generator/lib/importers/MembersProductsImporter.js index 631f563cf1..1eaf9b7eec 100644 --- a/ghost/data-generator/lib/importers/MembersProductsImporter.js +++ b/ghost/data-generator/lib/importers/MembersProductsImporter.js @@ -13,7 +13,7 @@ class MembersProductsImporter extends TableImporter { async import(quantity) { const members = await this.transaction.select('id').from('members').whereNot('status', 'free'); - this.products = await this.transaction.select('id').from('products').whereNot('name', 'Free'); + this.products = await this.transaction.select('id').from('products').whereNot('type', 'fee'); await this.importForEach(members, quantity ? quantity / members.length : 1); } diff --git a/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js b/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js index dea9a886e2..ecf59f4096 100644 --- a/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js +++ b/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js @@ -10,12 +10,24 @@ class MembersStripeCustomersImporter extends TableImporter { } async import(quantity) { - const members = await this.transaction.select('id', 'name', 'email', 'created_at').from('members').where('status', 'paid'); + const members = await this.transaction.select('id', 'name', 'email', 'created_at', 'status').from('members'); await this.importForEach(members, quantity ? quantity / members.length : 1); } generate() { + if (this.model.status !== 'paid') { + // Only 30% of free members should have a stripe customer = have had a subscription in the past or tried to subscribe + // The number should increase the older the member is + + const daysSinceMemberCreated = Math.floor((new Date() - new Date(this.model.created_at)) / (1000 * 60 * 60 * 24)); + const shouldHaveStripeCustomer = faker.datatype.number({min: 0, max: 100}) < Math.max(Math.min(daysSinceMemberCreated / 30, 30), 5); + + if (!shouldHaveStripeCustomer) { + return; + } + } + return { id: faker.database.mongodbObjectId(), member_id: this.model.id, diff --git a/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js b/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js index 4bf38f20a8..89e2d0ec85 100644 --- a/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js +++ b/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js @@ -1,52 +1,227 @@ const {faker} = require('@faker-js/faker'); const TableImporter = require('./TableImporter'); +const dateToDatabaseString = require('../utils/database-date'); +const generateEvents = require('../utils/event-generator'); +const {luck} = require('../utils/random'); class MembersStripeCustomersSubscriptionsImporter extends TableImporter { static table = 'members_stripe_customers_subscriptions'; - static dependencies = ['subscriptions', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices']; + static dependencies = ['members', 'members_products', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices']; constructor(knex, transaction) { super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction); } async import() { - const subscriptions = await this.transaction.select('id', 'member_id', 'tier_id', 'cadence', 'created_at', 'expires_at').from('subscriptions'); + this.membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products'); + this.members = await this.transaction.select('id', 'status', 'created_at').from('members');//.where('status', 'paid'); this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers'); - this.products = await this.transaction.select('id', 'name').from('products'); + this.products = await this.transaction.select('id', 'name').from('products').whereNot('type', 'free'); this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products'); this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices'); - await this.importForEach(subscriptions, 1); + await this.importForEach(this.members, 2); + } + + setReferencedModel(model) { + this.model = model; + this.count = 0; + this.lastSubscriptionStart = null; } generate() { - const customer = this.membersStripeCustomers.find(c => this.model.member_id === c.member_id); - const isMonthly = this.model.cadence === 'month'; - const ghostProduct = this.products.find(product => product.id === this.model.tier_id); - const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.tier_id); + this.count += 1; + + const member = this.model; + const customer = this.membersStripeCustomers.find(c => this.model.id === c.member_id); + + if (!customer) { + // This is a requirement, so skip if we don't have a customer + return; + } + + if (this.count > 1 && member.status !== 'paid') { + return; + } + + const memberProduct = this.membersProducts.find(p => p.member_id === this.model.id); + let ghostProduct = memberProduct ? this.products.find(product => product.id === memberProduct.product_id) : null; + + // Whether we should create a valid subscription or not + // We'll only create one valid subscription for each member if they are currently paid + let createValid = this.count === 1 && member.status === 'paid'; + + if (!ghostProduct) { + // Generate canceled, incomplete, incomplete_expired or unpaid subscriptions + // Choose a random paid product + ghostProduct = faker.helpers.arrayElement(this.products); + createValid = false; + } + + const isMonthly = luck(70); + const stripeProduct = this.stripeProducts.find(product => product.product_id === ghostProduct.id); const stripePrice = this.stripePrices.find((price) => { return price.stripe_product_id === stripeProduct.stripe_product_id && (isMonthly ? price.interval === 'month' : price.interval === 'year'); }); - const mrr = isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12); + const mrr = createValid ? (isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12)) : 0; + + const referenceEndDate = this.lastSubscriptionStart ?? new Date(); + + if (!createValid) { + if (isMonthly) { + referenceEndDate.setMonth(referenceEndDate.getMonth() - 1); + } else { + referenceEndDate.setFullYear(referenceEndDate.getFullYear() - 1); + } + } + + if (referenceEndDate < member.created_at) { + // Not possible to create an invalid subscription here + return; + } + + const [startDate] = generateEvents({ + total: 1, + trend: 'negative', + startTime: new Date(member.created_at), + endTime: referenceEndDate, + shape: 'ease-out' + }); + this.lastSubscriptionStart = startDate; + const endDate = new Date(startDate); + + if (createValid) { + // End date should be in the future + + if (isMonthly) { + endDate.setFullYear(new Date().getFullYear()); + endDate.setMonth(new Date().getMonth()); + if (endDate < new Date()) { + endDate.setMonth(endDate.getMonth() + 1); + } + } else { + endDate.setFullYear(new Date().getFullYear()); + if (endDate < new Date()) { + endDate.setFullYear(endDate.getFullYear() + 1); + } + } + } else { + // End date should be in the past + if (isMonthly) { + // What is the month difference between startDate and now? Pick a random number in between + const monthDiff = (new Date().getFullYear() - startDate.getFullYear()) * 12 + (new Date().getMonth() - startDate.getMonth()); + if (monthDiff === 0) { + // Not possible to create an invalid subscription here + return; + } + + const randomMonthDiff = faker.datatype.number({min: 1, max: monthDiff}); + endDate.setMonth(startDate.getMonth() + randomMonthDiff); + } else { + // What is the year difference between startDate and now? Pick a random number in between + const yearDiff = new Date().getFullYear() - startDate.getFullYear(); + + if (yearDiff === 0) { + // Not possible to create an invalid subscription here + return; + } + const randomYearDiff = faker.datatype.number({min: 1, max: yearDiff}); + + endDate.setFullYear(startDate.getFullYear() + randomYearDiff); + } + } + + // Simulate some different statusses here: + // - active, not ending (cancel_at_period_end = false) + // - active, ending (cancel_at_period_end = true) + // - canceled -> current_period_end can be in both past or present, cancel_at_period_end can be both true or false + // - incomplete_expired -> user tried to pay but 3D secure expired + // - incomplete -> waiting on 3D secure + // - trialing -> need to set trial_end_at to a date in the future + // - past_due -> last paymet failed, but subscription still active until tried a couple of times + // - unpaid -> all payment attempts failed - but keep the subscription active (special setting in Stripe) + + const validStatusses = new Array(10).fill({ + status: 'active', + cancel_at_period_end: false + }); + + // Trialing only possible when the startDate > 1 month ago + const monthAgo = new Date(); + + if (!isMonthly) { + // Year ago + monthAgo.setFullYear(monthAgo.getFullYear() - 1); + } else { + // Month ago + monthAgo.setMonth(monthAgo.getMonth() - 1); + } + + if (startDate > monthAgo) { + validStatusses.push({ + status: 'trialing', + cancel_at_period_end: false, + trial_end_at: dateToDatabaseString(endDate), + trial_start_at: dateToDatabaseString(startDate) + }); + } + + // Past due only possible if startDate < 1 month ago + if (startDate < monthAgo) { + validStatusses.push({ + status: 'past_due', + cancel_at_period_end: false + }); + validStatusses.push({ + status: 'unpaid', + cancel_at_period_end: false + }); + } + + const invalidStatusses = [ + { + status: 'canceled', + cancel_at_period_end: true + }, + { + status: 'canceled', + cancel_at_period_end: false + }, + { + status: 'incomplete_expired', + cancel_at_period_end: false + }, + { + status: 'incomplete', + cancel_at_period_end: false + } + ]; + + const status = createValid ? faker.helpers.arrayElement(validStatusses) : faker.helpers.arrayElement(invalidStatusses); + return { id: faker.database.mongodbObjectId(), customer_id: customer.customer_id, - ghost_subscription_id: this.model.id, subscription_id: `sub_${faker.random.alphaNumeric(14)}`, stripe_price_id: stripePrice.stripe_price_id, - status: 'active', - cancel_at_period_end: false, - current_period_end: this.model.expires_at, - start_date: this.model.created_at, - created_at: this.model.created_at, + start_date: dateToDatabaseString(startDate), + created_at: dateToDatabaseString(startDate), created_by: 'unused', mrr, plan_id: stripeProduct.stripe_product_id, plan_nickname: `${ghostProduct.name} - ${stripePrice.nickname}`, plan_interval: stripePrice.interval, plan_amount: stripePrice.amount, - plan_currency: stripePrice.currency + plan_currency: stripePrice.currency, + + // Defaults + status: 'active', + cancel_at_period_end: false, + current_period_end: dateToDatabaseString(endDate), + + // Override + ...status }; } } diff --git a/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js b/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js index 94984c54e0..cbd8fbb9e5 100644 --- a/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js +++ b/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js @@ -5,7 +5,7 @@ const dateToDatabaseString = require('../utils/database-date'); class MembersSubscribeEventsImporter extends TableImporter { static table = 'members_subscribe_events'; - static dependencies = ['members', 'newsletters'/*, 'subscriptions'*/]; + static dependencies = ['members', 'newsletters']; constructor(knex, transaction) { super(MembersSubscribeEventsImporter.table, knex, transaction); @@ -14,7 +14,6 @@ class MembersSubscribeEventsImporter extends TableImporter { async import(quantity) { const members = await this.transaction.select('id', 'created_at', 'status').from('members'); this.newsletters = await this.transaction.select('id').from('newsletters').orderBy('sort_order'); - //this.subscriptions = await this.transaction.select('member_id', 'created_at').from('subscriptions'); await this.importForEach(members, quantity ? quantity / members.length : this.newsletters.length); } diff --git a/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js b/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js index 081436a5cd..b35055f727 100644 --- a/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js +++ b/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js @@ -4,25 +4,24 @@ const {luck} = require('../utils/random'); class MembersSubscriptionCreatedEventsImporter extends TableImporter { static table = 'members_subscription_created_events'; - static dependencies = ['members_stripe_customers_subscriptions', 'subscriptions', 'posts']; + static dependencies = ['members_stripe_customers_subscriptions', 'posts']; constructor(knex, transaction) { super(MembersSubscriptionCreatedEventsImporter.table, knex, transaction); } async import(quantity) { - const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id').from('members_stripe_customers_subscriptions'); - this.subscriptions = await this.transaction.select('id', 'created_at', 'member_id').from('subscriptions'); + const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'created_at', 'customer_id').from('members_stripe_customers_subscriptions'); + this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers'); this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc'); await this.importForEach(membersStripeCustomersSubscriptions, quantity ? quantity / membersStripeCustomersSubscriptions.length : 1); } generate() { - const subscription = this.subscriptions.find(s => s.id === this.model.ghost_subscription_id); let attribution = {}; if (luck(10)) { - const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(subscription.created_at)); + const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(this.model.created_at)); if (post) { attribution = { attribution_id: post.id, @@ -31,10 +30,12 @@ class MembersSubscriptionCreatedEventsImporter extends TableImporter { }; } } + const memberCustomer = this.membersStripeCustomers.find(c => c.customer_id === this.model.customer_id); + return Object.assign({}, { id: faker.database.mongodbObjectId(), - created_at: subscription.created_at, - member_id: subscription.member_id, + created_at: this.model.created_at, + member_id: memberCustomer.member_id, subscription_id: this.model.id, // TODO: Implement referrers referrer_source: null, diff --git a/ghost/data-generator/lib/importers/index.js b/ghost/data-generator/lib/importers/index.js index b62f07cd54..54f7ff56f3 100644 --- a/ghost/data-generator/lib/importers/index.js +++ b/ghost/data-generator/lib/importers/index.js @@ -15,7 +15,6 @@ module.exports = [ require('./MembersNewslettersImporter'), require('./StripeProductsImporter'), require('./StripePricesImporter'), - require('./SubscriptionsImporter'), require('./EmailsImporter'), require('./EmailBatchesImporter'), require('./EmailRecipientsImporter'),