diff --git a/ghost/core/core/cli/generate-data.js b/ghost/core/core/cli/generate-data.js new file mode 100644 index 0000000000..e4771784b2 --- /dev/null +++ b/ghost/core/core/cli/generate-data.js @@ -0,0 +1,50 @@ +const Command = require('./command'); +const DataGenerator = require('@tryghost/data-generator'); + +module.exports = class REPL extends Command { + setup() { + this.help('Generates random data to populate the database for development & testing'); + this.argument('--events-only', {type: 'boolean', defaultValue: false, desc: 'Only generate events, skip other datatypes'}); + this.argument('--use-existing-posts', {type: 'boolean', defaultValue: false, desc: 'Generate data referencing the set of existing posts'}); + this.argument('--use-existing-tags', {type: 'boolean', defaultValue: false, desc: 'Generate data referencing the set of existing tags'}); + } + + initializeContext(context) { + const models = require('../server/models'); + const knex = require('../server/data/db/connection'); + + models.init(); + + context.models = models; + context.m = models; + context.knex = knex; + context.k = knex; + } + + async handle(argv = {}) { + const knex = require('../server/data/db/connection'); + const {tables: schema} = require('../server/data/schema/index'); + const dataGenerator = new DataGenerator({ + useExistingPosts: argv['use-existing-posts'], + useExistingTags: argv['use-existing-tags'], + knex, + schema, + logger: { + log: this.log, + ok: this.ok, + info: this.info, + warn: this.warn, + error: this.error, + fatal: this.fatal, + debug: this.debug + }, + modelQuantities: {} + }); + try { + await dataGenerator.importData(); + } catch (error) { + this.fatal('Failed while generating data: ', error); + } + knex.destroy(); + } +}; diff --git a/ghost/core/ghost.js b/ghost/core/ghost.js index 1b181dcc17..00e476f26e 100644 --- a/ghost/core/ghost.js +++ b/ghost/core/ghost.js @@ -18,6 +18,7 @@ const command = require('./core/cli/command'); switch (mode) { case 'repl': case 'timetravel': +case 'generate-data': command.run(mode); break; default: diff --git a/ghost/data-generator/.eslintrc.js b/ghost/data-generator/.eslintrc.js new file mode 100644 index 0000000000..c9c1bcb522 --- /dev/null +++ b/ghost/data-generator/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ] +}; diff --git a/ghost/data-generator/README.md b/ghost/data-generator/README.md new file mode 100644 index 0000000000..f36e880a8c --- /dev/null +++ b/ghost/data-generator/README.md @@ -0,0 +1,23 @@ +# Data Generator + +Generate fake data for testing Ghost + + +## Usage + + +## Develop + +This is a monorepo package. + +Follow the instructions for the top-level repo. +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies. + + + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests + diff --git a/ghost/data-generator/index.js b/ghost/data-generator/index.js new file mode 100644 index 0000000000..c26abb561d --- /dev/null +++ b/ghost/data-generator/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/data-generator'); diff --git a/ghost/data-generator/lib/data-generator.js b/ghost/data-generator/lib/data-generator.js new file mode 100644 index 0000000000..f19ffa5765 --- /dev/null +++ b/ghost/data-generator/lib/data-generator.js @@ -0,0 +1,236 @@ +const { + PostsImporter, + NewslettersImporter, + UsersImporter, + PostsAuthorsImporter, + TagsImporter, + PostsTagsImporter, + ProductsImporter, + MembersImporter, + BenefitsImporter, + ProductsBenefitsImporter, + MembersProductsImporter, + PostsProductsImporter, + MembersNewslettersImporter, + MembersCreatedEventsImporter, + MembersLoginEventsImporter, + MembersStatusEventsImporter, + StripeProductsImporter, + StripePricesImporter, + SubscriptionsImporter, + MembersStripeCustomersImporter, + MembersStripeCustomersSubscriptionsImporter, + MembersPaidSubscriptionEventsImporter, + MembersSubscriptionCreatedEventsImporter, + MembersSubscribeEventsImporter +} = require('./tables'); +const {faker} = require('@faker-js/faker'); + +/** + * @typedef {Object} DataGeneratorOptions + * @property {boolean} useExistingPosts + * @property {boolean} useExistingTags + * @property {import('knex/types').Knex} knex + * @property {Object} schema + * @property {Object} logger + * @property {Object} modelQuantities + */ + +const defaultQuantities = { + members: () => faker.datatype.number({ + min: 7000, + max: 8000 + }), + membersLoginEvents: 100, + posts: () => faker.datatype.number({ + min: 80, + max: 120 + }) +}; + +class DataGenerator { + /** + * + * @param {DataGeneratorOptions} options + */ + constructor({ + useExistingPosts = false, + useExistingTags = false, + knex, + schema, + logger, + modelQuantities = {} + }) { + this.useExistingPosts = useExistingPosts; + this.useExistingTags = useExistingTags; + this.knex = knex; + this.schema = schema; + this.logger = logger; + this.modelQuantities = Object.assign({}, defaultQuantities, modelQuantities); + } + + async importData() { + const transaction = await this.knex.transaction(); + + const newslettersImporter = new NewslettersImporter(transaction); + // First newsletter is free, second is paid + const newsletters = await newslettersImporter.import({amount: 2}); + + let posts = []; + const postsImporter = new PostsImporter(transaction, { + newsletters + }); + if (this.useExistingPosts) { + posts = await transaction.select('id').from('posts'); + postsImporter.addNewsletters({posts}); + posts = await transaction.select('id', 'newsletter_id').from('posts'); + } else { + posts = await postsImporter.import({ + amount: this.modelQuantities.posts, + rows: ['newsletter_id'] + }); + } + + const usersImporter = new UsersImporter(transaction); + const users = await usersImporter.import({amount: 8}); + + const postsAuthorsImporter = new PostsAuthorsImporter(transaction, { + users + }); + await postsAuthorsImporter.importForEach(posts, {amount: 1}); + + let tags = []; + if (this.useExistingTags) { + posts = await transaction.select('id').from('tags'); + } else { + const tagsImporter = new TagsImporter(transaction, { + users + }); + tags = await tagsImporter.import({amount: faker.datatype.number({ + min: 16, + max: 24 + })}); + } + + const postsTagsImporter = new PostsTagsImporter(transaction, { + tags + }); + await postsTagsImporter.importForEach(posts, { + amount: () => faker.datatype.number({ + min: 0, + max: 3 + }) + }); + + const productsImporter = new ProductsImporter(transaction); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const membersImporter = new MembersImporter(transaction); + const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email']}); + + const benefitsImporter = new BenefitsImporter(transaction); + const benefits = await benefitsImporter.import({amount: 5}); + + const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits}); + // Up to 5 benefits for each product + await productsBenefitsImporter.importForEach(products, {amount: 5}); + + // TODO: Use subscriptions to generate members_products table? + const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.slice(1)}); + const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), { + amount: 1, + rows: ['product_id', 'member_id'] + }); + const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: [products[0]]}); + await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), { + amount: 1, + rows: ['product_id', 'member_id'] + }); + + const postsProductsImporter = new PostsProductsImporter(transaction, {products}); + // Paid newsletters + await postsProductsImporter.importForEach(posts.filter(post => newsletters.findIndex(newsletter => newsletter.id === post.newsletter_id) === 1), { + // Each post is available on all 3 products + amount: 3 + }); + + const membersCreatedEventsImporter = new MembersCreatedEventsImporter(transaction); + await membersCreatedEventsImporter.importForEach(members, {amount: 1}); + + const membersLoginEventsImporter = new MembersLoginEventsImporter(transaction); + // Will create roughly 1 login event for every 3 days, up to a maximum of 100. + await membersLoginEventsImporter.importForEach(members, {amount: this.modelQuantities.membersLoginEvents}); + + const membersStatusEventsImporter = new MembersStatusEventsImporter(transaction); + // Up to 2 events per member - 1 from null -> free, 1 from free -> {paid, comped} + await membersStatusEventsImporter.importForEach(members, {amount: 2}); + + const stripeProductsImporter = new StripeProductsImporter(transaction); + const stripeProducts = await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const stripePricesImporter = new StripePricesImporter(transaction, {products}); + const stripePrices = await stripePricesImporter.importForEach(stripeProducts, { + amount: 2, + rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] + }); + + await productsImporter.addStripePrices({ + products, + stripeProducts, + stripePrices + }); + + const subscriptionsImporter = new SubscriptionsImporter(transaction, {members, stripeProducts, stripePrices}); + const subscriptions = await subscriptionsImporter.importForEach(membersProducts, { + amount: 1, + rows: ['cadence', 'tier_id', 'expires_at', 'created_at', 'member_id', 'currency'] + }); + + const membersStripeCustomersImporter = new MembersStripeCustomersImporter(transaction); + const membersStripeCustomers = await membersStripeCustomersImporter.importForEach(members, { + amount: 1, + rows: ['customer_id', 'member_id'] + }); + + const membersStripeCustomersSubscriptionsImporter = new MembersStripeCustomersSubscriptionsImporter(transaction, { + membersStripeCustomers, + products, + stripeProducts, + stripePrices + }); + const membersStripeCustomersSubscriptions = await membersStripeCustomersSubscriptionsImporter.importForEach(subscriptions, { + amount: 1, + rows: ['mrr', 'plan_id', 'subscription_id'] + }); + + const membersSubscribeEventsImporter = new MembersSubscribeEventsImporter(transaction, {newsletters, subscriptions}); + const membersSubscribeEvents = await membersSubscribeEventsImporter.importForEach(members, { + amount: 2, + rows: ['member_id', 'newsletter_id'] + }); + + const membersNewslettersImporter = new MembersNewslettersImporter(transaction); + await membersNewslettersImporter.importForEach(membersSubscribeEvents, {amount: 1}); + + const membersPaidSubscriptionEventsImporter = new MembersPaidSubscriptionEventsImporter(transaction, { + membersStripeCustomersSubscriptions + }); + await membersPaidSubscriptionEventsImporter.importForEach(subscriptions, {amount: 1}); + + const membersSubscriptionCreatedEventsImporter = new MembersSubscriptionCreatedEventsImporter(transaction, {subscriptions}); + await membersSubscriptionCreatedEventsImporter.importForEach(membersStripeCustomersSubscriptions, {amount: 1}); + + // TODO: Emails! (relies on posts & newsletters) + + // TODO: Email clicks - redirect, members_click_events (relies on emails) + + // TODO: Feedback - members_feedback (relies on members and posts) + + await transaction.commit(); + } +} + +module.exports = DataGenerator; diff --git a/ghost/data-generator/lib/tables/base.js b/ghost/data-generator/lib/tables/base.js new file mode 100644 index 0000000000..17aa476fc2 --- /dev/null +++ b/ghost/data-generator/lib/tables/base.js @@ -0,0 +1,88 @@ +class TableImporter { + /** + * @param {string} name Name of the table to be generated + * @param {import('knex/types').Knex} knex Database connection + */ + constructor(name, knex) { + this.name = name; + this.knex = knex; + } + + /** + * @typedef {Function} AmountFunction + * @returns {number} + */ + + /** + * @typedef {Object.} ImportOptions + * @property {number|AmountFunction} amount Number of events to generate + * @property {Object} [model] Used to reference another object during creation + */ + + /** + * @param {Array} models List of models to reference + * @param {ImportOptions} [options] Import options + * @returns {Promise>} + */ + async importForEach(models = [], options) { + const results = []; + for (const model of models) { + results.push(...await this.import(Object.assign({}, options, {model}))); + } + return results; + } + + /** + * @param {ImportOptions} options Import options + * @returns {Promise>} + */ + async import(options) { + if (options.amount === 0) { + return; + } + + // Use dynamic amount if faker function given + const amount = (typeof options.amount === 'function') ? options.amount() : options.amount; + + this.setImportOptions(Object.assign({}, options, {amount})); + + const data = []; + for (let i = 0; i < amount; i++) { + const model = this.generate(); + if (model) { + // Only push models when one is generated successfully + data.push(model); + } else { + // After first null assume that there is no more data + break; + } + } + + const rows = ['id']; + if (options && options.rows) { + rows.push(...options.rows); + } + return await this.knex.batchInsert(this.name, data, 500).returning(rows); + } + + /** + * + * @param {ImportOptions} options + * @returns {void} + */ + // eslint-disable-next-line no-unused-vars + setImportOptions(options) { + return; + } + + /** + * Generates the data for a single model to be imported + * @returns {Object|null} Data to import, optional + */ + generate() { + // Should never be called + return false; + } +} + +module.exports = TableImporter; diff --git a/ghost/data-generator/lib/tables/benefits.js b/ghost/data-generator/lib/tables/benefits.js new file mode 100644 index 0000000000..bd9061e90f --- /dev/null +++ b/ghost/data-generator/lib/tables/benefits.js @@ -0,0 +1,24 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {blogStartDate} = require('../utils/blog-info'); + +class BenefitsImporter extends TableImporter { + constructor(knex) { + super('benefits', knex); + } + + generate() { + const name = faker.company.catchPhrase(); + const sixMonthsLater = new Date(blogStartDate); + sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6); + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + created_at: faker.date.between(blogStartDate, sixMonthsLater) + }; + } +} + +module.exports = BenefitsImporter; diff --git a/ghost/data-generator/lib/tables/index.js b/ghost/data-generator/lib/tables/index.js new file mode 100644 index 0000000000..3fdcb0fb78 --- /dev/null +++ b/ghost/data-generator/lib/tables/index.js @@ -0,0 +1,51 @@ +const PostsImporter = require('./posts'); +const NewslettersImporter = require('./newsletters'); +const UsersImporter = require('./users'); +const PostsAuthorsImporter = require('./posts-authors'); +const TagsImporter = require('./tags'); +const PostsTagsImporter = require('./posts-tags'); +const ProductsImporter = require('./products'); +const MembersImporter = require('./members'); +const BenefitsImporter = require('./benefits'); +const ProductsBenefitsImporter = require('./products-benefits'); +const MembersProductsImporter = require('./members-products'); +const PostsProductsImporter = require('./posts-products'); +const MembersNewslettersImporter = require('./members-newsletters'); +const MembersCreatedEventsImporter = require('./members-created-events'); +const MembersLoginEventsImporter = require('./members-login-events'); +const MembersStatusEventsImporter = require('./members-status-events'); +const StripeProductsImporter = require('./stripe-products'); +const StripePricesImporter = require('./stripe-prices'); +const SubscriptionsImporter = require('./subscriptions'); +const MembersStripeCustomersImporter = require('./members-stripe-customers'); +const MembersStripeCustomersSubscriptionsImporter = require('./members-stripe-customers-subscriptions'); +const MembersPaidSubscriptionEventsImporter = require('./members-paid-subscription-events'); +const MembersSubscriptionCreatedEventsImporter = require('./members-subscription-created-events'); +const MembersSubscribeEventsImporter = require('./members-subscribe-events'); + +module.exports = { + PostsImporter, + NewslettersImporter, + UsersImporter, + PostsAuthorsImporter, + TagsImporter, + PostsTagsImporter, + ProductsImporter, + MembersImporter, + BenefitsImporter, + ProductsBenefitsImporter, + MembersProductsImporter, + PostsProductsImporter, + MembersNewslettersImporter, + MembersCreatedEventsImporter, + MembersLoginEventsImporter, + MembersStatusEventsImporter, + StripeProductsImporter, + StripePricesImporter, + SubscriptionsImporter, + MembersStripeCustomersImporter, + MembersStripeCustomersSubscriptionsImporter, + MembersPaidSubscriptionEventsImporter, + MembersSubscriptionCreatedEventsImporter, + MembersSubscribeEventsImporter +}; diff --git a/ghost/data-generator/lib/tables/members-created-events.js b/ghost/data-generator/lib/tables/members-created-events.js new file mode 100644 index 0000000000..8bf5bc3bfc --- /dev/null +++ b/ghost/data-generator/lib/tables/members-created-events.js @@ -0,0 +1,35 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); + +class MembersCreatedEventsImporter extends TableImporter { + constructor(knex) { + super('members_created_events', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generateSource() { + let source = 'member'; + if (luck(10)) { + source = 'admin'; + } else if (luck(5)) { + source = 'api'; + } + return source; + } + + generate() { + // TODO: Add attribution + return { + id: faker.database.mongodbObjectId(), + created_at: this.model.created_at, + member_id: this.model.id, + source: this.generateSource() + }; + } +} + +module.exports = MembersCreatedEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-login-events.js b/ghost/data-generator/lib/tables/members-login-events.js new file mode 100644 index 0000000000..d899c7eeb3 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-login-events.js @@ -0,0 +1,43 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); +const generateEvents = require('../utils/event-generator'); + +class MembersLoginEventsImporter extends TableImporter { + constructor(knex) { + super('members_login_events', knex); + } + + setImportOptions({model}) { + this.model = model; + const endDate = new Date(); + const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24)); + + // Assuming most people either subscribe and lose interest, or maintain steady readership + const shape = luck(40) ? 'ease-out' : 'flat'; + this.timestamps = generateEvents({ + shape, + trend: 'negative', + // Steady readers login more, readers who lose interest read less overall. + // ceil because members will all have logged in at least once + total: shape === 'flat' ? Math.ceil(daysBetween / 3) : Math.ceil(daysBetween / 7), + startTime: new Date(model.created_at), + endTime: endDate + }); + } + + generate() { + const timestamp = this.timestamps.shift(); + if (!timestamp) { + // Out of events for this user + return null; + } + return { + id: faker.database.mongodbObjectId(), + created_at: timestamp.toISOString(), + member_id: this.model.id + }; + } +} + +module.exports = MembersLoginEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-newsletters.js b/ghost/data-generator/lib/tables/members-newsletters.js new file mode 100644 index 0000000000..d6808d8ab1 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-newsletters.js @@ -0,0 +1,22 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersNewslettersImporter extends TableImporter { + constructor(knex) { + super('members_newsletters', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.member_id, + newsletter_id: this.model.newsletter_id + }; + } +} + +module.exports = MembersNewslettersImporter; diff --git a/ghost/data-generator/lib/tables/members-paid-subscription-events.js b/ghost/data-generator/lib/tables/members-paid-subscription-events.js new file mode 100644 index 0000000000..cd1ef37bfd --- /dev/null +++ b/ghost/data-generator/lib/tables/members-paid-subscription-events.js @@ -0,0 +1,39 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersPaidSubscriptionEventsImporter extends TableImporter { + constructor(knex, {membersStripeCustomersSubscriptions}) { + super('members_paid_subscription_events', knex); + this.membersStripeCustomersSubscriptions = membersStripeCustomersSubscriptions; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + if (!this.model.currency) { + // Not a paid subscription + return null; + } + // TODO: Implement upgrades + const membersStripeCustomersSubscription = this.membersStripeCustomersSubscriptions.find((m) => { + return m.subscription_id === this.model.id; + }); + return { + id: faker.database.mongodbObjectId(), + // TODO: Support expired / updated / cancelled events too + type: 'created', + member_id: this.model.member_id, + subscription_id: this.model.id, + from_plan: null, + to_plan: membersStripeCustomersSubscription.plan_id, + currency: this.model.currency, + source: 'stripe', + mrr_delta: membersStripeCustomersSubscription.mrr, + created_at: this.model.created_at + }; + } +} + +module.exports = MembersPaidSubscriptionEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-products.js b/ghost/data-generator/lib/tables/members-products.js new file mode 100644 index 0000000000..0e6ad4b1bf --- /dev/null +++ b/ghost/data-generator/lib/tables/members-products.js @@ -0,0 +1,35 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {luck} = require('../utils/random'); + +class MembersProductsImporter extends TableImporter { + constructor(knex, {products}) { + super('members_products', knex); + this.products = products; + } + + setImportOptions({model}) { + this.model = model; + } + + getProduct() { + if (this.products.length > 1) { + return luck(10) ? this.products[2] + : luck(50) ? this.products[1] + : this.products[0]; + } else { + return this.products[0]; + } + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + product_id: this.getProduct().id, + sort_order: 0 + }; + } +} + +module.exports = MembersProductsImporter; diff --git a/ghost/data-generator/lib/tables/members-status-events.js b/ghost/data-generator/lib/tables/members-status-events.js new file mode 100644 index 0000000000..76d18f68ff --- /dev/null +++ b/ghost/data-generator/lib/tables/members-status-events.js @@ -0,0 +1,37 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersStatusEventsImporter extends TableImporter { + constructor(knex) { + super('members_status_events', knex); + } + + setImportOptions({model}) { + this.events = [{ + id: faker.database.mongodbObjectId(), + member_id: model.id, + from_status: null, + to_status: 'free', + created_at: model.created_at + }]; + if (model.status !== 'free') { + this.events.push({ + id: faker.database.mongodbObjectId(), + member_id: model.id, + from_status: 'free', + to_status: model.status, + created_at: faker.date.between(new Date(model.created_at), new Date()).toISOString() + }); + } + } + + generate() { + const event = this.events.shift(); + if (!event) { + return null; + } + return event; + } +} + +module.exports = MembersStatusEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js b/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js new file mode 100644 index 0000000000..9c0e4491c2 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js @@ -0,0 +1,48 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersStripeCustomersSubscriptionsImporter extends TableImporter { + constructor(knex, {membersStripeCustomers, products, stripeProducts, stripePrices}) { + super('members_stripe_customers_subscriptions', knex); + this.membersStripeCustomers = membersStripeCustomers; + this.products = products; + this.stripeProducts = stripeProducts; + this.stripePrices = stripePrices; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const customer = this.membersStripeCustomers.find(c => this.model.member_id === c.member_id); + const isMonthly = this.model.cadence === 'month'; + const ghostProduct = this.products.find(product => product.id === this.model.tier_id); + const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.tier_id); + const stripePrice = this.stripePrices.find((price) => { + return price.stripe_product_id === stripeProduct.stripe_product_id && + (isMonthly ? price.interval === 'month' : price.interval === 'year'); + }); + const mrr = isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12); + return { + id: faker.database.mongodbObjectId(), + customer_id: customer.customer_id, + subscription_id: this.model.id, + stripe_price_id: stripePrice.stripe_price_id, + status: 'active', + cancel_at_period_end: false, + current_period_end: this.model.expires_at, + start_date: this.model.created_at, + created_at: this.model.created_at, + created_by: 'unused', + mrr, + plan_id: stripeProduct.stripe_product_id, + plan_nickname: `${ghostProduct.name} - ${stripePrice.nickname}`, + plan_interval: stripePrice.interval, + plan_amount: stripePrice.amount, + plan_currency: stripePrice.currency + }; + } +} + +module.exports = MembersStripeCustomersSubscriptionsImporter; diff --git a/ghost/data-generator/lib/tables/members-stripe-customers.js b/ghost/data-generator/lib/tables/members-stripe-customers.js new file mode 100644 index 0000000000..5722453597 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-stripe-customers.js @@ -0,0 +1,28 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersStripeCustomersImporter extends TableImporter { + constructor(knex) { + super('members_stripe_customers', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + customer_id: `cus_${faker.random.alphaNumeric(14, { + casing: 'mixed' + })}`, + name: this.model.name, + email: this.model.email, + created_at: this.model.created_at, + created_by: 'unused' + }; + } +} + +module.exports = MembersStripeCustomersImporter; diff --git a/ghost/data-generator/lib/tables/members-subscribe-events.js b/ghost/data-generator/lib/tables/members-subscribe-events.js new file mode 100644 index 0000000000..537f742736 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-subscribe-events.js @@ -0,0 +1,52 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); + +class MembersSubscribeEventsImporter extends TableImporter { + constructor(knex, {newsletters, subscriptions}) { + super('members_subscribe_events', knex); + this.newsletters = newsletters; + this.subscriptions = subscriptions; + } + + setImportOptions({model}) { + this.model = model; + this.count = 0; + } + + generate() { + const count = this.count; + this.count = this.count + 1; + + if (count === 1 && this.model.status === 'free') { + return null; + } + + let createdAt = faker.date.between(new Date(this.model.created_at), new Date()).toISOString(); + let subscribed = luck(80); + + // Free newsletter by default + let newsletterId = this.newsletters[0].id; + if (this.model.status === 'paid' && count === 0) { + // Paid newsletter + newsletterId = this.newsletters[1].id; + createdAt = this.subscriptions.find(s => s.member_id === this.model.id).created_at; + subscribed = luck(98); + } + + if (!subscribed) { + return null; + } + + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + newsletter_id: newsletterId, + subscribed: true, + created_at: createdAt, + source: 'member' + }; + } +} + +module.exports = MembersSubscribeEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-subscription-created-events.js b/ghost/data-generator/lib/tables/members-subscription-created-events.js new file mode 100644 index 0000000000..8e0f4c34e2 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-subscription-created-events.js @@ -0,0 +1,33 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersSubscriptionCreatedEventsImporter extends TableImporter { + constructor(knex, {subscriptions}) { + super('members_subscription_created_events', knex); + this.subscriptions = subscriptions; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const subscription = this.subscriptions.find(s => s.id === this.model.subscription_id); + return { + id: faker.database.mongodbObjectId(), + created_at: subscription.created_at, + member_id: subscription.member_id, + subscription_id: this.model.id, + // TODO: Implement attributions + attribution_id: null, + attribution_type: null, + attribution_url: null, + // TODO: Implement referrers + referrer_source: null, + referrer_medium: null, + referrer_url: null + }; + } +} + +module.exports = MembersSubscriptionCreatedEventsImporter; diff --git a/ghost/data-generator/lib/tables/members.js b/ghost/data-generator/lib/tables/members.js new file mode 100644 index 0000000000..a904c1ccba --- /dev/null +++ b/ghost/data-generator/lib/tables/members.js @@ -0,0 +1,65 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {faker: americanFaker} = require('@faker-js/faker/locale/en_US'); +const {blogStartDate: startTime} = require('../utils/blog-info'); +const generateEvents = require('../utils/event-generator'); +const {luck} = require('../utils/random'); + +class MembersImporter extends TableImporter { + constructor(knex) { + super('members', knex); + } + + setImportOptions({amount}) { + this.timestamps = generateEvents({ + shape: 'ease-in', + trend: 'positive', + total: amount, + startTime, + endTime: new Date() + }).sort(); + } + + generate() { + const id = faker.database.mongodbObjectId(); + // Use name from American locale to reflect an English-speaking audience + const name = `${americanFaker.name.firstName()} ${americanFaker.name.lastName()}`; + const timestamp = this.timestamps.shift(); + + return { + id, + uuid: faker.datatype.uuid(), + email: faker.internet.email(name, faker.date.birthdate().getFullYear().toString(), 'examplemail.com').toLowerCase(), + status: luck(5) ? 'comped' : luck(25) ? 'paid' : 'free', + name: name, + expertise: luck(30) ? faker.name.jobTitle() : undefined, + geolocation: JSON.stringify({ + organization_name: faker.company.name(), + region: faker.address.state(), + accuracy: 50, + asn: parseInt(faker.random.numeric(4)), + organization: `${faker.random.alpha({count: 2, casing: 'upper'})}${faker.random.numeric(4)} ${faker.company.name()}`, + timezone: faker.address.timeZone(), + longitude: faker.address.longitude(), + country_code3: faker.address.countryCode('alpha-3'), + area_code: '0', + ip: faker.internet.ipv4(), + city: faker.address.cityName(), + country: faker.address.country(), + continent_code: 'EU', + country_code: faker.address.countryCode('alpha-2'), + latitude: faker.address.latitude() + }), + email_count: 0, // Depends on number of emails sent since created_at, the newsletter they're a part of and subscription status + email_opened_count: 0, + email_open_rate: null, + // 40% of users logged in within a week, 60% sometime since registering + last_seen_at: luck(40) ? faker.date.recent(7).toISOString() : faker.date.between(timestamp, new Date()).toISOString(), + created_at: timestamp.toISOString(), + created_by: id, + updated_at: timestamp.toISOString() + }; + } +} + +module.exports = MembersImporter; diff --git a/ghost/data-generator/lib/tables/newsletters.js b/ghost/data-generator/lib/tables/newsletters.js new file mode 100644 index 0000000000..53d73690bf --- /dev/null +++ b/ghost/data-generator/lib/tables/newsletters.js @@ -0,0 +1,33 @@ +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); + +class NewslettersImporter extends TableImporter { + constructor(knex) { + super('newsletters', knex); + this.sortOrder = 0; + this.names = ['Occasional freebie', 'Regular premium']; + } + + generate() { + const name = this.names.shift(); + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + const weekAfter = new Date(blogStartDate); + weekAfter.setDate(weekAfter.getDate() + 7); + return { + id: faker.database.mongodbObjectId(), + uuid: faker.datatype.uuid(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + sender_reply_to: 'hello@example.com', + status: 'active', + subscribe_on_signup: faker.datatype.boolean(), + sort_order: sortOrder, + created_at: faker.date.between(blogStartDate, weekAfter) + }; + } +} + +module.exports = NewslettersImporter; diff --git a/ghost/data-generator/lib/tables/posts-authors.js b/ghost/data-generator/lib/tables/posts-authors.js new file mode 100644 index 0000000000..58679a883e --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-authors.js @@ -0,0 +1,27 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsAuthorsImporter extends TableImporter { + constructor(knex, {users}) { + super('posts_authors', knex); + this.users = users; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + author_id: this.users[faker.datatype.number(this.users.length - 1)].id, + sort_order: sortOrder + }; + } +} + +module.exports = PostsAuthorsImporter; diff --git a/ghost/data-generator/lib/tables/posts-products.js b/ghost/data-generator/lib/tables/posts-products.js new file mode 100644 index 0000000000..24576c12d2 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-products.js @@ -0,0 +1,27 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsProductsImporter extends TableImporter { + constructor(knex, {products}) { + super('posts_products', knex); + this.products = products; + } + + setImportOptions({model}) { + this.sortOrder = 0; + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + product_id: this.products[sortOrder].id, + sort_order: this.sortOrder + }; + } +} + +module.exports = PostsProductsImporter; diff --git a/ghost/data-generator/lib/tables/posts-tags.js b/ghost/data-generator/lib/tables/posts-tags.js new file mode 100644 index 0000000000..7ffb28d850 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-tags.js @@ -0,0 +1,35 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsTagsImporter extends TableImporter { + constructor(knex, {tags}) { + super('posts_tags', knex); + this.tags = tags; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.notIndex = []; + this.sortOrder = 0; + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + let tagIndex = 0; + do { + tagIndex = faker.datatype.number(this.tags.length - 1); + } while (this.notIndex.includes(tagIndex)); + this.notIndex.push(tagIndex); + + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + tag_id: this.tags[tagIndex].id, + sort_order: sortOrder + }; + } +} + +module.exports = PostsTagsImporter; diff --git a/ghost/data-generator/lib/tables/posts.js b/ghost/data-generator/lib/tables/posts.js new file mode 100644 index 0000000000..4d453df752 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts.js @@ -0,0 +1,66 @@ +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {luck} = require('../utils/random'); +const TableImporter = require('./base'); + +class PostsImporter extends TableImporter { + constructor(knex, {newsletters}) { + super('posts', knex); + this.newsletters = newsletters; + } + + async addNewsletters({posts}) { + for (const {id} of posts) { + await this.knex('posts').update({ + newsletter_id: luck(10) ? this.newsletters[0].id : this.newsletters[1].id + }).where({id}); + } + } + + generate() { + const title = faker.lorem.sentence(); + const content = faker.lorem.paragraphs(faker.datatype.number({ + min: 3, + max: 10 + })).split('\n'); + const twoYearsAgo = new Date(); + twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); + const twoWeeksAgo = new Date(); + twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14); + const timestamp = faker.date.between(twoYearsAgo, twoWeeksAgo); + return { + id: faker.database.mongodbObjectId(), + created_at: timestamp.toISOString(), + created_by: 'unused', + updated_at: timestamp.toISOString(), + published_at: faker.date.soon(5, timestamp).toISOString(), + uuid: faker.datatype.uuid(), + title: title, + slug: `${slugify(title)}-${faker.random.numeric(3)}`, + status: 'published', + mobiledoc: JSON.stringify({ + version: '0.3.1', + atoms: [], + cards: [], + markups: [['em']], + sections: content.map(paragraph => [ + 1, + 'p', + [ + [ + 0, + [], + 0, + paragraph + ] + ] + ]) + }), + html: content.map(paragraph => `

${paragraph}

`).join(''), + email_recipient_filter: 'all', + newsletter_id: luck(10) ? this.newsletters[0].id : this.newsletters[1].id + }; + } +} + +module.exports = PostsImporter; diff --git a/ghost/data-generator/lib/tables/products-benefits.js b/ghost/data-generator/lib/tables/products-benefits.js new file mode 100644 index 0000000000..c606a4f0ea --- /dev/null +++ b/ghost/data-generator/lib/tables/products-benefits.js @@ -0,0 +1,46 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class ProductsBenefitsImporter extends TableImporter { + constructor(knex, {benefits}) { + super('products_benefits', knex); + this.benefits = benefits; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.model = model; + this.sortOrder = 0; + switch (this.model.name) { + case 'Bronze': + this.benefitCount = 1; + break; + case 'Silver': + this.benefitCount = 3; + break; + case 'Gold': + this.benefitCount = 5; + break; + case 'Free Preview': + this.benefitCount = 0; + break; + } + } + + generate() { + if (this.sortOrder >= this.benefitCount) { + // No more benefits than benefitCount + return null; + } + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + product_id: this.model.id, + benefit_id: this.benefits[sortOrder].id, + sort_order: sortOrder + }; + } +} + +module.exports = ProductsBenefitsImporter; diff --git a/ghost/data-generator/lib/tables/products.js b/ghost/data-generator/lib/tables/products.js new file mode 100644 index 0000000000..1083a8738b --- /dev/null +++ b/ghost/data-generator/lib/tables/products.js @@ -0,0 +1,71 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {blogStartDate} = require('../utils/blog-info'); + +class ProductsImporter extends TableImporter { + constructor(knex) { + super('products', knex); + } + + setImportOptions() { + this.names = ['Free Preview', 'Bronze', 'Silver', 'Gold']; + this.count = 0; + } + + async addStripePrices({products, stripeProducts, stripePrices}) { + for (const {id} of products) { + const stripeProduct = stripeProducts.find(p => id === p.product_id); + const monthlyPrice = stripePrices.find((p) => { + return p.stripe_product_id === stripeProduct.stripe_product_id && + p.interval === 'monthly'; + }); + const yearlyPrice = stripePrices.find((p) => { + return p.stripe_product_id === stripeProduct.stripe_product_id && + p.interval === 'yearly'; + }); + + const update = {}; + if (monthlyPrice) { + update.monthly_price_id = monthlyPrice.id; + } + if (yearlyPrice) { + update.yearly_price_id = yearlyPrice.id; + } + + if (Object.keys(update).length > 0) { + await this.knex('products').update(update).where({ + id + }); + } + } + } + + generate() { + const name = this.names.shift(); + const count = this.count; + this.count = this.count + 1; + const sixMonthsLater = new Date(blogStartDate); + sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6); + const tierInfo = { + type: 'free', + description: 'A free sample of content' + }; + if (count !== 0) { + tierInfo.type = 'paid'; + tierInfo.description = `${name} star member`; + tierInfo.currency = 'USD'; + tierInfo.monthly_price = count * 500; + tierInfo.yearly_price = count * 5000; + } + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + visibility: 'public', + created_at: faker.date.between(blogStartDate, sixMonthsLater) + }, tierInfo); + } +} + +module.exports = ProductsImporter; diff --git a/ghost/data-generator/lib/tables/stripe-prices.js b/ghost/data-generator/lib/tables/stripe-prices.js new file mode 100644 index 0000000000..f4090d4cab --- /dev/null +++ b/ghost/data-generator/lib/tables/stripe-prices.js @@ -0,0 +1,61 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); + +class StripePricesImporter extends TableImporter { + constructor(knex, {products}) { + super('stripe_prices', knex); + this.products = products; + } + + setImportOptions({model}) { + this.model = model; + + this.count = 0; + } + + generate() { + const sixWeeksLater = new Date(blogStartDate); + sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + + const count = this.count; + this.count = this.count + 1; + + const relatedProduct = this.products.find(product => product.id === this.model.product_id); + + if (count === 1 && relatedProduct.monthly_price === null) { + // Only single complimentary price (yearly) + return null; + } + + const billingCycle = { + nickname: 'Monthly', + interval: 'month', + type: 'recurring', + currency: 'usd', + amount: relatedProduct.monthly_price + }; + if (count === 1) { + billingCycle.nickname = 'Yearly'; + billingCycle.interval = 'year'; + billingCycle.amount = relatedProduct.yearly_price; + } else if (relatedProduct.monthly_price === null) { + billingCycle.nickname = 'Complimentary'; + billingCycle.interval = 'year'; + billingCycle.amount = 0; + } + + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + stripe_price_id: faker.datatype.hexadecimal({ + length: 64, + prefix: '' + }), + stripe_product_id: this.model.stripe_product_id, + active: true, + created_at: faker.date.between(blogStartDate, sixWeeksLater) + }, billingCycle); + } +} + +module.exports = StripePricesImporter; diff --git a/ghost/data-generator/lib/tables/stripe-products.js b/ghost/data-generator/lib/tables/stripe-products.js new file mode 100644 index 0000000000..fd198a52b2 --- /dev/null +++ b/ghost/data-generator/lib/tables/stripe-products.js @@ -0,0 +1,29 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); + +class StripeProductsImporter extends TableImporter { + constructor(knex) { + super('stripe_products', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const sixWeeksLater = new Date(blogStartDate); + sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + return { + id: faker.database.mongodbObjectId(), + product_id: this.model.id, + stripe_product_id: faker.datatype.hexadecimal({ + length: 64, + prefix: '' + }), + created_at: faker.date.between(blogStartDate, sixWeeksLater) + }; + } +} + +module.exports = StripeProductsImporter; diff --git a/ghost/data-generator/lib/tables/subscriptions.js b/ghost/data-generator/lib/tables/subscriptions.js new file mode 100644 index 0000000000..8f63f33e6c --- /dev/null +++ b/ghost/data-generator/lib/tables/subscriptions.js @@ -0,0 +1,64 @@ +const {faker} = require('@faker-js/faker'); +const generateEvents = require('../utils/event-generator'); +const TableImporter = require('./base'); + +class SubscriptionsImporter extends TableImporter { + constructor(knex, {members, stripeProducts, stripePrices}) { + super('subscriptions', knex); + this.members = members; + this.stripeProducts = stripeProducts; + this.stripePrices = stripePrices; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const member = this.members.find(m => m.id === this.model.member_id); + const status = member.status; + const billingInfo = {}; + const isMonthly = faker.datatype.boolean(); + if (status === 'paid') { + const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.product_id); + const stripePrice = this.stripePrices.find((price) => { + return price.stripe_product_id === stripeProduct.stripe_product_id && + (isMonthly ? price.interval === 'month' : price.interval === 'year'); + }); + billingInfo.cadence = isMonthly ? 'month' : 'year'; + billingInfo.currency = stripePrice.currency; + billingInfo.amount = stripePrice.amount; + } + const [startDate] = generateEvents({ + total: 1, + trend: 'negative', + startTime: new Date(member.created_at), + endTime: new Date(), + shape: 'ease-out' + }); + const endDate = new Date(startDate); + if (isMonthly) { + endDate.setMonth(new Date().getMonth()); + if (endDate < new Date()) { + endDate.setMonth(endDate.getMonth() + 1); + } + } else { + endDate.setFullYear(new Date().getFullYear()); + if (endDate < new Date()) { + endDate.setFullYear(endDate.getFullYear() + 1); + } + } + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + type: status, + status: 'active', + member_id: this.model.member_id, + tier_id: this.model.product_id, + payment_provider: 'stripe', + expires_at: endDate.toISOString(), + created_at: startDate.toISOString() + }, billingInfo); + } +} + +module.exports = SubscriptionsImporter; diff --git a/ghost/data-generator/lib/tables/tags.js b/ghost/data-generator/lib/tables/tags.js new file mode 100644 index 0000000000..116ef0c0bf --- /dev/null +++ b/ghost/data-generator/lib/tables/tags.js @@ -0,0 +1,29 @@ +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const TableImporter = require('./base'); + +class TagsImporter extends TableImporter { + constructor(knex, {users}) { + super('tags', knex); + this.users = users; + } + + generate() { + let name = `${faker.color.human()} ${faker.name.jobType()}`; + name = `${name[0].toUpperCase()}${name.slice(1)}`; + const threeYearsAgo = new Date(); + threeYearsAgo.setFullYear(threeYearsAgo.getFullYear() - 3); + const twoYearsAgo = new Date(); + twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + description: faker.lorem.sentence(), + created_at: faker.date.between(threeYearsAgo, twoYearsAgo).toISOString(), + created_by: this.users[faker.datatype.number(this.users.length - 1)].id + }; + } +} + +module.exports = TagsImporter; diff --git a/ghost/data-generator/lib/tables/users.js b/ghost/data-generator/lib/tables/users.js new file mode 100644 index 0000000000..b7fea165d9 --- /dev/null +++ b/ghost/data-generator/lib/tables/users.js @@ -0,0 +1,26 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const security = require('@tryghost/security'); + +class UsersImporter extends TableImporter { + constructor(knex) { + super('users', knex); + } + + generate() { + const name = `${faker.name.firstName()} ${faker.name.lastName()}`; + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: slugify(name), + password: security.password.hash(faker.color.human()), + email: faker.internet.email(name), + profile_image: faker.internet.avatar(), + created_at: faker.date.between(new Date(2016, 0), new Date()).toISOString(), + created_by: 'unused' + }; + } +} + +module.exports = UsersImporter; diff --git a/ghost/data-generator/lib/utils/blog-info.js b/ghost/data-generator/lib/utils/blog-info.js new file mode 100644 index 0000000000..cf6ed68af6 --- /dev/null +++ b/ghost/data-generator/lib/utils/blog-info.js @@ -0,0 +1,3 @@ +module.exports = { + blogStartDate: new Date(2018, 5, 4) +}; diff --git a/ghost/data-generator/lib/utils/event-generator.js b/ghost/data-generator/lib/utils/event-generator.js new file mode 100644 index 0000000000..d72597004d --- /dev/null +++ b/ghost/data-generator/lib/utils/event-generator.js @@ -0,0 +1,43 @@ +const probabilityDistributions = require('probability-distributions'); + +const generateEvents = ({ + shape = 'flat', + trend = 'positive', + total = 0, + startTime = new Date(), + endTime = new Date() +} = {}) => { + let alpha = 0; + let beta = 0; + let positiveTrend = trend === 'positive'; + switch (shape) { + case 'linear': + alpha = 2; + beta = 1; + break; + case 'ease-in': + alpha = 4; + beta = 1; + break; + case 'ease-out': + alpha = 1; + beta = 4; + positiveTrend = !positiveTrend; + break; + case 'flat': + alpha = 1; + beta = 1; + break; + } + const data = probabilityDistributions.rbeta(total, alpha, beta, 0); + const startTimeValue = startTime.valueOf(); + const timeDifference = endTime.valueOf() - startTimeValue; + return data.map((x) => { + if (!positiveTrend) { + x = 1 - x; + } + return new Date(startTimeValue + timeDifference * x); + }); +}; + +module.exports = generateEvents; diff --git a/ghost/data-generator/lib/utils/random.js b/ghost/data-generator/lib/utils/random.js new file mode 100644 index 0000000000..49a9b34597 --- /dev/null +++ b/ghost/data-generator/lib/utils/random.js @@ -0,0 +1,13 @@ +const {faker} = require('@faker-js/faker'); + +/** + * Adds another degree of randomness into some decisions + * @param {number} lowerThan Only this % of people will achieve this luck + * @returns {boolean} Whether this person is lucky enough for the condition + */ +const luck = lowerThan => faker.datatype.number({ + min: 1, + max: 100 +}) <= lowerThan; + +module.exports = {luck}; diff --git a/ghost/data-generator/package.json b/ghost/data-generator/package.json new file mode 100644 index 0000000000..e1460b70e4 --- /dev/null +++ b/ghost/data-generator/package.json @@ -0,0 +1,32 @@ +{ + "name": "@tryghost/data-generator", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/data-generator", + "author": "Ghost Foundation", + "private": true, + "main": "index.js", + "scripts": { + "dev": "echo \"Implement me!\"", + "test:unit": "NODE_ENV=testing c8 --all --check-coverage --reporter text --reporter cobertura mocha './test/**/*.test.js'", + "test": "yarn test:unit", + "lint:code": "eslint *.js lib/ --ext .js --cache", + "lint": "yarn lint:code && yarn lint:test", + "lint:test": "eslint -c test/.eslintrc.js test/ --ext .js --cache" + }, + "files": [ + "index.js", + "lib" + ], + "devDependencies": { + "c8": "7.12.0", + "knex": "2.3.0", + "mocha": "10.1.0", + "should": "13.2.3", + "sinon": "14.0.1" + }, + "dependencies": { + "@faker-js/faker": "7.6.0", + "@tryghost/string": "0.2.1", + "probability-distributions": "0.9.1" + } +} diff --git a/ghost/data-generator/test/.eslintrc.js b/ghost/data-generator/test/.eslintrc.js new file mode 100644 index 0000000000..829b601eb0 --- /dev/null +++ b/ghost/data-generator/test/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/data-generator/test/data-generator.test.js b/ghost/data-generator/test/data-generator.test.js new file mode 100644 index 0000000000..acf7241881 --- /dev/null +++ b/ghost/data-generator/test/data-generator.test.js @@ -0,0 +1,249 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const knex = require('knex'); +const { + ProductsImporter, + StripeProductsImporter, + StripePricesImporter +} = require('../lib/tables'); + +const generateEvents = require('../lib/utils/event-generator'); + +const DataGenerator = require('../index'); + +const schema = require('../../core/core/server/data/schema'); + +describe('Data Generator', function () { + let db; + + beforeEach(async function () { + db = knex({ + client: 'sqlite3', + useNullAsDefault: true, + connection: { + filename: ':memory:' + } + }); + + for (const tableName of Object.keys(schema.tables)) { + await db.schema.createTable(tableName, function (table) { + for (const rowName of Object.keys(schema.tables[tableName])) { + const row = schema.tables[tableName][rowName]; + + if (rowName === '@@UNIQUE_CONSTRAINTS@@') { + for (const constraints of row) { + table.unique(constraints); + } + break; + } else if (rowName === '@@INDEXES@@') { + for (const indexes of row) { + table.index(indexes); + } + break; + } + + let rowChain = table[row.type.toLowerCase()](rowName); + if ('nullable' in row) { + if (row.nullable) { + rowChain = rowChain.nullable(); + } else { + rowChain = rowChain.notNullable(); + } + } + if ('defaultTo' in row) { + rowChain = rowChain.defaultTo(row.defaultTo); + } + if ('references' in row) { + const [foreignTable, foreignRow] = row.references.split('.'); + rowChain = rowChain.references(foreignRow).inTable(foreignTable); + } + if (row.unique) { + table.unique([rowName]); + } + if (row.primary) { + table.primary(rowName); + } + } + }); + } + }); + + afterEach(async function () { + await db.destroy(); + }); + + it('Can import the whole dataset without error', async function () { + const dataGenerator = new DataGenerator({ + eventsOnly: false, + knex: db, + schema: schema, + logger: {}, + modelQuantities: { + members: 10, + membersLoginEvents: 5, + posts: 2 + } + }); + try { + return await dataGenerator.importData(); + } catch (err) { + (false).should.eql(true, err.message); + } + }); +}); + +describe('Importer', function () { + let db; + + beforeEach(async function () { + db = knex({ + client: 'sqlite3', + useNullAsDefault: true, + connection: { + filename: ':memory:' + } + }); + + await db.schema.createTable('products', function (table) { + table.string('id'); + table.string('name'); + table.string('slug'); + table.string('visibility'); + table.date('created_at'); + table.string('type'); + table.string('description'); + table.string('currency'); + table.integer('monthly_price'); + table.integer('yearly_price'); + table.string('monthly_price_id'); + table.string('yearly_price_id'); + }); + + await db.schema.createTable('stripe_products', function (table) { + table.string('id'); + table.string('product_id'); + table.string('stripe_product_id'); + table.date('created_at'); + table.date('updated_at'); + }); + + await db.schema.createTable('stripe_prices', function (table) { + table.string('id'); + table.string('stripe_price_id'); + table.string('stripe_product_id'); + table.boolean('active'); + table.string('nickname'); + table.string('currency'); + table.integer('amount'); + table.string('type'); + table.string('interval'); + table.string('description'); + table.date('created_at'); + table.date('updated_at'); + }); + }); + + afterEach(async function () { + await db.destroy(); + }); + + it('Should import a single item', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 1, rows: ['name', 'monthly_price', 'yearly_price']}); + + products.length.should.eql(1); + products[0].name.should.eql('Free Preview'); + + const results = await db.select('id', 'name').from('products'); + + results.length.should.eql(1); + results[0].name.should.eql('Free Preview'); + }); + + it('Should import an item for each entry in an array', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const stripeProductsImporter = new StripeProductsImporter(db); + await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const results = await db.select('id').from('stripe_products'); + + results.length.should.eql(4); + }); + + it('Should update products to reference price ids', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const stripeProductsImporter = new StripeProductsImporter(db); + const stripeProducts = await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const stripePricesImporter = new StripePricesImporter(db, {products}); + const stripePrices = await stripePricesImporter.importForEach(stripeProducts, { + amount: 2, + rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] + }); + + await productsImporter.addStripePrices({ + products, + stripeProducts, + stripePrices + }); + + const results = await db.select('id', 'name', 'monthly_price_id', 'yearly_price_id').from('products'); + + results.length.should.eql(4); + results[0].name.should.eql('Free Preview'); + }); +}); + +describe('Events Generator', function () { + it('Generates a set of timestamps which meet the criteria', function () { + const startTime = new Date(); + startTime.setDate(startTime.getDate() - 30); + const endTime = new Date(); + const timestamps = generateEvents({ + shape: 'flat', + total: 100, + trend: 'positive', + startTime, + endTime + }); + + for (const timestamp of timestamps) { + timestamp.valueOf().should.be.lessThanOrEqual(endTime.valueOf()); + timestamp.valueOf().should.be.greaterThanOrEqual(startTime.valueOf()); + } + }); + + it('Works for a set of shapes', function () { + const startTime = new Date(); + startTime.setDate(startTime.getDate() - 30); + const endTime = new Date(); + + const options = { + startTime, + endTime, + total: 100, + trend: 'positive' + }; + + const shapes = ['linear', 'flat', 'ease-in', 'ease-out']; + + for (const shape of shapes) { + try { + generateEvents(Object.assign({}, options, {shape})); + } catch (err) { + (false).should.eql(true, err.message); + } + } + }); +}); diff --git a/ghost/data-generator/test/utils/assertions.js b/ghost/data-generator/test/utils/assertions.js new file mode 100644 index 0000000000..7364ee8aa1 --- /dev/null +++ b/ghost/data-generator/test/utils/assertions.js @@ -0,0 +1,11 @@ +/** + * Custom Should Assertions + * + * Add any custom assertions to this file. + */ + +// Example Assertion +// should.Assertion.add('ExampleAssertion', function () { +// this.params = {operator: 'to be a valid Example Assertion'}; +// this.obj.should.be.an.Object; +// }); diff --git a/ghost/data-generator/test/utils/index.js b/ghost/data-generator/test/utils/index.js new file mode 100644 index 0000000000..0d67d86ff8 --- /dev/null +++ b/ghost/data-generator/test/utils/index.js @@ -0,0 +1,11 @@ +/** + * Test Utilities + * + * Shared utils for writing tests + */ + +// Require overrides - these add globals for tests +require('./overrides'); + +// Require assertions - adds custom should assertions +require('./assertions'); diff --git a/ghost/data-generator/test/utils/overrides.js b/ghost/data-generator/test/utils/overrides.js new file mode 100644 index 0000000000..90203424ee --- /dev/null +++ b/ghost/data-generator/test/utils/overrides.js @@ -0,0 +1,10 @@ +// This file is required before any test is run + +// Taken from the should wiki, this is how to make should global +// Should is a global in our eslint test config +global.should = require('should').noConflict(); +should.extend(); + +// Sinon is a simple case +// Sinon is a global in our eslint test config +global.sinon = require('sinon'); diff --git a/yarn.lock b/yarn.lock index 5ba2e12ac9..0cff3f00ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2447,6 +2447,11 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@faker-js/faker@7.6.0": + version "7.6.0" + resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-7.6.0.tgz#9ea331766084288634a9247fcd8b84f16ff4ba07" + integrity sha512-XK6BTq1NDMo9Xqw/YkYyGjSsg44fbNwYRx7QK2CuoQgyy+f1rrTDHoExVM5PsyXCtfl2vs2vVJ0MN0yN6LppRw== + "@fortawesome/fontawesome-common-types@^0.2.36": version "0.2.36" resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-0.2.36.tgz#b44e52db3b6b20523e0c57ef8c42d315532cb903" @@ -9849,6 +9854,11 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== +crypto@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-0.0.3.tgz#470a81b86be4c5ee17acc8207a1f5315ae20dbb0" + integrity sha512-Q6Ka98WcvWXXg+9cnqd3jHpTSIOaH6/q0m/bESMfQo/0uFxy6e/7EqVS4JdaWx9qLdqV56tDufy2b12dj7BHJg== + css-blank-pseudo@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/css-blank-pseudo/-/css-blank-pseudo-3.0.3.tgz#36523b01c12a25d812df343a32c322d2a2324561" @@ -12147,7 +12157,6 @@ ember-power-calendar@^0.16.3: ember-power-datepicker@cibernox/ember-power-datepicker: version "0.8.1" - uid da580474a2c449b715444934ddb626b7c07f46a7 resolved "https://codeload.github.com/cibernox/ember-power-datepicker/tar.gz/da580474a2c449b715444934ddb626b7c07f46a7" dependencies: ember-basic-dropdown "^3.0.11" @@ -14723,7 +14732,6 @@ globrex@^0.1.2: "google-caja-bower@https://github.com/acburdine/google-caja-bower#ghost": version "6011.0.0" - uid "275cb75249f038492094a499756a73719ae071fd" resolved "https://github.com/acburdine/google-caja-bower#275cb75249f038492094a499756a73719ae071fd" got@9.6.0: @@ -17520,7 +17528,6 @@ keygrip@~1.1.0: "keymaster@https://github.com/madrobby/keymaster.git": version "1.6.3" - uid f8f43ddafad663b505dc0908e72853bcf8daea49 resolved "https://github.com/madrobby/keymaster.git#f8f43ddafad663b505dc0908e72853bcf8daea49" keypair@1.0.4: @@ -19368,7 +19375,6 @@ mocha@^2.5.3: mock-knex@TryGhost/mock-knex#8ecb8c227bf463c991c3d820d33f59efc3ab9682: version "0.4.9" - uid "8ecb8c227bf463c991c3d820d33f59efc3ab9682" resolved "https://codeload.github.com/TryGhost/mock-knex/tar.gz/8ecb8c227bf463c991c3d820d33f59efc3ab9682" dependencies: bluebird "^3.4.1" @@ -21905,6 +21911,13 @@ private@^0.1.6, private@^0.1.8: resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg== +probability-distributions@0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/probability-distributions/-/probability-distributions-0.9.1.tgz#c689c8d62173bb281a0999e98263acc76a628e55" + integrity sha512-guES77A321GzcK6swYmxhBeYQ10UNyiG0Gf4Ks+3239BuS27EFH/rrlfBW/oYlPyiufdey/3OlVqKvlXFB9LTQ== + dependencies: + crypto "0.0.3" + probe-image-size@7.2.3: version "7.2.3" resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.3.tgz#d49c64be540ec8edea538f6f585f65a9b3ab4309" @@ -23795,7 +23808,6 @@ simple-update-notifier@^1.0.7: "simplemde@https://github.com/kevinansfield/simplemde-markdown-editor.git#ghost": version "1.11.2" - uid "4c39702de7d97f9b32d5c101f39237b6dab7c3ee" resolved "https://github.com/kevinansfield/simplemde-markdown-editor.git#4c39702de7d97f9b32d5c101f39237b6dab7c3ee" sinon@14.0.1: