From 28b11e6fed3f7634f1e07ef3b6215221bf82a83a Mon Sep 17 00:00:00 2001 From: Sam Lord Date: Wed, 26 Oct 2022 17:55:08 +0100 Subject: [PATCH] Added command to generate demo data (#15691) refs: https://github.com/TryGhost/Toolbox/issues/440 New command to generate demo data, creates data for over 20 tables in Ghost, suitable for testing most features of the dashboard, as well as making guided product tours using newsletters, tiers, many posts and tags. Usage: `yarn start generate-data` Optionally, keep your existing posts / tags with: `yarn start generate-data --use-existing-tags --use-existing-posts` --- ghost/core/core/cli/generate-data.js | 50 ++++ ghost/core/ghost.js | 1 + ghost/data-generator/.eslintrc.js | 6 + ghost/data-generator/README.md | 23 ++ ghost/data-generator/index.js | 1 + ghost/data-generator/lib/data-generator.js | 236 +++++++++++++++++ ghost/data-generator/lib/tables/base.js | 88 +++++++ ghost/data-generator/lib/tables/benefits.js | 24 ++ ghost/data-generator/lib/tables/index.js | 51 ++++ .../lib/tables/members-created-events.js | 35 +++ .../lib/tables/members-login-events.js | 43 +++ .../lib/tables/members-newsletters.js | 22 ++ .../members-paid-subscription-events.js | 39 +++ .../lib/tables/members-products.js | 35 +++ .../lib/tables/members-status-events.js | 37 +++ .../members-stripe-customers-subscriptions.js | 48 ++++ .../lib/tables/members-stripe-customers.js | 28 ++ .../lib/tables/members-subscribe-events.js | 52 ++++ .../members-subscription-created-events.js | 33 +++ ghost/data-generator/lib/tables/members.js | 65 +++++ .../data-generator/lib/tables/newsletters.js | 33 +++ .../lib/tables/posts-authors.js | 27 ++ .../lib/tables/posts-products.js | 27 ++ ghost/data-generator/lib/tables/posts-tags.js | 35 +++ ghost/data-generator/lib/tables/posts.js | 66 +++++ .../lib/tables/products-benefits.js | 46 ++++ ghost/data-generator/lib/tables/products.js | 71 +++++ .../lib/tables/stripe-prices.js | 61 +++++ .../lib/tables/stripe-products.js | 29 ++ .../lib/tables/subscriptions.js | 64 +++++ ghost/data-generator/lib/tables/tags.js | 29 ++ ghost/data-generator/lib/tables/users.js | 26 ++ ghost/data-generator/lib/utils/blog-info.js | 3 + .../lib/utils/event-generator.js | 43 +++ ghost/data-generator/lib/utils/random.js | 13 + ghost/data-generator/package.json | 32 +++ ghost/data-generator/test/.eslintrc.js | 6 + .../test/data-generator.test.js | 249 ++++++++++++++++++ ghost/data-generator/test/utils/assertions.js | 11 + ghost/data-generator/test/utils/index.js | 11 + ghost/data-generator/test/utils/overrides.js | 10 + yarn.lock | 22 +- 42 files changed, 1826 insertions(+), 5 deletions(-) create mode 100644 ghost/core/core/cli/generate-data.js create mode 100644 ghost/data-generator/.eslintrc.js create mode 100644 ghost/data-generator/README.md create mode 100644 ghost/data-generator/index.js create mode 100644 ghost/data-generator/lib/data-generator.js create mode 100644 ghost/data-generator/lib/tables/base.js create mode 100644 ghost/data-generator/lib/tables/benefits.js create mode 100644 ghost/data-generator/lib/tables/index.js create mode 100644 ghost/data-generator/lib/tables/members-created-events.js create mode 100644 ghost/data-generator/lib/tables/members-login-events.js create mode 100644 ghost/data-generator/lib/tables/members-newsletters.js create mode 100644 ghost/data-generator/lib/tables/members-paid-subscription-events.js create mode 100644 ghost/data-generator/lib/tables/members-products.js create mode 100644 ghost/data-generator/lib/tables/members-status-events.js create mode 100644 ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js create mode 100644 ghost/data-generator/lib/tables/members-stripe-customers.js create mode 100644 ghost/data-generator/lib/tables/members-subscribe-events.js create mode 100644 ghost/data-generator/lib/tables/members-subscription-created-events.js create mode 100644 ghost/data-generator/lib/tables/members.js create mode 100644 ghost/data-generator/lib/tables/newsletters.js create mode 100644 ghost/data-generator/lib/tables/posts-authors.js create mode 100644 ghost/data-generator/lib/tables/posts-products.js create mode 100644 ghost/data-generator/lib/tables/posts-tags.js create mode 100644 ghost/data-generator/lib/tables/posts.js create mode 100644 ghost/data-generator/lib/tables/products-benefits.js create mode 100644 ghost/data-generator/lib/tables/products.js create mode 100644 ghost/data-generator/lib/tables/stripe-prices.js create mode 100644 ghost/data-generator/lib/tables/stripe-products.js create mode 100644 ghost/data-generator/lib/tables/subscriptions.js create mode 100644 ghost/data-generator/lib/tables/tags.js create mode 100644 ghost/data-generator/lib/tables/users.js create mode 100644 ghost/data-generator/lib/utils/blog-info.js create mode 100644 ghost/data-generator/lib/utils/event-generator.js create mode 100644 ghost/data-generator/lib/utils/random.js create mode 100644 ghost/data-generator/package.json create mode 100644 ghost/data-generator/test/.eslintrc.js create mode 100644 ghost/data-generator/test/data-generator.test.js create mode 100644 ghost/data-generator/test/utils/assertions.js create mode 100644 ghost/data-generator/test/utils/index.js create mode 100644 ghost/data-generator/test/utils/overrides.js diff --git a/ghost/core/core/cli/generate-data.js b/ghost/core/core/cli/generate-data.js new file mode 100644 index 0000000000..e4771784b2 --- /dev/null +++ b/ghost/core/core/cli/generate-data.js @@ -0,0 +1,50 @@ +const Command = require('./command'); +const DataGenerator = require('@tryghost/data-generator'); + +module.exports = class REPL extends Command { + setup() { + this.help('Generates random data to populate the database for development & testing'); + this.argument('--events-only', {type: 'boolean', defaultValue: false, desc: 'Only generate events, skip other datatypes'}); + this.argument('--use-existing-posts', {type: 'boolean', defaultValue: false, desc: 'Generate data referencing the set of existing posts'}); + this.argument('--use-existing-tags', {type: 'boolean', defaultValue: false, desc: 'Generate data referencing the set of existing tags'}); + } + + initializeContext(context) { + const models = require('../server/models'); + const knex = require('../server/data/db/connection'); + + models.init(); + + context.models = models; + context.m = models; + context.knex = knex; + context.k = knex; + } + + async handle(argv = {}) { + const knex = require('../server/data/db/connection'); + const {tables: schema} = require('../server/data/schema/index'); + const dataGenerator = new DataGenerator({ + useExistingPosts: argv['use-existing-posts'], + useExistingTags: argv['use-existing-tags'], + knex, + schema, + logger: { + log: this.log, + ok: this.ok, + info: this.info, + warn: this.warn, + error: this.error, + fatal: this.fatal, + debug: this.debug + }, + modelQuantities: {} + }); + try { + await dataGenerator.importData(); + } catch (error) { + this.fatal('Failed while generating data: ', error); + } + knex.destroy(); + } +}; diff --git a/ghost/core/ghost.js b/ghost/core/ghost.js index 1b181dcc17..00e476f26e 100644 --- a/ghost/core/ghost.js +++ b/ghost/core/ghost.js @@ -18,6 +18,7 @@ const command = require('./core/cli/command'); switch (mode) { case 'repl': case 'timetravel': +case 'generate-data': command.run(mode); break; default: diff --git a/ghost/data-generator/.eslintrc.js b/ghost/data-generator/.eslintrc.js new file mode 100644 index 0000000000..c9c1bcb522 --- /dev/null +++ b/ghost/data-generator/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ] +}; diff --git a/ghost/data-generator/README.md b/ghost/data-generator/README.md new file mode 100644 index 0000000000..f36e880a8c --- /dev/null +++ b/ghost/data-generator/README.md @@ -0,0 +1,23 @@ +# Data Generator + +Generate fake data for testing Ghost + + +## Usage + + +## Develop + +This is a monorepo package. + +Follow the instructions for the top-level repo. +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies. + + + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests + diff --git a/ghost/data-generator/index.js b/ghost/data-generator/index.js new file mode 100644 index 0000000000..c26abb561d --- /dev/null +++ b/ghost/data-generator/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/data-generator'); diff --git a/ghost/data-generator/lib/data-generator.js b/ghost/data-generator/lib/data-generator.js new file mode 100644 index 0000000000..f19ffa5765 --- /dev/null +++ b/ghost/data-generator/lib/data-generator.js @@ -0,0 +1,236 @@ +const { + PostsImporter, + NewslettersImporter, + UsersImporter, + PostsAuthorsImporter, + TagsImporter, + PostsTagsImporter, + ProductsImporter, + MembersImporter, + BenefitsImporter, + ProductsBenefitsImporter, + MembersProductsImporter, + PostsProductsImporter, + MembersNewslettersImporter, + MembersCreatedEventsImporter, + MembersLoginEventsImporter, + MembersStatusEventsImporter, + StripeProductsImporter, + StripePricesImporter, + SubscriptionsImporter, + MembersStripeCustomersImporter, + MembersStripeCustomersSubscriptionsImporter, + MembersPaidSubscriptionEventsImporter, + MembersSubscriptionCreatedEventsImporter, + MembersSubscribeEventsImporter +} = require('./tables'); +const {faker} = require('@faker-js/faker'); + +/** + * @typedef {Object} DataGeneratorOptions + * @property {boolean} useExistingPosts + * @property {boolean} useExistingTags + * @property {import('knex/types').Knex} knex + * @property {Object} schema + * @property {Object} logger + * @property {Object} modelQuantities + */ + +const defaultQuantities = { + members: () => faker.datatype.number({ + min: 7000, + max: 8000 + }), + membersLoginEvents: 100, + posts: () => faker.datatype.number({ + min: 80, + max: 120 + }) +}; + +class DataGenerator { + /** + * + * @param {DataGeneratorOptions} options + */ + constructor({ + useExistingPosts = false, + useExistingTags = false, + knex, + schema, + logger, + modelQuantities = {} + }) { + this.useExistingPosts = useExistingPosts; + this.useExistingTags = useExistingTags; + this.knex = knex; + this.schema = schema; + this.logger = logger; + this.modelQuantities = Object.assign({}, defaultQuantities, modelQuantities); + } + + async importData() { + const transaction = await this.knex.transaction(); + + const newslettersImporter = new NewslettersImporter(transaction); + // First newsletter is free, second is paid + const newsletters = await newslettersImporter.import({amount: 2}); + + let posts = []; + const postsImporter = new PostsImporter(transaction, { + newsletters + }); + if (this.useExistingPosts) { + posts = await transaction.select('id').from('posts'); + postsImporter.addNewsletters({posts}); + posts = await transaction.select('id', 'newsletter_id').from('posts'); + } else { + posts = await postsImporter.import({ + amount: this.modelQuantities.posts, + rows: ['newsletter_id'] + }); + } + + const usersImporter = new UsersImporter(transaction); + const users = await usersImporter.import({amount: 8}); + + const postsAuthorsImporter = new PostsAuthorsImporter(transaction, { + users + }); + await postsAuthorsImporter.importForEach(posts, {amount: 1}); + + let tags = []; + if (this.useExistingTags) { + posts = await transaction.select('id').from('tags'); + } else { + const tagsImporter = new TagsImporter(transaction, { + users + }); + tags = await tagsImporter.import({amount: faker.datatype.number({ + min: 16, + max: 24 + })}); + } + + const postsTagsImporter = new PostsTagsImporter(transaction, { + tags + }); + await postsTagsImporter.importForEach(posts, { + amount: () => faker.datatype.number({ + min: 0, + max: 3 + }) + }); + + const productsImporter = new ProductsImporter(transaction); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const membersImporter = new MembersImporter(transaction); + const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email']}); + + const benefitsImporter = new BenefitsImporter(transaction); + const benefits = await benefitsImporter.import({amount: 5}); + + const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits}); + // Up to 5 benefits for each product + await productsBenefitsImporter.importForEach(products, {amount: 5}); + + // TODO: Use subscriptions to generate members_products table? + const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.slice(1)}); + const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), { + amount: 1, + rows: ['product_id', 'member_id'] + }); + const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: [products[0]]}); + await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), { + amount: 1, + rows: ['product_id', 'member_id'] + }); + + const postsProductsImporter = new PostsProductsImporter(transaction, {products}); + // Paid newsletters + await postsProductsImporter.importForEach(posts.filter(post => newsletters.findIndex(newsletter => newsletter.id === post.newsletter_id) === 1), { + // Each post is available on all 3 products + amount: 3 + }); + + const membersCreatedEventsImporter = new MembersCreatedEventsImporter(transaction); + await membersCreatedEventsImporter.importForEach(members, {amount: 1}); + + const membersLoginEventsImporter = new MembersLoginEventsImporter(transaction); + // Will create roughly 1 login event for every 3 days, up to a maximum of 100. + await membersLoginEventsImporter.importForEach(members, {amount: this.modelQuantities.membersLoginEvents}); + + const membersStatusEventsImporter = new MembersStatusEventsImporter(transaction); + // Up to 2 events per member - 1 from null -> free, 1 from free -> {paid, comped} + await membersStatusEventsImporter.importForEach(members, {amount: 2}); + + const stripeProductsImporter = new StripeProductsImporter(transaction); + const stripeProducts = await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const stripePricesImporter = new StripePricesImporter(transaction, {products}); + const stripePrices = await stripePricesImporter.importForEach(stripeProducts, { + amount: 2, + rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] + }); + + await productsImporter.addStripePrices({ + products, + stripeProducts, + stripePrices + }); + + const subscriptionsImporter = new SubscriptionsImporter(transaction, {members, stripeProducts, stripePrices}); + const subscriptions = await subscriptionsImporter.importForEach(membersProducts, { + amount: 1, + rows: ['cadence', 'tier_id', 'expires_at', 'created_at', 'member_id', 'currency'] + }); + + const membersStripeCustomersImporter = new MembersStripeCustomersImporter(transaction); + const membersStripeCustomers = await membersStripeCustomersImporter.importForEach(members, { + amount: 1, + rows: ['customer_id', 'member_id'] + }); + + const membersStripeCustomersSubscriptionsImporter = new MembersStripeCustomersSubscriptionsImporter(transaction, { + membersStripeCustomers, + products, + stripeProducts, + stripePrices + }); + const membersStripeCustomersSubscriptions = await membersStripeCustomersSubscriptionsImporter.importForEach(subscriptions, { + amount: 1, + rows: ['mrr', 'plan_id', 'subscription_id'] + }); + + const membersSubscribeEventsImporter = new MembersSubscribeEventsImporter(transaction, {newsletters, subscriptions}); + const membersSubscribeEvents = await membersSubscribeEventsImporter.importForEach(members, { + amount: 2, + rows: ['member_id', 'newsletter_id'] + }); + + const membersNewslettersImporter = new MembersNewslettersImporter(transaction); + await membersNewslettersImporter.importForEach(membersSubscribeEvents, {amount: 1}); + + const membersPaidSubscriptionEventsImporter = new MembersPaidSubscriptionEventsImporter(transaction, { + membersStripeCustomersSubscriptions + }); + await membersPaidSubscriptionEventsImporter.importForEach(subscriptions, {amount: 1}); + + const membersSubscriptionCreatedEventsImporter = new MembersSubscriptionCreatedEventsImporter(transaction, {subscriptions}); + await membersSubscriptionCreatedEventsImporter.importForEach(membersStripeCustomersSubscriptions, {amount: 1}); + + // TODO: Emails! (relies on posts & newsletters) + + // TODO: Email clicks - redirect, members_click_events (relies on emails) + + // TODO: Feedback - members_feedback (relies on members and posts) + + await transaction.commit(); + } +} + +module.exports = DataGenerator; diff --git a/ghost/data-generator/lib/tables/base.js b/ghost/data-generator/lib/tables/base.js new file mode 100644 index 0000000000..17aa476fc2 --- /dev/null +++ b/ghost/data-generator/lib/tables/base.js @@ -0,0 +1,88 @@ +class TableImporter { + /** + * @param {string} name Name of the table to be generated + * @param {import('knex/types').Knex} knex Database connection + */ + constructor(name, knex) { + this.name = name; + this.knex = knex; + } + + /** + * @typedef {Function} AmountFunction + * @returns {number} + */ + + /** + * @typedef {Object.} ImportOptions + * @property {number|AmountFunction} amount Number of events to generate + * @property {Object} [model] Used to reference another object during creation + */ + + /** + * @param {Array} models List of models to reference + * @param {ImportOptions} [options] Import options + * @returns {Promise>} + */ + async importForEach(models = [], options) { + const results = []; + for (const model of models) { + results.push(...await this.import(Object.assign({}, options, {model}))); + } + return results; + } + + /** + * @param {ImportOptions} options Import options + * @returns {Promise>} + */ + async import(options) { + if (options.amount === 0) { + return; + } + + // Use dynamic amount if faker function given + const amount = (typeof options.amount === 'function') ? options.amount() : options.amount; + + this.setImportOptions(Object.assign({}, options, {amount})); + + const data = []; + for (let i = 0; i < amount; i++) { + const model = this.generate(); + if (model) { + // Only push models when one is generated successfully + data.push(model); + } else { + // After first null assume that there is no more data + break; + } + } + + const rows = ['id']; + if (options && options.rows) { + rows.push(...options.rows); + } + return await this.knex.batchInsert(this.name, data, 500).returning(rows); + } + + /** + * + * @param {ImportOptions} options + * @returns {void} + */ + // eslint-disable-next-line no-unused-vars + setImportOptions(options) { + return; + } + + /** + * Generates the data for a single model to be imported + * @returns {Object|null} Data to import, optional + */ + generate() { + // Should never be called + return false; + } +} + +module.exports = TableImporter; diff --git a/ghost/data-generator/lib/tables/benefits.js b/ghost/data-generator/lib/tables/benefits.js new file mode 100644 index 0000000000..bd9061e90f --- /dev/null +++ b/ghost/data-generator/lib/tables/benefits.js @@ -0,0 +1,24 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {blogStartDate} = require('../utils/blog-info'); + +class BenefitsImporter extends TableImporter { + constructor(knex) { + super('benefits', knex); + } + + generate() { + const name = faker.company.catchPhrase(); + const sixMonthsLater = new Date(blogStartDate); + sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6); + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + created_at: faker.date.between(blogStartDate, sixMonthsLater) + }; + } +} + +module.exports = BenefitsImporter; diff --git a/ghost/data-generator/lib/tables/index.js b/ghost/data-generator/lib/tables/index.js new file mode 100644 index 0000000000..3fdcb0fb78 --- /dev/null +++ b/ghost/data-generator/lib/tables/index.js @@ -0,0 +1,51 @@ +const PostsImporter = require('./posts'); +const NewslettersImporter = require('./newsletters'); +const UsersImporter = require('./users'); +const PostsAuthorsImporter = require('./posts-authors'); +const TagsImporter = require('./tags'); +const PostsTagsImporter = require('./posts-tags'); +const ProductsImporter = require('./products'); +const MembersImporter = require('./members'); +const BenefitsImporter = require('./benefits'); +const ProductsBenefitsImporter = require('./products-benefits'); +const MembersProductsImporter = require('./members-products'); +const PostsProductsImporter = require('./posts-products'); +const MembersNewslettersImporter = require('./members-newsletters'); +const MembersCreatedEventsImporter = require('./members-created-events'); +const MembersLoginEventsImporter = require('./members-login-events'); +const MembersStatusEventsImporter = require('./members-status-events'); +const StripeProductsImporter = require('./stripe-products'); +const StripePricesImporter = require('./stripe-prices'); +const SubscriptionsImporter = require('./subscriptions'); +const MembersStripeCustomersImporter = require('./members-stripe-customers'); +const MembersStripeCustomersSubscriptionsImporter = require('./members-stripe-customers-subscriptions'); +const MembersPaidSubscriptionEventsImporter = require('./members-paid-subscription-events'); +const MembersSubscriptionCreatedEventsImporter = require('./members-subscription-created-events'); +const MembersSubscribeEventsImporter = require('./members-subscribe-events'); + +module.exports = { + PostsImporter, + NewslettersImporter, + UsersImporter, + PostsAuthorsImporter, + TagsImporter, + PostsTagsImporter, + ProductsImporter, + MembersImporter, + BenefitsImporter, + ProductsBenefitsImporter, + MembersProductsImporter, + PostsProductsImporter, + MembersNewslettersImporter, + MembersCreatedEventsImporter, + MembersLoginEventsImporter, + MembersStatusEventsImporter, + StripeProductsImporter, + StripePricesImporter, + SubscriptionsImporter, + MembersStripeCustomersImporter, + MembersStripeCustomersSubscriptionsImporter, + MembersPaidSubscriptionEventsImporter, + MembersSubscriptionCreatedEventsImporter, + MembersSubscribeEventsImporter +}; diff --git a/ghost/data-generator/lib/tables/members-created-events.js b/ghost/data-generator/lib/tables/members-created-events.js new file mode 100644 index 0000000000..8bf5bc3bfc --- /dev/null +++ b/ghost/data-generator/lib/tables/members-created-events.js @@ -0,0 +1,35 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); + +class MembersCreatedEventsImporter extends TableImporter { + constructor(knex) { + super('members_created_events', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generateSource() { + let source = 'member'; + if (luck(10)) { + source = 'admin'; + } else if (luck(5)) { + source = 'api'; + } + return source; + } + + generate() { + // TODO: Add attribution + return { + id: faker.database.mongodbObjectId(), + created_at: this.model.created_at, + member_id: this.model.id, + source: this.generateSource() + }; + } +} + +module.exports = MembersCreatedEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-login-events.js b/ghost/data-generator/lib/tables/members-login-events.js new file mode 100644 index 0000000000..d899c7eeb3 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-login-events.js @@ -0,0 +1,43 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); +const generateEvents = require('../utils/event-generator'); + +class MembersLoginEventsImporter extends TableImporter { + constructor(knex) { + super('members_login_events', knex); + } + + setImportOptions({model}) { + this.model = model; + const endDate = new Date(); + const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24)); + + // Assuming most people either subscribe and lose interest, or maintain steady readership + const shape = luck(40) ? 'ease-out' : 'flat'; + this.timestamps = generateEvents({ + shape, + trend: 'negative', + // Steady readers login more, readers who lose interest read less overall. + // ceil because members will all have logged in at least once + total: shape === 'flat' ? Math.ceil(daysBetween / 3) : Math.ceil(daysBetween / 7), + startTime: new Date(model.created_at), + endTime: endDate + }); + } + + generate() { + const timestamp = this.timestamps.shift(); + if (!timestamp) { + // Out of events for this user + return null; + } + return { + id: faker.database.mongodbObjectId(), + created_at: timestamp.toISOString(), + member_id: this.model.id + }; + } +} + +module.exports = MembersLoginEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-newsletters.js b/ghost/data-generator/lib/tables/members-newsletters.js new file mode 100644 index 0000000000..d6808d8ab1 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-newsletters.js @@ -0,0 +1,22 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersNewslettersImporter extends TableImporter { + constructor(knex) { + super('members_newsletters', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.member_id, + newsletter_id: this.model.newsletter_id + }; + } +} + +module.exports = MembersNewslettersImporter; diff --git a/ghost/data-generator/lib/tables/members-paid-subscription-events.js b/ghost/data-generator/lib/tables/members-paid-subscription-events.js new file mode 100644 index 0000000000..cd1ef37bfd --- /dev/null +++ b/ghost/data-generator/lib/tables/members-paid-subscription-events.js @@ -0,0 +1,39 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersPaidSubscriptionEventsImporter extends TableImporter { + constructor(knex, {membersStripeCustomersSubscriptions}) { + super('members_paid_subscription_events', knex); + this.membersStripeCustomersSubscriptions = membersStripeCustomersSubscriptions; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + if (!this.model.currency) { + // Not a paid subscription + return null; + } + // TODO: Implement upgrades + const membersStripeCustomersSubscription = this.membersStripeCustomersSubscriptions.find((m) => { + return m.subscription_id === this.model.id; + }); + return { + id: faker.database.mongodbObjectId(), + // TODO: Support expired / updated / cancelled events too + type: 'created', + member_id: this.model.member_id, + subscription_id: this.model.id, + from_plan: null, + to_plan: membersStripeCustomersSubscription.plan_id, + currency: this.model.currency, + source: 'stripe', + mrr_delta: membersStripeCustomersSubscription.mrr, + created_at: this.model.created_at + }; + } +} + +module.exports = MembersPaidSubscriptionEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-products.js b/ghost/data-generator/lib/tables/members-products.js new file mode 100644 index 0000000000..0e6ad4b1bf --- /dev/null +++ b/ghost/data-generator/lib/tables/members-products.js @@ -0,0 +1,35 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {luck} = require('../utils/random'); + +class MembersProductsImporter extends TableImporter { + constructor(knex, {products}) { + super('members_products', knex); + this.products = products; + } + + setImportOptions({model}) { + this.model = model; + } + + getProduct() { + if (this.products.length > 1) { + return luck(10) ? this.products[2] + : luck(50) ? this.products[1] + : this.products[0]; + } else { + return this.products[0]; + } + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + product_id: this.getProduct().id, + sort_order: 0 + }; + } +} + +module.exports = MembersProductsImporter; diff --git a/ghost/data-generator/lib/tables/members-status-events.js b/ghost/data-generator/lib/tables/members-status-events.js new file mode 100644 index 0000000000..76d18f68ff --- /dev/null +++ b/ghost/data-generator/lib/tables/members-status-events.js @@ -0,0 +1,37 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersStatusEventsImporter extends TableImporter { + constructor(knex) { + super('members_status_events', knex); + } + + setImportOptions({model}) { + this.events = [{ + id: faker.database.mongodbObjectId(), + member_id: model.id, + from_status: null, + to_status: 'free', + created_at: model.created_at + }]; + if (model.status !== 'free') { + this.events.push({ + id: faker.database.mongodbObjectId(), + member_id: model.id, + from_status: 'free', + to_status: model.status, + created_at: faker.date.between(new Date(model.created_at), new Date()).toISOString() + }); + } + } + + generate() { + const event = this.events.shift(); + if (!event) { + return null; + } + return event; + } +} + +module.exports = MembersStatusEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js b/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js new file mode 100644 index 0000000000..9c0e4491c2 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js @@ -0,0 +1,48 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersStripeCustomersSubscriptionsImporter extends TableImporter { + constructor(knex, {membersStripeCustomers, products, stripeProducts, stripePrices}) { + super('members_stripe_customers_subscriptions', knex); + this.membersStripeCustomers = membersStripeCustomers; + this.products = products; + this.stripeProducts = stripeProducts; + this.stripePrices = stripePrices; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const customer = this.membersStripeCustomers.find(c => this.model.member_id === c.member_id); + const isMonthly = this.model.cadence === 'month'; + const ghostProduct = this.products.find(product => product.id === this.model.tier_id); + const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.tier_id); + const stripePrice = this.stripePrices.find((price) => { + return price.stripe_product_id === stripeProduct.stripe_product_id && + (isMonthly ? price.interval === 'month' : price.interval === 'year'); + }); + const mrr = isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12); + return { + id: faker.database.mongodbObjectId(), + customer_id: customer.customer_id, + subscription_id: this.model.id, + stripe_price_id: stripePrice.stripe_price_id, + status: 'active', + cancel_at_period_end: false, + current_period_end: this.model.expires_at, + start_date: this.model.created_at, + created_at: this.model.created_at, + created_by: 'unused', + mrr, + plan_id: stripeProduct.stripe_product_id, + plan_nickname: `${ghostProduct.name} - ${stripePrice.nickname}`, + plan_interval: stripePrice.interval, + plan_amount: stripePrice.amount, + plan_currency: stripePrice.currency + }; + } +} + +module.exports = MembersStripeCustomersSubscriptionsImporter; diff --git a/ghost/data-generator/lib/tables/members-stripe-customers.js b/ghost/data-generator/lib/tables/members-stripe-customers.js new file mode 100644 index 0000000000..5722453597 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-stripe-customers.js @@ -0,0 +1,28 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class MembersStripeCustomersImporter extends TableImporter { + constructor(knex) { + super('members_stripe_customers', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + customer_id: `cus_${faker.random.alphaNumeric(14, { + casing: 'mixed' + })}`, + name: this.model.name, + email: this.model.email, + created_at: this.model.created_at, + created_by: 'unused' + }; + } +} + +module.exports = MembersStripeCustomersImporter; diff --git a/ghost/data-generator/lib/tables/members-subscribe-events.js b/ghost/data-generator/lib/tables/members-subscribe-events.js new file mode 100644 index 0000000000..537f742736 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-subscribe-events.js @@ -0,0 +1,52 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {luck} = require('../utils/random'); + +class MembersSubscribeEventsImporter extends TableImporter { + constructor(knex, {newsletters, subscriptions}) { + super('members_subscribe_events', knex); + this.newsletters = newsletters; + this.subscriptions = subscriptions; + } + + setImportOptions({model}) { + this.model = model; + this.count = 0; + } + + generate() { + const count = this.count; + this.count = this.count + 1; + + if (count === 1 && this.model.status === 'free') { + return null; + } + + let createdAt = faker.date.between(new Date(this.model.created_at), new Date()).toISOString(); + let subscribed = luck(80); + + // Free newsletter by default + let newsletterId = this.newsletters[0].id; + if (this.model.status === 'paid' && count === 0) { + // Paid newsletter + newsletterId = this.newsletters[1].id; + createdAt = this.subscriptions.find(s => s.member_id === this.model.id).created_at; + subscribed = luck(98); + } + + if (!subscribed) { + return null; + } + + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.id, + newsletter_id: newsletterId, + subscribed: true, + created_at: createdAt, + source: 'member' + }; + } +} + +module.exports = MembersSubscribeEventsImporter; diff --git a/ghost/data-generator/lib/tables/members-subscription-created-events.js b/ghost/data-generator/lib/tables/members-subscription-created-events.js new file mode 100644 index 0000000000..8e0f4c34e2 --- /dev/null +++ b/ghost/data-generator/lib/tables/members-subscription-created-events.js @@ -0,0 +1,33 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); + +class MembersSubscriptionCreatedEventsImporter extends TableImporter { + constructor(knex, {subscriptions}) { + super('members_subscription_created_events', knex); + this.subscriptions = subscriptions; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const subscription = this.subscriptions.find(s => s.id === this.model.subscription_id); + return { + id: faker.database.mongodbObjectId(), + created_at: subscription.created_at, + member_id: subscription.member_id, + subscription_id: this.model.id, + // TODO: Implement attributions + attribution_id: null, + attribution_type: null, + attribution_url: null, + // TODO: Implement referrers + referrer_source: null, + referrer_medium: null, + referrer_url: null + }; + } +} + +module.exports = MembersSubscriptionCreatedEventsImporter; diff --git a/ghost/data-generator/lib/tables/members.js b/ghost/data-generator/lib/tables/members.js new file mode 100644 index 0000000000..a904c1ccba --- /dev/null +++ b/ghost/data-generator/lib/tables/members.js @@ -0,0 +1,65 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {faker: americanFaker} = require('@faker-js/faker/locale/en_US'); +const {blogStartDate: startTime} = require('../utils/blog-info'); +const generateEvents = require('../utils/event-generator'); +const {luck} = require('../utils/random'); + +class MembersImporter extends TableImporter { + constructor(knex) { + super('members', knex); + } + + setImportOptions({amount}) { + this.timestamps = generateEvents({ + shape: 'ease-in', + trend: 'positive', + total: amount, + startTime, + endTime: new Date() + }).sort(); + } + + generate() { + const id = faker.database.mongodbObjectId(); + // Use name from American locale to reflect an English-speaking audience + const name = `${americanFaker.name.firstName()} ${americanFaker.name.lastName()}`; + const timestamp = this.timestamps.shift(); + + return { + id, + uuid: faker.datatype.uuid(), + email: faker.internet.email(name, faker.date.birthdate().getFullYear().toString(), 'examplemail.com').toLowerCase(), + status: luck(5) ? 'comped' : luck(25) ? 'paid' : 'free', + name: name, + expertise: luck(30) ? faker.name.jobTitle() : undefined, + geolocation: JSON.stringify({ + organization_name: faker.company.name(), + region: faker.address.state(), + accuracy: 50, + asn: parseInt(faker.random.numeric(4)), + organization: `${faker.random.alpha({count: 2, casing: 'upper'})}${faker.random.numeric(4)} ${faker.company.name()}`, + timezone: faker.address.timeZone(), + longitude: faker.address.longitude(), + country_code3: faker.address.countryCode('alpha-3'), + area_code: '0', + ip: faker.internet.ipv4(), + city: faker.address.cityName(), + country: faker.address.country(), + continent_code: 'EU', + country_code: faker.address.countryCode('alpha-2'), + latitude: faker.address.latitude() + }), + email_count: 0, // Depends on number of emails sent since created_at, the newsletter they're a part of and subscription status + email_opened_count: 0, + email_open_rate: null, + // 40% of users logged in within a week, 60% sometime since registering + last_seen_at: luck(40) ? faker.date.recent(7).toISOString() : faker.date.between(timestamp, new Date()).toISOString(), + created_at: timestamp.toISOString(), + created_by: id, + updated_at: timestamp.toISOString() + }; + } +} + +module.exports = MembersImporter; diff --git a/ghost/data-generator/lib/tables/newsletters.js b/ghost/data-generator/lib/tables/newsletters.js new file mode 100644 index 0000000000..53d73690bf --- /dev/null +++ b/ghost/data-generator/lib/tables/newsletters.js @@ -0,0 +1,33 @@ +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); + +class NewslettersImporter extends TableImporter { + constructor(knex) { + super('newsletters', knex); + this.sortOrder = 0; + this.names = ['Occasional freebie', 'Regular premium']; + } + + generate() { + const name = this.names.shift(); + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + const weekAfter = new Date(blogStartDate); + weekAfter.setDate(weekAfter.getDate() + 7); + return { + id: faker.database.mongodbObjectId(), + uuid: faker.datatype.uuid(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + sender_reply_to: 'hello@example.com', + status: 'active', + subscribe_on_signup: faker.datatype.boolean(), + sort_order: sortOrder, + created_at: faker.date.between(blogStartDate, weekAfter) + }; + } +} + +module.exports = NewslettersImporter; diff --git a/ghost/data-generator/lib/tables/posts-authors.js b/ghost/data-generator/lib/tables/posts-authors.js new file mode 100644 index 0000000000..58679a883e --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-authors.js @@ -0,0 +1,27 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsAuthorsImporter extends TableImporter { + constructor(knex, {users}) { + super('posts_authors', knex); + this.users = users; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + author_id: this.users[faker.datatype.number(this.users.length - 1)].id, + sort_order: sortOrder + }; + } +} + +module.exports = PostsAuthorsImporter; diff --git a/ghost/data-generator/lib/tables/posts-products.js b/ghost/data-generator/lib/tables/posts-products.js new file mode 100644 index 0000000000..24576c12d2 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-products.js @@ -0,0 +1,27 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsProductsImporter extends TableImporter { + constructor(knex, {products}) { + super('posts_products', knex); + this.products = products; + } + + setImportOptions({model}) { + this.sortOrder = 0; + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + product_id: this.products[sortOrder].id, + sort_order: this.sortOrder + }; + } +} + +module.exports = PostsProductsImporter; diff --git a/ghost/data-generator/lib/tables/posts-tags.js b/ghost/data-generator/lib/tables/posts-tags.js new file mode 100644 index 0000000000..7ffb28d850 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts-tags.js @@ -0,0 +1,35 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class PostsTagsImporter extends TableImporter { + constructor(knex, {tags}) { + super('posts_tags', knex); + this.tags = tags; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.notIndex = []; + this.sortOrder = 0; + this.model = model; + } + + generate() { + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + let tagIndex = 0; + do { + tagIndex = faker.datatype.number(this.tags.length - 1); + } while (this.notIndex.includes(tagIndex)); + this.notIndex.push(tagIndex); + + return { + id: faker.database.mongodbObjectId(), + post_id: this.model.id, + tag_id: this.tags[tagIndex].id, + sort_order: sortOrder + }; + } +} + +module.exports = PostsTagsImporter; diff --git a/ghost/data-generator/lib/tables/posts.js b/ghost/data-generator/lib/tables/posts.js new file mode 100644 index 0000000000..4d453df752 --- /dev/null +++ b/ghost/data-generator/lib/tables/posts.js @@ -0,0 +1,66 @@ +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {luck} = require('../utils/random'); +const TableImporter = require('./base'); + +class PostsImporter extends TableImporter { + constructor(knex, {newsletters}) { + super('posts', knex); + this.newsletters = newsletters; + } + + async addNewsletters({posts}) { + for (const {id} of posts) { + await this.knex('posts').update({ + newsletter_id: luck(10) ? this.newsletters[0].id : this.newsletters[1].id + }).where({id}); + } + } + + generate() { + const title = faker.lorem.sentence(); + const content = faker.lorem.paragraphs(faker.datatype.number({ + min: 3, + max: 10 + })).split('\n'); + const twoYearsAgo = new Date(); + twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); + const twoWeeksAgo = new Date(); + twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14); + const timestamp = faker.date.between(twoYearsAgo, twoWeeksAgo); + return { + id: faker.database.mongodbObjectId(), + created_at: timestamp.toISOString(), + created_by: 'unused', + updated_at: timestamp.toISOString(), + published_at: faker.date.soon(5, timestamp).toISOString(), + uuid: faker.datatype.uuid(), + title: title, + slug: `${slugify(title)}-${faker.random.numeric(3)}`, + status: 'published', + mobiledoc: JSON.stringify({ + version: '0.3.1', + atoms: [], + cards: [], + markups: [['em']], + sections: content.map(paragraph => [ + 1, + 'p', + [ + [ + 0, + [], + 0, + paragraph + ] + ] + ]) + }), + html: content.map(paragraph => `

${paragraph}

`).join(''), + email_recipient_filter: 'all', + newsletter_id: luck(10) ? this.newsletters[0].id : this.newsletters[1].id + }; + } +} + +module.exports = PostsImporter; diff --git a/ghost/data-generator/lib/tables/products-benefits.js b/ghost/data-generator/lib/tables/products-benefits.js new file mode 100644 index 0000000000..c606a4f0ea --- /dev/null +++ b/ghost/data-generator/lib/tables/products-benefits.js @@ -0,0 +1,46 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); + +class ProductsBenefitsImporter extends TableImporter { + constructor(knex, {benefits}) { + super('products_benefits', knex); + this.benefits = benefits; + this.sortOrder = 0; + } + + setImportOptions({model}) { + this.model = model; + this.sortOrder = 0; + switch (this.model.name) { + case 'Bronze': + this.benefitCount = 1; + break; + case 'Silver': + this.benefitCount = 3; + break; + case 'Gold': + this.benefitCount = 5; + break; + case 'Free Preview': + this.benefitCount = 0; + break; + } + } + + generate() { + if (this.sortOrder >= this.benefitCount) { + // No more benefits than benefitCount + return null; + } + const sortOrder = this.sortOrder; + this.sortOrder = this.sortOrder + 1; + return { + id: faker.database.mongodbObjectId(), + product_id: this.model.id, + benefit_id: this.benefits[sortOrder].id, + sort_order: sortOrder + }; + } +} + +module.exports = ProductsBenefitsImporter; diff --git a/ghost/data-generator/lib/tables/products.js b/ghost/data-generator/lib/tables/products.js new file mode 100644 index 0000000000..1083a8738b --- /dev/null +++ b/ghost/data-generator/lib/tables/products.js @@ -0,0 +1,71 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const {blogStartDate} = require('../utils/blog-info'); + +class ProductsImporter extends TableImporter { + constructor(knex) { + super('products', knex); + } + + setImportOptions() { + this.names = ['Free Preview', 'Bronze', 'Silver', 'Gold']; + this.count = 0; + } + + async addStripePrices({products, stripeProducts, stripePrices}) { + for (const {id} of products) { + const stripeProduct = stripeProducts.find(p => id === p.product_id); + const monthlyPrice = stripePrices.find((p) => { + return p.stripe_product_id === stripeProduct.stripe_product_id && + p.interval === 'monthly'; + }); + const yearlyPrice = stripePrices.find((p) => { + return p.stripe_product_id === stripeProduct.stripe_product_id && + p.interval === 'yearly'; + }); + + const update = {}; + if (monthlyPrice) { + update.monthly_price_id = monthlyPrice.id; + } + if (yearlyPrice) { + update.yearly_price_id = yearlyPrice.id; + } + + if (Object.keys(update).length > 0) { + await this.knex('products').update(update).where({ + id + }); + } + } + } + + generate() { + const name = this.names.shift(); + const count = this.count; + this.count = this.count + 1; + const sixMonthsLater = new Date(blogStartDate); + sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6); + const tierInfo = { + type: 'free', + description: 'A free sample of content' + }; + if (count !== 0) { + tierInfo.type = 'paid'; + tierInfo.description = `${name} star member`; + tierInfo.currency = 'USD'; + tierInfo.monthly_price = count * 500; + tierInfo.yearly_price = count * 5000; + } + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + visibility: 'public', + created_at: faker.date.between(blogStartDate, sixMonthsLater) + }, tierInfo); + } +} + +module.exports = ProductsImporter; diff --git a/ghost/data-generator/lib/tables/stripe-prices.js b/ghost/data-generator/lib/tables/stripe-prices.js new file mode 100644 index 0000000000..f4090d4cab --- /dev/null +++ b/ghost/data-generator/lib/tables/stripe-prices.js @@ -0,0 +1,61 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); + +class StripePricesImporter extends TableImporter { + constructor(knex, {products}) { + super('stripe_prices', knex); + this.products = products; + } + + setImportOptions({model}) { + this.model = model; + + this.count = 0; + } + + generate() { + const sixWeeksLater = new Date(blogStartDate); + sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + + const count = this.count; + this.count = this.count + 1; + + const relatedProduct = this.products.find(product => product.id === this.model.product_id); + + if (count === 1 && relatedProduct.monthly_price === null) { + // Only single complimentary price (yearly) + return null; + } + + const billingCycle = { + nickname: 'Monthly', + interval: 'month', + type: 'recurring', + currency: 'usd', + amount: relatedProduct.monthly_price + }; + if (count === 1) { + billingCycle.nickname = 'Yearly'; + billingCycle.interval = 'year'; + billingCycle.amount = relatedProduct.yearly_price; + } else if (relatedProduct.monthly_price === null) { + billingCycle.nickname = 'Complimentary'; + billingCycle.interval = 'year'; + billingCycle.amount = 0; + } + + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + stripe_price_id: faker.datatype.hexadecimal({ + length: 64, + prefix: '' + }), + stripe_product_id: this.model.stripe_product_id, + active: true, + created_at: faker.date.between(blogStartDate, sixWeeksLater) + }, billingCycle); + } +} + +module.exports = StripePricesImporter; diff --git a/ghost/data-generator/lib/tables/stripe-products.js b/ghost/data-generator/lib/tables/stripe-products.js new file mode 100644 index 0000000000..fd198a52b2 --- /dev/null +++ b/ghost/data-generator/lib/tables/stripe-products.js @@ -0,0 +1,29 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./base'); +const {blogStartDate} = require('../utils/blog-info'); + +class StripeProductsImporter extends TableImporter { + constructor(knex) { + super('stripe_products', knex); + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const sixWeeksLater = new Date(blogStartDate); + sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + return { + id: faker.database.mongodbObjectId(), + product_id: this.model.id, + stripe_product_id: faker.datatype.hexadecimal({ + length: 64, + prefix: '' + }), + created_at: faker.date.between(blogStartDate, sixWeeksLater) + }; + } +} + +module.exports = StripeProductsImporter; diff --git a/ghost/data-generator/lib/tables/subscriptions.js b/ghost/data-generator/lib/tables/subscriptions.js new file mode 100644 index 0000000000..8f63f33e6c --- /dev/null +++ b/ghost/data-generator/lib/tables/subscriptions.js @@ -0,0 +1,64 @@ +const {faker} = require('@faker-js/faker'); +const generateEvents = require('../utils/event-generator'); +const TableImporter = require('./base'); + +class SubscriptionsImporter extends TableImporter { + constructor(knex, {members, stripeProducts, stripePrices}) { + super('subscriptions', knex); + this.members = members; + this.stripeProducts = stripeProducts; + this.stripePrices = stripePrices; + } + + setImportOptions({model}) { + this.model = model; + } + + generate() { + const member = this.members.find(m => m.id === this.model.member_id); + const status = member.status; + const billingInfo = {}; + const isMonthly = faker.datatype.boolean(); + if (status === 'paid') { + const stripeProduct = this.stripeProducts.find(product => product.product_id === this.model.product_id); + const stripePrice = this.stripePrices.find((price) => { + return price.stripe_product_id === stripeProduct.stripe_product_id && + (isMonthly ? price.interval === 'month' : price.interval === 'year'); + }); + billingInfo.cadence = isMonthly ? 'month' : 'year'; + billingInfo.currency = stripePrice.currency; + billingInfo.amount = stripePrice.amount; + } + const [startDate] = generateEvents({ + total: 1, + trend: 'negative', + startTime: new Date(member.created_at), + endTime: new Date(), + shape: 'ease-out' + }); + const endDate = new Date(startDate); + if (isMonthly) { + endDate.setMonth(new Date().getMonth()); + if (endDate < new Date()) { + endDate.setMonth(endDate.getMonth() + 1); + } + } else { + endDate.setFullYear(new Date().getFullYear()); + if (endDate < new Date()) { + endDate.setFullYear(endDate.getFullYear() + 1); + } + } + return Object.assign({}, { + id: faker.database.mongodbObjectId(), + type: status, + status: 'active', + member_id: this.model.member_id, + tier_id: this.model.product_id, + payment_provider: 'stripe', + expires_at: endDate.toISOString(), + created_at: startDate.toISOString() + }, billingInfo); + } +} + +module.exports = SubscriptionsImporter; diff --git a/ghost/data-generator/lib/tables/tags.js b/ghost/data-generator/lib/tables/tags.js new file mode 100644 index 0000000000..116ef0c0bf --- /dev/null +++ b/ghost/data-generator/lib/tables/tags.js @@ -0,0 +1,29 @@ +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const TableImporter = require('./base'); + +class TagsImporter extends TableImporter { + constructor(knex, {users}) { + super('tags', knex); + this.users = users; + } + + generate() { + let name = `${faker.color.human()} ${faker.name.jobType()}`; + name = `${name[0].toUpperCase()}${name.slice(1)}`; + const threeYearsAgo = new Date(); + threeYearsAgo.setFullYear(threeYearsAgo.getFullYear() - 3); + const twoYearsAgo = new Date(); + twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: `${slugify(name)}-${faker.random.numeric(3)}`, + description: faker.lorem.sentence(), + created_at: faker.date.between(threeYearsAgo, twoYearsAgo).toISOString(), + created_by: this.users[faker.datatype.number(this.users.length - 1)].id + }; + } +} + +module.exports = TagsImporter; diff --git a/ghost/data-generator/lib/tables/users.js b/ghost/data-generator/lib/tables/users.js new file mode 100644 index 0000000000..b7fea165d9 --- /dev/null +++ b/ghost/data-generator/lib/tables/users.js @@ -0,0 +1,26 @@ +const TableImporter = require('./base'); +const {faker} = require('@faker-js/faker'); +const {slugify} = require('@tryghost/string'); +const security = require('@tryghost/security'); + +class UsersImporter extends TableImporter { + constructor(knex) { + super('users', knex); + } + + generate() { + const name = `${faker.name.firstName()} ${faker.name.lastName()}`; + return { + id: faker.database.mongodbObjectId(), + name: name, + slug: slugify(name), + password: security.password.hash(faker.color.human()), + email: faker.internet.email(name), + profile_image: faker.internet.avatar(), + created_at: faker.date.between(new Date(2016, 0), new Date()).toISOString(), + created_by: 'unused' + }; + } +} + +module.exports = UsersImporter; diff --git a/ghost/data-generator/lib/utils/blog-info.js b/ghost/data-generator/lib/utils/blog-info.js new file mode 100644 index 0000000000..cf6ed68af6 --- /dev/null +++ b/ghost/data-generator/lib/utils/blog-info.js @@ -0,0 +1,3 @@ +module.exports = { + blogStartDate: new Date(2018, 5, 4) +}; diff --git a/ghost/data-generator/lib/utils/event-generator.js b/ghost/data-generator/lib/utils/event-generator.js new file mode 100644 index 0000000000..d72597004d --- /dev/null +++ b/ghost/data-generator/lib/utils/event-generator.js @@ -0,0 +1,43 @@ +const probabilityDistributions = require('probability-distributions'); + +const generateEvents = ({ + shape = 'flat', + trend = 'positive', + total = 0, + startTime = new Date(), + endTime = new Date() +} = {}) => { + let alpha = 0; + let beta = 0; + let positiveTrend = trend === 'positive'; + switch (shape) { + case 'linear': + alpha = 2; + beta = 1; + break; + case 'ease-in': + alpha = 4; + beta = 1; + break; + case 'ease-out': + alpha = 1; + beta = 4; + positiveTrend = !positiveTrend; + break; + case 'flat': + alpha = 1; + beta = 1; + break; + } + const data = probabilityDistributions.rbeta(total, alpha, beta, 0); + const startTimeValue = startTime.valueOf(); + const timeDifference = endTime.valueOf() - startTimeValue; + return data.map((x) => { + if (!positiveTrend) { + x = 1 - x; + } + return new Date(startTimeValue + timeDifference * x); + }); +}; + +module.exports = generateEvents; diff --git a/ghost/data-generator/lib/utils/random.js b/ghost/data-generator/lib/utils/random.js new file mode 100644 index 0000000000..49a9b34597 --- /dev/null +++ b/ghost/data-generator/lib/utils/random.js @@ -0,0 +1,13 @@ +const {faker} = require('@faker-js/faker'); + +/** + * Adds another degree of randomness into some decisions + * @param {number} lowerThan Only this % of people will achieve this luck + * @returns {boolean} Whether this person is lucky enough for the condition + */ +const luck = lowerThan => faker.datatype.number({ + min: 1, + max: 100 +}) <= lowerThan; + +module.exports = {luck}; diff --git a/ghost/data-generator/package.json b/ghost/data-generator/package.json new file mode 100644 index 0000000000..e1460b70e4 --- /dev/null +++ b/ghost/data-generator/package.json @@ -0,0 +1,32 @@ +{ + "name": "@tryghost/data-generator", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/data-generator", + "author": "Ghost Foundation", + "private": true, + "main": "index.js", + "scripts": { + "dev": "echo \"Implement me!\"", + "test:unit": "NODE_ENV=testing c8 --all --check-coverage --reporter text --reporter cobertura mocha './test/**/*.test.js'", + "test": "yarn test:unit", + "lint:code": "eslint *.js lib/ --ext .js --cache", + "lint": "yarn lint:code && yarn lint:test", + "lint:test": "eslint -c test/.eslintrc.js test/ --ext .js --cache" + }, + "files": [ + "index.js", + "lib" + ], + "devDependencies": { + "c8": "7.12.0", + "knex": "2.3.0", + "mocha": "10.1.0", + "should": "13.2.3", + "sinon": "14.0.1" + }, + "dependencies": { + "@faker-js/faker": "7.6.0", + "@tryghost/string": "0.2.1", + "probability-distributions": "0.9.1" + } +} diff --git a/ghost/data-generator/test/.eslintrc.js b/ghost/data-generator/test/.eslintrc.js new file mode 100644 index 0000000000..829b601eb0 --- /dev/null +++ b/ghost/data-generator/test/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/data-generator/test/data-generator.test.js b/ghost/data-generator/test/data-generator.test.js new file mode 100644 index 0000000000..acf7241881 --- /dev/null +++ b/ghost/data-generator/test/data-generator.test.js @@ -0,0 +1,249 @@ +// Switch these lines once there are useful utils +// const testUtils = require('./utils'); +require('./utils'); +const knex = require('knex'); +const { + ProductsImporter, + StripeProductsImporter, + StripePricesImporter +} = require('../lib/tables'); + +const generateEvents = require('../lib/utils/event-generator'); + +const DataGenerator = require('../index'); + +const schema = require('../../core/core/server/data/schema'); + +describe('Data Generator', function () { + let db; + + beforeEach(async function () { + db = knex({ + client: 'sqlite3', + useNullAsDefault: true, + connection: { + filename: ':memory:' + } + }); + + for (const tableName of Object.keys(schema.tables)) { + await db.schema.createTable(tableName, function (table) { + for (const rowName of Object.keys(schema.tables[tableName])) { + const row = schema.tables[tableName][rowName]; + + if (rowName === '@@UNIQUE_CONSTRAINTS@@') { + for (const constraints of row) { + table.unique(constraints); + } + break; + } else if (rowName === '@@INDEXES@@') { + for (const indexes of row) { + table.index(indexes); + } + break; + } + + let rowChain = table[row.type.toLowerCase()](rowName); + if ('nullable' in row) { + if (row.nullable) { + rowChain = rowChain.nullable(); + } else { + rowChain = rowChain.notNullable(); + } + } + if ('defaultTo' in row) { + rowChain = rowChain.defaultTo(row.defaultTo); + } + if ('references' in row) { + const [foreignTable, foreignRow] = row.references.split('.'); + rowChain = rowChain.references(foreignRow).inTable(foreignTable); + } + if (row.unique) { + table.unique([rowName]); + } + if (row.primary) { + table.primary(rowName); + } + } + }); + } + }); + + afterEach(async function () { + await db.destroy(); + }); + + it('Can import the whole dataset without error', async function () { + const dataGenerator = new DataGenerator({ + eventsOnly: false, + knex: db, + schema: schema, + logger: {}, + modelQuantities: { + members: 10, + membersLoginEvents: 5, + posts: 2 + } + }); + try { + return await dataGenerator.importData(); + } catch (err) { + (false).should.eql(true, err.message); + } + }); +}); + +describe('Importer', function () { + let db; + + beforeEach(async function () { + db = knex({ + client: 'sqlite3', + useNullAsDefault: true, + connection: { + filename: ':memory:' + } + }); + + await db.schema.createTable('products', function (table) { + table.string('id'); + table.string('name'); + table.string('slug'); + table.string('visibility'); + table.date('created_at'); + table.string('type'); + table.string('description'); + table.string('currency'); + table.integer('monthly_price'); + table.integer('yearly_price'); + table.string('monthly_price_id'); + table.string('yearly_price_id'); + }); + + await db.schema.createTable('stripe_products', function (table) { + table.string('id'); + table.string('product_id'); + table.string('stripe_product_id'); + table.date('created_at'); + table.date('updated_at'); + }); + + await db.schema.createTable('stripe_prices', function (table) { + table.string('id'); + table.string('stripe_price_id'); + table.string('stripe_product_id'); + table.boolean('active'); + table.string('nickname'); + table.string('currency'); + table.integer('amount'); + table.string('type'); + table.string('interval'); + table.string('description'); + table.date('created_at'); + table.date('updated_at'); + }); + }); + + afterEach(async function () { + await db.destroy(); + }); + + it('Should import a single item', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 1, rows: ['name', 'monthly_price', 'yearly_price']}); + + products.length.should.eql(1); + products[0].name.should.eql('Free Preview'); + + const results = await db.select('id', 'name').from('products'); + + results.length.should.eql(1); + results[0].name.should.eql('Free Preview'); + }); + + it('Should import an item for each entry in an array', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const stripeProductsImporter = new StripeProductsImporter(db); + await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const results = await db.select('id').from('stripe_products'); + + results.length.should.eql(4); + }); + + it('Should update products to reference price ids', async function () { + const productsImporter = new ProductsImporter(db); + const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + + const stripeProductsImporter = new StripeProductsImporter(db); + const stripeProducts = await stripeProductsImporter.importForEach(products, { + amount: 1, + rows: ['product_id', 'stripe_product_id'] + }); + + const stripePricesImporter = new StripePricesImporter(db, {products}); + const stripePrices = await stripePricesImporter.importForEach(stripeProducts, { + amount: 2, + rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] + }); + + await productsImporter.addStripePrices({ + products, + stripeProducts, + stripePrices + }); + + const results = await db.select('id', 'name', 'monthly_price_id', 'yearly_price_id').from('products'); + + results.length.should.eql(4); + results[0].name.should.eql('Free Preview'); + }); +}); + +describe('Events Generator', function () { + it('Generates a set of timestamps which meet the criteria', function () { + const startTime = new Date(); + startTime.setDate(startTime.getDate() - 30); + const endTime = new Date(); + const timestamps = generateEvents({ + shape: 'flat', + total: 100, + trend: 'positive', + startTime, + endTime + }); + + for (const timestamp of timestamps) { + timestamp.valueOf().should.be.lessThanOrEqual(endTime.valueOf()); + timestamp.valueOf().should.be.greaterThanOrEqual(startTime.valueOf()); + } + }); + + it('Works for a set of shapes', function () { + const startTime = new Date(); + startTime.setDate(startTime.getDate() - 30); + const endTime = new Date(); + + const options = { + startTime, + endTime, + total: 100, + trend: 'positive' + }; + + const shapes = ['linear', 'flat', 'ease-in', 'ease-out']; + + for (const shape of shapes) { + try { + generateEvents(Object.assign({}, options, {shape})); + } catch (err) { + (false).should.eql(true, err.message); + } + } + }); +}); diff --git a/ghost/data-generator/test/utils/assertions.js b/ghost/data-generator/test/utils/assertions.js new file mode 100644 index 0000000000..7364ee8aa1 --- /dev/null +++ b/ghost/data-generator/test/utils/assertions.js @@ -0,0 +1,11 @@ +/** + * Custom Should Assertions + * + * Add any custom assertions to this file. + */ + +// Example Assertion +// should.Assertion.add('ExampleAssertion', function () { +// this.params = {operator: 'to be a valid Example Assertion'}; +// this.obj.should.be.an.Object; +// }); diff --git a/ghost/data-generator/test/utils/index.js b/ghost/data-generator/test/utils/index.js new file mode 100644 index 0000000000..0d67d86ff8 --- /dev/null +++ b/ghost/data-generator/test/utils/index.js @@ -0,0 +1,11 @@ +/** + * Test Utilities + * + * Shared utils for writing tests + */ + +// Require overrides - these add globals for tests +require('./overrides'); + +// Require assertions - adds custom should assertions +require('./assertions'); diff --git a/ghost/data-generator/test/utils/overrides.js b/ghost/data-generator/test/utils/overrides.js new file mode 100644 index 0000000000..90203424ee --- /dev/null +++ b/ghost/data-generator/test/utils/overrides.js @@ -0,0 +1,10 @@ +// This file is required before any test is run + +// Taken from the should wiki, this is how to make should global +// Should is a global in our eslint test config +global.should = require('should').noConflict(); +should.extend(); + +// Sinon is a simple case +// Sinon is a global in our eslint test config +global.sinon = require('sinon'); diff --git a/yarn.lock b/yarn.lock index 5ba2e12ac9..0cff3f00ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2447,6 +2447,11 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@faker-js/faker@7.6.0": + version "7.6.0" + resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-7.6.0.tgz#9ea331766084288634a9247fcd8b84f16ff4ba07" + integrity sha512-XK6BTq1NDMo9Xqw/YkYyGjSsg44fbNwYRx7QK2CuoQgyy+f1rrTDHoExVM5PsyXCtfl2vs2vVJ0MN0yN6LppRw== + "@fortawesome/fontawesome-common-types@^0.2.36": version "0.2.36" resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-0.2.36.tgz#b44e52db3b6b20523e0c57ef8c42d315532cb903" @@ -9849,6 +9854,11 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== +crypto@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-0.0.3.tgz#470a81b86be4c5ee17acc8207a1f5315ae20dbb0" + integrity sha512-Q6Ka98WcvWXXg+9cnqd3jHpTSIOaH6/q0m/bESMfQo/0uFxy6e/7EqVS4JdaWx9qLdqV56tDufy2b12dj7BHJg== + css-blank-pseudo@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/css-blank-pseudo/-/css-blank-pseudo-3.0.3.tgz#36523b01c12a25d812df343a32c322d2a2324561" @@ -12147,7 +12157,6 @@ ember-power-calendar@^0.16.3: ember-power-datepicker@cibernox/ember-power-datepicker: version "0.8.1" - uid da580474a2c449b715444934ddb626b7c07f46a7 resolved "https://codeload.github.com/cibernox/ember-power-datepicker/tar.gz/da580474a2c449b715444934ddb626b7c07f46a7" dependencies: ember-basic-dropdown "^3.0.11" @@ -14723,7 +14732,6 @@ globrex@^0.1.2: "google-caja-bower@https://github.com/acburdine/google-caja-bower#ghost": version "6011.0.0" - uid "275cb75249f038492094a499756a73719ae071fd" resolved "https://github.com/acburdine/google-caja-bower#275cb75249f038492094a499756a73719ae071fd" got@9.6.0: @@ -17520,7 +17528,6 @@ keygrip@~1.1.0: "keymaster@https://github.com/madrobby/keymaster.git": version "1.6.3" - uid f8f43ddafad663b505dc0908e72853bcf8daea49 resolved "https://github.com/madrobby/keymaster.git#f8f43ddafad663b505dc0908e72853bcf8daea49" keypair@1.0.4: @@ -19368,7 +19375,6 @@ mocha@^2.5.3: mock-knex@TryGhost/mock-knex#8ecb8c227bf463c991c3d820d33f59efc3ab9682: version "0.4.9" - uid "8ecb8c227bf463c991c3d820d33f59efc3ab9682" resolved "https://codeload.github.com/TryGhost/mock-knex/tar.gz/8ecb8c227bf463c991c3d820d33f59efc3ab9682" dependencies: bluebird "^3.4.1" @@ -21905,6 +21911,13 @@ private@^0.1.6, private@^0.1.8: resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg== +probability-distributions@0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/probability-distributions/-/probability-distributions-0.9.1.tgz#c689c8d62173bb281a0999e98263acc76a628e55" + integrity sha512-guES77A321GzcK6swYmxhBeYQ10UNyiG0Gf4Ks+3239BuS27EFH/rrlfBW/oYlPyiufdey/3OlVqKvlXFB9LTQ== + dependencies: + crypto "0.0.3" + probe-image-size@7.2.3: version "7.2.3" resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.3.tgz#d49c64be540ec8edea538f6f585f65a9b3ab4309" @@ -23795,7 +23808,6 @@ simple-update-notifier@^1.0.7: "simplemde@https://github.com/kevinansfield/simplemde-markdown-editor.git#ghost": version "1.11.2" - uid "4c39702de7d97f9b32d5c101f39237b6dab7c3ee" resolved "https://github.com/kevinansfield/simplemde-markdown-editor.git#4c39702de7d97f9b32d5c101f39237b6dab7c3ee" sinon@14.0.1: