Entirely rewrote data generator to simplify codebase

refs: https://github.com/TryGhost/DevOps/issues/11

This is a pretty huge commit, but the relevant points are:
* Each importer no longer needs to be passed a set of data, it just gets the data it needs
* Each importer specifies its dependencies, so that the order of import can be determined at runtime using a topological sort
* The main data generator function can just tell each importer to import the data it has

This makes working on the data generator much easier.

Some other benefits are:
* Batched importing, massively speeding up the whole process
* `--tables` to set the exact tables you want to import, and specify the quantity of each

ghost/core/core/cli/generate-data.js

@@ -6,10 +6,10 @@ module.exports = class DataGeneratorCommand extends Command {
     setup() {
         this.help('Generates random data to populate the database for development & testing');
         this.argument('--base-data-pack', {type: 'string', defaultValue: '', desc: 'Base data pack file location, imported instead of random content'});
-        this.argument('--scale', {type: 'string', defaultValue: 'small', desc: 'Scale of the data to generate. `small` for a quick run, `large` for more content'});
-        this.argument('--single-table', {type: 'string', desc: 'Import a single table'});
         this.argument('--quantity', {type: 'number', desc: 'When importing a single table, the quantity to import'});
         this.argument('--clear-database', {type: 'boolean', defaultValue: false, desc: 'Clear all entries in the database before importing'});
+        this.argument('--tables', {type: 'string', desc: 'Only import the specified list of tables, where quantities can be specified by appending a colon followed by the quantity for each table. Example: --tables=members:1000,posts,tags,members_login_events'});
+        this.argument('--with-default', {type: 'boolean', defaultValue: false, desc: 'Include default tables as well as those specified (simply override quantities)'});
     }
 
     initializeContext(context) {
@@ -30,22 +30,15 @@ module.exports = class DataGeneratorCommand extends Command {
 
     async handle(argv = {}) {
         const knex = require('../server/data/db/connection');
-        const {tables: schema} = require('../server/data/schema/index');
 
-        const modelQuantities = {};
+        const tables = (argv.tables ? argv.tables.split(',') : []).map(table => ({
-        if (argv.scale) {
+            name: table.split(':')[0],
-            if (argv.scale === 'small') {
+            quantity: parseInt(table.split(':')[1]) || undefined
-                modelQuantities.members = 200;
+        }));
-                modelQuantities.membersLoginEvents = 1;
-                modelQuantities.posts = 10;
-            }
-            // Defaults in data-generator package make a large set
-        }
 
         const dataGenerator = new DataGenerator({
             baseDataPack: argv['base-data-pack'],
             knex,
-            schema,
             logger: {
                 log: this.log,
                 ok: this.ok,
@@ -55,16 +48,13 @@ module.exports = class DataGeneratorCommand extends Command {
                 fatal: this.fatal,
                 debug: this.debug
             },
-            modelQuantities,
             baseUrl: config.getSiteUrl(),
-            clearDatabase: argv['clear-database']
+            clearDatabase: argv['clear-database'],
+            tables,
+            withDefault: argv['with-default']
         });
         try {
-            if (argv['single-table']) {
+            await dataGenerator.importData();
-                await dataGenerator.importSingleTable(argv['single-table'], argv.quantity ?? undefined);
-            } else {
-                await dataGenerator.importData();
-            }
         } catch (error) {
             this.fatal('Failed while generating data: ', error);
         }

ghost/data-generator/index.js

@@ -1 +1 @@
-module.exports = require('./lib/data-generator');
+module.exports = require('./lib/DataGenerator');

ghost/data-generator/lib/DataGenerator.js

@@ -0,0 +1,190 @@
+const path = require('path');
+const fs = require('fs/promises');
+const JsonImporter = require('./utils/JsonImporter');
+const {getProcessRoot} = require('@tryghost/root-utils');
+const topologicalSort = require('./utils/topological-sort');
+
+const importers = require('./importers').reduce((acc, val) => {
+    acc[val.table] = val;
+    return acc;
+}, {});
+const schema = require('../../core/core/server/data/schema').tables;
+
+class DataGenerator {
+    constructor({
+        knex,
+        tables,
+        clearDatabase = false,
+        baseDataPack = '',
+        baseUrl,
+        logger,
+        withDefault
+    }) {
+        this.knex = knex;
+        this.tableList = tables || [];
+        this.willClearData = clearDatabase;
+        this.useBaseDataPack = baseDataPack !== '';
+        this.baseDataPack = baseDataPack;
+        this.baseUrl = baseUrl;
+        this.logger = logger;
+        this.withDefault = withDefault;
+    }
+
+    sortTableList() {
+        // Add missing dependencies
+        for (const table of this.tableList) {
+            table.importer = importers[table.name];
+            // eslint-disable-next-line no-unused-vars
+            table.dependencies = Object.entries(schema[table.name]).reduce((acc, [_col, data]) => {
+                if (data.references) {
+                    const referencedTable = data.references.split('.')[0];
+                    if (!acc.includes(referencedTable)) {
+                        acc.push(referencedTable);
+                    }
+                }
+                return acc;
+            }, table.importer.dependencies);
+
+            for (const dependency of table.dependencies) {
+                if (!this.tableList.find(t => t.name === dependency)) {
+                    this.tableList.push({
+                        name: dependency,
+                        importer: importers[dependency]
+                    });
+                }
+            }
+        }
+
+        // Order to ensure dependencies are created before dependants
+        this.tableList = topologicalSort(this.tableList);
+    }
+
+    /**
+     * TODO: This needs to reverse through all dependency chains to clear data from all tables
+     * @param {import('knex/types').Knex.Transaction} transaction
+     */
+    async clearData(transaction) {
+        const tables = this.tableList.map(t => t.name).reverse();
+
+        // TODO: Remove this once we import posts_meta
+        tables.unshift('posts_meta');
+
+        // Clear data from any tables that are being imported
+        for (const table of tables) {
+            this.logger.debug(`Clearing table ${table}`);
+
+            if (table === 'roles_users') {
+                await transaction(table).del().whereNot('user_id', '1');
+            } else if (table === 'users') {
+                // Avoid deleting the admin user
+                await transaction(table).del().whereNot('id', '1');
+            } else {
+                await transaction(table).del();
+            }
+        }
+    }
+
+    async importBasePack(transaction) {
+        let baseDataPack = this.baseDataPack;
+        if (!path.isAbsolute(this.baseDataPack)) {
+            baseDataPack = path.join(getProcessRoot(), baseDataPack);
+        }
+        let baseData = {};
+        try {
+            baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString());
+            this.logger.info('Read base data pack');
+        } catch (error) {
+            this.logger.error('Failed to read data pack: ', error);
+            throw error;
+        }
+
+        this.logger.info('Starting base data import');
+        const jsonImporter = new JsonImporter(transaction);
+
+        // Clear settings table
+        await transaction('settings').del();
+
+        // Hard-coded for order
+        const tablesToImport = [
+            'newsletters',
+            'posts',
+            'tags',
+            'products',
+            'benefits',
+            'products_benefits',
+            'stripe_products',
+            'stripe_prices',
+            'settings',
+            'custom_theme_settings'
+        ];
+        for (const table of tablesToImport) {
+            this.logger.info(`Importing content for table ${table} from base data pack`);
+            await jsonImporter.import({
+                name: table,
+                data: baseData[table]
+            });
+            const tableIndex = this.tableList.findIndex(t => t.name === table);
+            if (tableIndex !== -1) {
+                this.tableList.splice(tableIndex, 1);
+            }
+        }
+
+        this.logger.info('Completed base data import');
+    }
+
+    async importData() {
+        const transaction = await this.knex.transaction();
+
+        // Add default tables if none are specified
+        if (this.tableList.length === 0) {
+            this.tableList = Object.keys(importers).map(name => ({name}));
+        } else if (this.withDefault) {
+            // Add default tables to the end of the list
+            const defaultTables = Object.keys(importers).map(name => ({name}));
+            for (const table of defaultTables) {
+                if (!this.tableList.find(t => t.name === table.name)) {
+                    this.tableList.push(table);
+                }
+            }
+        }
+
+        // Error if we have an unknown table
+        for (const table of this.tableList) {
+            if (importers[table.name] === undefined) {
+                // eslint-disable-next-line
+                throw new Error(`Unknown table: ${table.name}`);
+            }
+        }
+
+        this.sortTableList();
+
+        if (this.willClearData) {
+            await this.clearData(transaction);
+        }
+
+        if (this.useBaseDataPack) {
+            await this.importBasePack(transaction);
+        }
+
+        for (const table of this.tableList) {
+            this.logger.info('Importing content for table', table.name);
+            // Add all common options to every importer, whether they use them or not
+            const tableImporter = new table.importer(this.knex, transaction, {
+                baseUrl: this.baseUrl
+            });
+            await tableImporter.import(table.quantity ?? undefined);
+        }
+
+        // Finalise all tables - uses new table importer objects to avoid keeping all data in memory
+        for (const table of this.tableList) {
+            const tableImporter = new table.importer(this.knex, transaction, {
+                baseUrl: this.baseUrl
+            });
+            await tableImporter.finalise();
+        }
+
+        await transaction.commit();
+    }
+}
+
+module.exports = DataGenerator;

ghost/data-generator/lib/data-generator.js

@@ -1,461 +0,0 @@
-const tables = require('./tables');
-// Order here does not matter
-const {
-    NewslettersImporter,
-    PostsImporter,
-    UsersImporter,
-    TagsImporter,
-    ProductsImporter,
-    MembersImporter,
-    BenefitsImporter,
-    MentionsImporter,
-    PostsAuthorsImporter,
-    PostsTagsImporter,
-    ProductsBenefitsImporter,
-    MembersProductsImporter,
-    PostsProductsImporter,
-    MembersNewslettersImporter,
-    StripeProductsImporter,
-    StripePricesImporter,
-    SubscriptionsImporter,
-    EmailsImporter,
-    MembersCreatedEventsImporter,
-    MembersLoginEventsImporter,
-    MembersStatusEventsImporter,
-    MembersSubscribeEventsImporter,
-    MembersStripeCustomersImporter,
-    MembersPaidSubscriptionEventsImporter,
-    EmailBatchesImporter,
-    EmailRecipientsImporter,
-    RedirectsImporter,
-    MembersClickEventsImporter,
-    OffersImporter,
-    MembersStripeCustomersSubscriptionsImporter,
-    MembersSubscriptionCreatedEventsImporter,
-    LabelsImporter,
-    MembersLabelsImporter,
-    RolesUsersImporter,
-    MembersFeedbackImporter
-} = tables;
-const path = require('path');
-const fs = require('fs/promises');
-const {faker} = require('@faker-js/faker');
-const JsonImporter = require('./utils/json-importer');
-const {getProcessRoot} = require('@tryghost/root-utils');
-
-/**
- * @typedef {Object} DataGeneratorOptions
- * @property {string} baseDataPack
- * @property {import('knex/types').Knex} knex
- * @property {Object} schema
- * @property {Object} logger
- * @property {Object} modelQuantities
- * @property {string} baseUrl
- * @property {boolean} clearDatabase
- */
-
-const defaultQuantities = {
-    members: () => faker.datatype.number({
-        min: 7000,
-        max: 8000
-    }),
-    // This will generate n * <members> events, is not worth being very high
-    membersLoginEvents: 5,
-    posts: () => faker.datatype.number({
-        min: 80,
-        max: 120
-    })
-};
-
-class DataGenerator {
-    /**
-     *
-     * @param {DataGeneratorOptions} options
-     */
-    constructor({
-        baseDataPack = '',
-        knex,
-        schema,
-        logger,
-        modelQuantities = {},
-        baseUrl,
-        clearDatabase
-    }) {
-        this.useBaseData = baseDataPack !== '';
-        this.baseDataPack = baseDataPack;
-        this.knex = knex;
-        this.schema = schema;
-        this.logger = logger;
-        this.modelQuantities = Object.assign({}, defaultQuantities, modelQuantities);
-        this.baseUrl = baseUrl;
-        this.clearDatabase = clearDatabase;
-    }
-
-    async importData() {
-        const transaction = await this.knex.transaction();
-
-        if (this.clearDatabase) {
-            this.logger.info('Clearing existing database');
-
-            // List of tables ordered to avoid dependencies when deleting
-            const tableNames = Object.values(tables).map(importer => importer.table).reverse();
-            // We don't currently generate posts_meta, but we need to clear it to ensure posts can be removed
-            tableNames.unshift('posts_meta');
-            for (const table of tableNames) {
-                this.logger.debug(`Clearing table ${table}`);
-                if (table === 'roles_users') {
-                    await transaction(table).del().whereNot('user_id', '1');
-                    continue;
-                }
-                if (table === 'users') {
-                    // Avoid deleting the admin user
-                    await transaction(table).del().whereNot('id', '1');
-                    continue;
-                }
-                await transaction(table).del();
-            }
-            this.logger.info('Finished clearing database');
-        }
-
-        this.logger.info('Starting import process, this has two parts: base data and member data. It can take a while...');
-
-        const usersImporter = new UsersImporter(transaction);
-        const users = await usersImporter.import({amount: 8});
-
-        let newsletters;
-        let posts;
-        let tags;
-        let products;
-        let stripeProducts;
-        let stripePrices;
-        let benefits;
-        let labels;
-
-        // Use an existant set of data for a more realisitic looking site
-        if (this.useBaseData) {
-            let baseDataPack = this.baseDataPack;
-            if (!path.isAbsolute(this.baseDataPack)) {
-                baseDataPack = path.join(getProcessRoot(), baseDataPack);
-            }
-            let baseData = {};
-            try {
-                baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString());
-                this.logger.info('Read base data pack');
-            } catch (error) {
-                this.logger.error('Failed to read data pack: ', error);
-                throw error;
-            }
-
-            this.logger.info('Starting base data import');
-            const jsonImporter = new JsonImporter(transaction);
-
-            // Must have at least 2 in base data set
-            newsletters = await jsonImporter.import({
-                name: 'newsletters',
-                data: baseData.newsletters,
-                rows: ['sort_order']
-            });
-            newsletters.sort((a, b) => a.sort_order - b.sort_order);
-
-            const postsImporter = new PostsImporter(transaction, {
-                newsletters
-            });
-            posts = await jsonImporter.import({
-                name: 'posts',
-                data: baseData.posts
-            });
-            await postsImporter.addNewsletters({posts});
-            posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title').from('posts');
-
-            tags = await jsonImporter.import({
-                name: 'tags',
-                data: baseData.tags
-            });
-
-            products = await jsonImporter.import({
-                name: 'products',
-                data: baseData.products,
-                rows: ['name', 'monthly_price', 'yearly_price']
-            });
-
-            benefits = await jsonImporter.import({
-                name: 'benefits',
-                data: baseData.benefits
-            });
-            await jsonImporter.import({
-                name: 'products_benefits',
-                data: baseData.products_benefits
-            });
-
-            stripeProducts = await jsonImporter.import({
-                name: 'stripe_products',
-                data: baseData.stripe_products,
-                rows: ['product_id', 'stripe_product_id']
-            });
-            stripePrices = await jsonImporter.import({
-                name: 'stripe_prices',
-                data: baseData.stripe_prices,
-                rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
-            });
-
-            labels = await jsonImporter.import({
-                name: 'labels',
-                data: baseData.labels
-            });
-
-            // Import settings
-            await transaction('settings').del();
-            await jsonImporter.import({
-                name: 'settings',
-                data: baseData.settings
-            });
-            await jsonImporter.import({
-                name: 'custom_theme_settings',
-                data: baseData.custom_theme_settings
-            });
-
-            this.logger.info('Completed base data import');
-        } else {
-            this.logger.info('No base data pack specified, starting random base data generation');
-            const newslettersImporter = new NewslettersImporter(transaction);
-            // First newsletter is paid, second is free
-            newsletters = await newslettersImporter.import({amount: 2, rows: ['sort_order']});
-            newsletters.sort((a, b) => a.sort_order - b.sort_order);
-
-            const postsImporter = new PostsImporter(transaction, {
-                newsletters
-            });
-            posts = await postsImporter.import({
-                amount: this.modelQuantities.posts,
-                rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type']
-            });
-            posts.push(...await postsImporter.import({
-                amount: 3,
-                type: 'page',
-                rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type']
-            }));
-
-            const tagsImporter = new TagsImporter(transaction, {
-                users
-            });
-            tags = await tagsImporter.import({amount: faker.datatype.number({
-                min: 16,
-                max: 24
-            })});
-
-            const productsImporter = new ProductsImporter(transaction);
-            products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
-
-            const stripeProductsImporter = new StripeProductsImporter(transaction);
-            stripeProducts = await stripeProductsImporter.importForEach(products.filter(product => product.name !== 'Free'), {
-                amount: 1,
-                rows: ['product_id', 'stripe_product_id']
-            });
-
-            const stripePricesImporter = new StripePricesImporter(transaction, {products});
-            stripePrices = await stripePricesImporter.importForEach(stripeProducts, {
-                amount: 2,
-                rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
-            });
-
-            await productsImporter.addStripePrices({
-                products,
-                stripeProducts,
-                stripePrices
-            });
-
-            const benefitsImporter = new BenefitsImporter(transaction);
-            benefits = await benefitsImporter.import({amount: 5});
-
-            const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits});
-            // Up to 5 benefits for each product
-            await productsBenefitsImporter.importForEach(products, {amount: 5});
-
-            const labelsImporter = new LabelsImporter(transaction);
-            labels = await labelsImporter.import({amount: 10});
-
-            this.logger.info('Completed random base data generation');
-        }
-
-        this.logger.info('Started member data generation');
-
-        const postsTagsImporter = new PostsTagsImporter(transaction, {
-            tags
-        });
-        await postsTagsImporter.importForEach(posts, {
-            amount: () => faker.datatype.number({
-                min: 0,
-                max: 3
-            })
-        });
-
-        const membersImporter = new MembersImporter(transaction);
-        const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email', 'uuid']});
-
-        const postsAuthorsImporter = new PostsAuthorsImporter(transaction, {
-            users
-        });
-        await postsAuthorsImporter.importForEach(posts, {amount: 1});
-
-        // TODO: Use subscriptions to generate members_products table?
-        const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name !== 'Free')});
-        const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), {
-            amount: 1,
-            rows: ['product_id', 'member_id']
-        });
-        const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name === 'Free')});
-        await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), {
-            amount: 1,
-            rows: ['product_id', 'member_id']
-        });
-
-        const postsProductsImporter = new PostsProductsImporter(transaction, {products: products.slice(1)});
-        // Paid newsletters
-        await postsProductsImporter.importForEach(posts.filter(post => newsletters.findIndex(newsletter => newsletter.id === post.newsletter_id) === 0), {
-            // Each post is available on all 3 premium products
-            amount: 3
-        });
-
-        const membersCreatedEventsImporter = new MembersCreatedEventsImporter(transaction, {posts});
-        await membersCreatedEventsImporter.importForEach(members, {amount: 1});
-
-        const membersLoginEventsImporter = new MembersLoginEventsImporter(transaction);
-        // Will create roughly 1 login event for every 3 days, up to a maximum of 100.
-        await membersLoginEventsImporter.importForEach(members, {amount: this.modelQuantities.membersLoginEvents});
-
-        const membersStatusEventsImporter = new MembersStatusEventsImporter(transaction);
-        // Up to 2 events per member - 1 from null -> free, 1 from free -> {paid, comped}
-        await membersStatusEventsImporter.importForEach(members, {amount: 2});
-
-        const subscriptionsImporter = new SubscriptionsImporter(transaction, {members, stripeProducts, stripePrices});
-        const subscriptions = await subscriptionsImporter.importForEach(membersProducts, {
-            amount: 1,
-            rows: ['cadence', 'tier_id', 'expires_at', 'created_at', 'member_id', 'currency']
-        });
-
-        const membersStripeCustomersImporter = new MembersStripeCustomersImporter(transaction);
-        const membersStripeCustomers = await membersStripeCustomersImporter.importForEach(members, {
-            amount: 1,
-            rows: ['customer_id', 'member_id']
-        });
-
-        const membersStripeCustomersSubscriptionsImporter = new MembersStripeCustomersSubscriptionsImporter(transaction, {
-            membersStripeCustomers,
-            products,
-            stripeProducts,
-            stripePrices
-        });
-        const membersStripeCustomersSubscriptions = await membersStripeCustomersSubscriptionsImporter.importForEach(subscriptions, {
-            amount: 1,
-            rows: ['mrr', 'plan_id', 'ghost_subscription_id']
-        });
-
-        const membersSubscribeEventsImporter = new MembersSubscribeEventsImporter(transaction, {newsletters, subscriptions});
-        const membersSubscribeEvents = await membersSubscribeEventsImporter.importForEach(members, {
-            amount: 2,
-            rows: ['member_id', 'newsletter_id', 'created_at']
-        });
-
-        const membersNewslettersImporter = new MembersNewslettersImporter(transaction);
-        await membersNewslettersImporter.importForEach(membersSubscribeEvents, {amount: 1});
-
-        const membersPaidSubscriptionEventsImporter = new MembersPaidSubscriptionEventsImporter(transaction, {
-            membersStripeCustomersSubscriptions
-        });
-        await membersPaidSubscriptionEventsImporter.importForEach(subscriptions, {amount: 1});
-
-        const membersSubscriptionCreatedEventsImporter = new MembersSubscriptionCreatedEventsImporter(transaction, {subscriptions, posts});
-        await membersSubscriptionCreatedEventsImporter.importForEach(membersStripeCustomersSubscriptions, {amount: 1});
-
-        const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl});
-        // Generate up to 4 webmentions per post
-        await mentionsImporter.importForEach(posts, {amount: 4});
-
-        const emailsImporter = new EmailsImporter(transaction, {newsletters, members, membersSubscribeEvents});
-        const emails = await emailsImporter.importForEach(posts.filter(post => post.newsletter_id), {
-            amount: 1,
-            rows: ['created_at', 'email_count', 'delivered_count', 'opened_count', 'failed_count', 'newsletter_id', 'post_id']
-        });
-
-        const emailBatchesImporter = new EmailBatchesImporter(transaction);
-        const emailBatches = await emailBatchesImporter.importForEach(emails, {
-            amount: 1,
-            rows: ['email_id', 'updated_at']
-        });
-
-        const emailRecipientsImporter = new EmailRecipientsImporter(transaction, {emailBatches, members, membersSubscribeEvents});
-        const emailRecipients = await emailRecipientsImporter.importForEach(emails, {
-            amount: this.modelQuantities.members,
-            rows: ['opened_at', 'email_id', 'member_id']
-        });
-
-        await membersImporter.addOpenRate({emailRecipients});
-
-        const redirectsImporter = new RedirectsImporter(transaction);
-        const redirects = await redirectsImporter.importForEach(posts.filter(post => post.newsletter_id), {
-            amount: 10,
-            rows: ['post_id']
-        });
-
-        const membersClickEventsImporter = new MembersClickEventsImporter(transaction, {redirects, emails});
-        await membersClickEventsImporter.importForEach(emailRecipients, {amount: 2});
-
-        const offersImporter = new OffersImporter(transaction, {products: products.filter(product => product.name !== 'Free')});
-        await offersImporter.import({amount: 2});
-
-        const membersLabelsImporter = new MembersLabelsImporter(transaction, {labels});
-        await membersLabelsImporter.importForEach(members, {
-            amount: 1
-        });
-
-        const roles = await transaction.select('id', 'name').from('roles');
-        const rolesUsersImporter = new RolesUsersImporter(transaction, {roles});
-        await rolesUsersImporter.importForEach(users, {amount: 1});
-
-        const membersFeedbackImporter = new MembersFeedbackImporter(transaction, {emails});
-        await membersFeedbackImporter.importForEach(emailRecipients, {amount: 1});
-
-        await transaction.commit();
-
-        this.logger.info('Completed member data generation');
-        this.logger.ok('Completed import process.');
-    }
-
-    async importSingleTable(tableName, quantity) {
-        this.logger.info('Importing a single table');
-        const transaction = await this.knex.transaction();
-
-        const importMembers = async () => {
-            this.logger.info(`Importing ${quantity ?? this.modelQuantities.members} members`);
-            const membersImporter = new MembersImporter(transaction);
-            await membersImporter.import({amount: quantity ?? this.modelQuantities.members});
-        };
-
-        const importMentions = async () => {
-            const posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility').from('posts');
-            this.logger.info(`Importing up to ${posts.length * 4} mentions`);
-
-            const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl});
-            await mentionsImporter.importForEach(posts, {amount: 4});
-        };
-
-        switch (tableName) {
-        case 'members':
-            await importMembers();
-            break;
-        case 'mentions':
-            await importMentions();
-            break;
-        default:
-            this.logger.warn(`Cannot import single table '${tableName}'`);
-            await transaction.commit(); // no-op, just close the transaction
-            return;
-        }
-
-        this.logger.ok('Completed import process.');
-        await transaction.commit();
-    }
-}
-
-module.exports = DataGenerator;
-module.exports.tables = tables;

ghost/data-generator/lib/importers/BenefitsImporter.js

@@ -1,13 +1,15 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const {blogStartDate} = require('../utils/blog-info');
 
 class BenefitsImporter extends TableImporter {
     static table = 'benefits';
+    static dependencies = [];
+    defaultQuantity = 5;
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(BenefitsImporter.table, knex);
+        super(BenefitsImporter.table, knex, transaction);
     }
 
     generate() {

ghost/data-generator/lib/importers/EmailBatchesImporter.js

@@ -1,16 +1,20 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const dateToDatabaseString = require('../utils/database-date');
 
 class EmailBatchesImporter extends TableImporter {
     static table = 'email_batches';
+    static dependencies = ['emails'];
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(EmailBatchesImporter.table, knex);
+        super(EmailBatchesImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const emails = await this.transaction.select('id', 'created_at').from('emails');
+
+        // TODO: Generate >1 batch per email
+        await this.importForEach(emails, quantity ?? emails.length);
     }
 
     generate() {

ghost/data-generator/lib/importers/EmailRecipientsImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const generateEvents = require('../utils/event-generator');
 const dateToDatabaseString = require('../utils/database-date');
@@ -12,15 +12,30 @@ const emailStatus = {
 
 class EmailRecipientsImporter extends TableImporter {
     static table = 'email_recipients';
+    static dependencies = ['emails', 'email_batches', 'members', 'members_subscribe_events'];
 
-    constructor(knex, {emailBatches, members, membersSubscribeEvents}) {
+    constructor(knex, transaction) {
-        super(EmailRecipientsImporter.table, knex);
+        super(EmailRecipientsImporter.table, knex, transaction);
-        this.emailBatches = emailBatches;
-        this.members = members;
-        this.membersSubscribeEvents = membersSubscribeEvents;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
+        const emails = await this.transaction
+            .select(
+                'id',
+                'newsletter_id',
+                'email_count',
+                'delivered_count',
+                'opened_count',
+                'failed_count')
+            .from('emails');
+        this.emailBatches = await this.transaction.select('id', 'email_id', 'updated_at').from('email_batches');
+        this.members = await this.transaction.select('id', 'uuid', 'email', 'name').from('members');
+        this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at', 'member_id').from('members_subscribe_events');
+
+        await this.importForEach(emails, quantity ? quantity / emails.length : this.members.length);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
         this.batch = this.emailBatches.find(batch => batch.email_id === model.id);
         // Shallow clone members list so we can shuffle and modify it

ghost/data-generator/lib/importers/EmailsImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const generateEvents = require('../utils/event-generator');
 const {luck} = require('../utils/random');
@@ -6,16 +6,18 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class EmailsImporter extends TableImporter {
     static table = 'emails';
+    static dependencies = ['posts', 'newsletters', 'members_subscribe_events'];
 
-    constructor(knex, {newsletters, members, membersSubscribeEvents}) {
+    constructor(knex, transaction) {
-        super(EmailsImporter.table, knex);
+        super(EmailsImporter.table, knex, transaction);
-        this.newsletters = newsletters;
-        this.members = members;
-        this.membersSubscribeEvents = membersSubscribeEvents;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const posts = await this.transaction.select('id', 'title', 'published_at').from('posts').where('type', 'post');
+        this.newsletters = await this.transaction.select('id').from('newsletters');
+        this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at').from('members_subscribe_events');
+
+        await this.importForEach(posts, quantity ? quantity / posts.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/LabelsImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const {blogStartDate} = require('../utils/blog-info');
@@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class LabelsImporter extends TableImporter {
     static table = 'labels';
+    static dependencies = [];
+    defaultQuantity = 10;
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(LabelsImporter.table, knex);
+        super(LabelsImporter.table, knex, transaction);
         this.generatedNames = new Set();
     }
 

ghost/data-generator/lib/importers/MembersClickEventsImporter.js

@@ -1,30 +1,37 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 const dateToDatabaseString = require('../utils/database-date');
 
 class MembersClickEventsImporter extends TableImporter {
     static table = 'members_click_events';
+    static dependencies = ['email_recipients', 'redirects', 'emails'];
 
-    constructor(knex, {redirects, emails}) {
+    constructor(knex, transaction) {
-        super(MembersClickEventsImporter.table, knex);
+        super(MembersClickEventsImporter.table, knex, transaction);
-
-        this.redirects = redirects;
-        this.emails = emails;
     }
 
-    setImportOptions({model, amount}) {
+    async import(quantity) {
+        const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients');
+        this.redirects = await this.transaction.select('id', 'post_id').from('redirects');
+        this.emails = await this.transaction.select('id', 'post_id').from('emails');
+        this.quantity = quantity ? quantity / emailRecipients.length : 2;
+
+        await this.importForEach(emailRecipients, this.quantity);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
         this.amount = model.opened_at === null ? 0 : luck(40) ? faker.datatype.number({
             min: 0,
-            max: amount
+            max: this.quantity
         }) : 0;
         const email = this.emails.find(e => e.id === this.model.email_id);
         this.redirectList = this.redirects.filter(redirect => redirect.post_id === email.post_id);
     }
 
     generate() {
-        if (this.amount <= 0) {
+        if (this.amount <= 0 || this.redirectList.length === 0) {
             return;
         }
         this.amount -= 1;

ghost/data-generator/lib/importers/MembersCreatedEventsImporter.js

@@ -1,20 +1,20 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 
 class MembersCreatedEventsImporter extends TableImporter {
     static table = 'members_created_events';
+    static dependencies = ['members', 'posts'];
 
-    constructor(knex, {posts}) {
+    constructor(knex, transaction) {
-        super(MembersCreatedEventsImporter.table, knex);
+        super(MembersCreatedEventsImporter.table, knex, transaction);
-
-        this.posts = [...posts];
-        // Sort posts in reverse chronologoical order
-        this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf());
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const members = await this.transaction.select('id', 'created_at').from('members');
+        this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
     }
 
     generateSource() {

ghost/data-generator/lib/importers/MembersFeedbackImporter.js

@@ -1,18 +1,22 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 const dateToDatabaseString = require('../utils/database-date');
 
 class MembersFeedbackImporter extends TableImporter {
     static table = 'members_feedback';
+    static dependencies = ['emails', 'email_recipients'];
 
-    constructor(knex, {emails}) {
+    constructor(knex, transaction, {emails}) {
-        super(MembersFeedbackImporter.table, knex);
+        super(MembersFeedbackImporter.table, knex, transaction);
         this.emails = emails;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients');
+        this.emails = await this.transaction.select('id', 'post_id').from('emails');
+
+        await this.importForEach(emailRecipients, quantity ? quantity / emailRecipients.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/MembersImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {faker: americanFaker} = require('@faker-js/faker/locale/en_US');
 const {blogStartDate: startTime} = require('../utils/blog-info');
@@ -8,22 +8,34 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class MembersImporter extends TableImporter {
     static table = 'members';
+    static dependencies = [];
+    defaultQuantity = faker.datatype.number({
+        min: 7000,
+        max: 8000
+    });
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(MembersImporter.table, knex);
+        super(MembersImporter.table, knex, transaction);
     }
 
-    setImportOptions({amount}) {
+    async import(quantity = this.defaultQuantity) {
         this.timestamps = generateEvents({
             shape: 'ease-in',
             trend: 'positive',
-            total: amount,
+            total: quantity,
             startTime,
             endTime: new Date()
         }).sort();
+
+        await super.import(quantity);
     }
 
-    async addOpenRate({emailRecipients}) {
+    /**
+     * Add open rate data to members table
+     */
+    async finalise() {
+        const emailRecipients = await this.transaction.select('id', 'member_id', 'opened_at').from('email_recipients');
+
         const memberData = {};
         for (const emailRecipient of emailRecipients) {
             if (!(emailRecipient.member_id in memberData)) {
@@ -41,7 +53,7 @@ class MembersImporter extends TableImporter {
 
         for (const [memberId, emailInfo] of Object.entries(memberData)) {
             const openRate = Math.round(100 * (emailInfo.openedCount / emailInfo.emailCount));
-            await this.knex('members').update({
+            await this.transaction('members').update({
                 email_count: emailInfo.emailCount,
                 email_opened_count: emailInfo.openedCount,
                 email_open_rate: emailInfo.emailCount >= 5 ? openRate : null

ghost/data-generator/lib/importers/MembersLabelsImporter.js

@@ -1,17 +1,21 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 
 class MembersLabelsImporter extends TableImporter {
     static table = 'members_labels';
+    static dependencies = ['labels', 'members'];
 
-    constructor(knex, {labels}) {
+    constructor(knex, transaction, {labels}) {
-        super(MembersLabelsImporter.table, knex);
+        super(MembersLabelsImporter.table, knex, transaction);
         this.labels = labels;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const members = await this.transaction.select('id').from('members');
+        this.labels = await this.transaction.select('id').from('labels');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
     }
 
     generate() {
@@ -19,6 +23,7 @@ class MembersLabelsImporter extends TableImporter {
             // 90% of members don't have labels
             return;
         }
+        // TODO: Ensure we don't generate the same member label twice
         return {
             id: faker.database.mongodbObjectId(),
             member_id: this.model.id,

ghost/data-generator/lib/importers/MembersLoginEventsImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 const generateEvents = require('../utils/event-generator');
@@ -6,13 +6,21 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class MembersLoginEventsImporter extends TableImporter {
     static table = 'members_login_events';
+    static dependencies = ['members'];
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(MembersLoginEventsImporter.table, knex);
+        super(MembersLoginEventsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
+        const members = await this.transaction.select('id', 'created_at').from('members');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 5);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
+
         const endDate = new Date();
         const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24));
 

ghost/data-generator/lib/importers/MembersNewslettersImporter.js

@@ -0,0 +1,27 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+
+class MembersNewslettersImporter extends TableImporter {
+    static table = 'members_newsletters';
+    static dependencies = ['members_subscribe_events'];
+
+    constructor(knex, transaction) {
+        super(MembersNewslettersImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const membersSubscribeEvents = await this.transaction.select('member_id', 'newsletter_id').from('members_subscribe_events');
+
+        await this.importForEach(membersSubscribeEvents, quantity ? quantity / membersSubscribeEvents.length : 1);
+    }
+
+    generate() {
+        return {
+            id: faker.database.mongodbObjectId(),
+            member_id: this.model.member_id,
+            newsletter_id: this.model.newsletter_id
+        };
+    }
+}
+
+module.exports = MembersNewslettersImporter;

ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js

@@ -1,16 +1,19 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 
 class MembersPaidSubscriptionEventsImporter extends TableImporter {
     static table = 'members_paid_subscription_events';
+    static dependencies = ['subscriptions', 'members_stripe_customers_subscriptions'];
 
-    constructor(knex, {membersStripeCustomersSubscriptions}) {
+    constructor(knex, transaction) {
-        super(MembersPaidSubscriptionEventsImporter.table, knex);
+        super(MembersPaidSubscriptionEventsImporter.table, knex, transaction);
-        this.membersStripeCustomersSubscriptions = membersStripeCustomersSubscriptions;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const subscriptions = await this.transaction.select('id', 'member_id', 'currency', 'created_at').from('subscriptions');
+        this.membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id', 'plan_id', 'mrr').from('members_stripe_customers_subscriptions');
+
+        await this.importForEach(subscriptions, quantity ? quantity / subscriptions.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/MembersProductsImporter.js

@@ -1,16 +1,20 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {luck} = require('../utils/random');
 
 class MembersProductsImporter extends TableImporter {
     static table = 'members_products';
-    constructor(knex, {products}) {
+    static dependencies = ['products', 'members'];
-        super(MembersProductsImporter.table, knex);
+
-        this.products = products;
+    constructor(knex, transaction) {
+        super(MembersProductsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const members = await this.transaction.select('id').from('members').whereNot('status', 'free');
+        this.products = await this.transaction.select('id').from('products').whereNot('name', 'Free');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
     }
 
     getProduct() {

ghost/data-generator/lib/importers/MembersStatusEventsImporter.js

@@ -1,15 +1,22 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const dateToDatabaseString = require('../utils/database-date');
 
 class MembersStatusEventsImporter extends TableImporter {
     static table = 'members_status_events';
+    static dependencies = ['members'];
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(MembersStatusEventsImporter.table, knex);
+        super(MembersStatusEventsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
+        const members = await this.transaction.select('id', 'created_at', 'status').from('members');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 2);
+    }
+
+    setReferencedModel(model) {
         this.events = [{
             id: faker.database.mongodbObjectId(),
             member_id: model.id,

ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js

@@ -1,15 +1,18 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class MembersStripeCustomersImporter extends TableImporter {
     static table = 'members_stripe_customers';
+    static dependencies = ['members'];
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(MembersStripeCustomersImporter.table, knex);
+        super(MembersStripeCustomersImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const members = await this.transaction.select('id', 'name', 'email', 'created_at').from('members');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js

@@ -1,19 +1,22 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
     static table = 'members_stripe_customers_subscriptions';
+    static dependencies = ['subscriptions', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices'];
 
-    constructor(knex, {membersStripeCustomers, products, stripeProducts, stripePrices}) {
+    constructor(knex, transaction) {
-        super(MembersStripeCustomersSubscriptionsImporter.table, knex);
+        super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction);
-        this.membersStripeCustomers = membersStripeCustomers;
-        this.products = products;
-        this.stripeProducts = stripeProducts;
-        this.stripePrices = stripePrices;
     }
 
-    setImportOptions({model}) {
+    async import() {
-        this.model = model;
+        const subscriptions = await this.transaction.select('id', 'member_id', 'tier_id', 'cadence', 'created_at', 'expires_at').from('subscriptions');
+        this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers');
+        this.products = await this.transaction.select('id', 'name').from('products');
+        this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
+        this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices');
+
+        await this.importForEach(subscriptions, 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js

@@ -1,17 +1,25 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 const dateToDatabaseString = require('../utils/database-date');
 
 class MembersSubscribeEventsImporter extends TableImporter {
     static table = 'members_subscribe_events';
-    constructor(knex, {newsletters, subscriptions}) {
+    static dependencies = ['members', 'newsletters', 'subscriptions'];
-        super(MembersSubscribeEventsImporter.table, knex);
+
-        this.newsletters = newsletters;
+    constructor(knex, transaction) {
-        this.subscriptions = subscriptions;
+        super(MembersSubscribeEventsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
+        const members = await this.transaction.select('id', 'created_at', 'status').from('members');
+        this.newsletters = await this.transaction.select('id').from('newsletters');
+        this.subscriptions = await this.transaction.select('member_id', 'created_at').from('subscriptions');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 2);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
         this.count = 0;
     }

ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js

@@ -1,21 +1,21 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {luck} = require('../utils/random');
 
 class MembersSubscriptionCreatedEventsImporter extends TableImporter {
     static table = 'members_subscription_created_events';
+    static dependencies = ['members_stripe_customers_subscriptions', 'subscriptions', 'posts'];
 
-    constructor(knex, {subscriptions, posts}) {
+    constructor(knex, transaction) {
-        super(MembersSubscriptionCreatedEventsImporter.table, knex);
+        super(MembersSubscriptionCreatedEventsImporter.table, knex, transaction);
-        this.subscriptions = subscriptions;
-
-        this.posts = [...posts];
-        // Sort posts in reverse chronologoical order
-        this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf());
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id').from('members_stripe_customers_subscriptions');
+        this.subscriptions = await this.transaction.select('id', 'created_at', 'member_id').from('subscriptions');
+        this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
+
+        await this.importForEach(membersStripeCustomersSubscriptions, quantity ? quantity / membersStripeCustomersSubscriptions.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/NewslettersImporter.js

@@ -1,14 +1,17 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {blogStartDate} = require('../utils/blog-info');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 
 class NewslettersImporter extends TableImporter {
     static table = 'newsletters';
+    static dependencies = [];
+    defaultQuantity = 2;
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(NewslettersImporter.table, knex);
+        super(NewslettersImporter.table, knex, transaction);
         this.sortOrder = 0;
+        // TODO: Use random names if we ever need more than 2 newsletters
         this.names = ['Regular premium', 'Occasional freebie'];
     }
 

ghost/data-generator/lib/importers/OffersImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const {blogStartDate} = require('../utils/blog-info');
@@ -6,15 +6,19 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class OffersImporter extends TableImporter {
     static table = 'offers';
+    static dependencies = ['products'];
+    defaultQuantity = 2;
 
-    constructor(knex, {products}) {
+    constructor(knex, transaction) {
-        super(OffersImporter.table, knex);
+        super(OffersImporter.table, knex, transaction);
-        this.products = products;
     }
 
-    setImportOptions() {
+    async import(quantity = this.defaultQuantity) {
+        this.products = await this.transaction.select('id', 'currency').from('products');
         this.names = ['Black Friday', 'Free Trial'];
         this.count = 0;
+
+        await super.import(quantity);
     }
 
     generate() {

ghost/data-generator/lib/importers/PostsAuthorsImporter.js

@@ -1,17 +1,20 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class PostsAuthorsImporter extends TableImporter {
     static table = 'posts_authors';
+    static dependencies = ['posts', 'users'];
 
-    constructor(knex, {users}) {
+    constructor(knex, transaction) {
-        super(PostsAuthorsImporter.table, knex);
+        super(PostsAuthorsImporter.table, knex, transaction);
-        this.users = users;
         this.sortOrder = 0;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.model = model;
+        const posts = await this.transaction.select('id').from('posts');
+        this.users = await this.transaction.select('id').from('users');
+
+        await this.importForEach(posts, quantity ? quantity / posts.length : 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/PostsImporter.js

@@ -1,27 +1,27 @@
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const {luck} = require('../utils/random');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const dateToDatabaseString = require('../utils/database-date');
 
 class PostsImporter extends TableImporter {
     static table = 'posts';
+    static dependencies = ['newsletters'];
+    defaultQuantity = faker.datatype.number({
+        min: 80,
+        max: 120
+    });
 
-    constructor(knex, {newsletters}) {
+    type = 'post';
-        super(PostsImporter.table, knex);
+
-        this.newsletters = newsletters;
+    constructor(knex, transaction) {
+        super(PostsImporter.table, knex, transaction);
     }
 
-    setImportOptions({type = 'post'}) {
+    async import(quantity = this.defaultQuantity) {
-        this.type = type;
+        this.newsletters = await this.transaction.select('id').from('newsletters');
-    }
 
-    async addNewsletters({posts}) {
+        await super.import(quantity);
-        for (const {id, visibility} of posts) {
-            await this.knex('posts').update({
-                newsletter_id: luck(90) ? (visibility === 'paid' ? this.newsletters[0].id : this.newsletters[1].id) : null
-            }).where({id, type: 'post', status: 'published'});
-        }
     }
 
     generate() {

ghost/data-generator/lib/importers/PostsProductsImporter.js

@@ -1,17 +1,24 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class PostsProductsImporter extends TableImporter {
     static table = 'posts_products';
+    static dependencies = ['posts', 'products'];
 
-    constructor(knex, {products}) {
+    constructor(knex, transaction) {
-        super(PostsProductsImporter.table, knex);
+        super(PostsProductsImporter.table, knex, transaction);
-        this.products = products;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.sortOrder = 0;
+        const posts = await this.transaction.select('id').from('posts').where('type', 'post');
+        this.products = await this.transaction.select('id').from('products');
+
+        await this.importForEach(posts, quantity ? quantity / posts.length : 1);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
+        this.sortOrder = 0;
     }
 
     generate() {

ghost/data-generator/lib/importers/PostsTagsImporter.js

@@ -1,18 +1,28 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class PostsTagsImporter extends TableImporter {
     static table = 'posts_tags';
-    constructor(knex, {tags}) {
+    static dependencies = ['posts', 'tags'];
-        super(PostsTagsImporter.table, knex);
+
-        this.tags = tags;
+    constructor(knex, transaction) {
+        super(PostsTagsImporter.table, knex, transaction);
         this.sortOrder = 0;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
-        this.notIndex = [];
+        const posts = await this.transaction.select('id').from('posts').where('type', 'post');
-        this.sortOrder = 0;
+        this.tags = await this.transaction.select('id').from('tags');
+
+        await this.importForEach(posts, quantity ? quantity / posts.length : () => faker.datatype.number({
+            min: 0,
+            max: 3
+        }));
+    }
+
+    setReferencedModel(model) {
         this.model = model;
+        this.notIndex = [];
     }
 
     generate() {

ghost/data-generator/lib/importers/ProductsBenefitsImporter.js

@@ -1,17 +1,24 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class ProductsBenefitsImporter extends TableImporter {
     static table = 'products_benefits';
+    static dependencies = ['benefits', 'products'];
 
-    constructor(knex, {benefits}) {
+    constructor(knex, transaction) {
-        super(ProductsBenefitsImporter.table, knex);
+        super(ProductsBenefitsImporter.table, knex, transaction);
-        this.benefits = benefits;
-        this.sortOrder = 0;
     }
 
-    setImportOptions({model}) {
+    async import(quantity) {
+        const products = await this.transaction.select('id', 'name').from('products');
+        this.benefits = await this.transaction.select('id').from('benefits');
+
+        await this.importForEach(products, quantity ? quantity / products.length : 5);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
+
         this.sortOrder = 0;
         switch (this.model.name) {
         case 'Bronze':

ghost/data-generator/lib/importers/ProductsImporter.js

@@ -1,21 +1,34 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const {blogStartDate} = require('../utils/blog-info');
 
 class ProductsImporter extends TableImporter {
     static table = 'products';
+    static dependencies = [];
+    defaultQuantity = 4;
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(ProductsImporter.table, knex);
+        super(ProductsImporter.table, knex, transaction);
     }
 
-    setImportOptions() {
+    async import(quantity = this.defaultQuantity) {
+        // TODO: Add random products if quantity != 4
         this.names = ['Free', 'Bronze', 'Silver', 'Gold'];
         this.count = 0;
+
+        await super.import(quantity);
     }
 
-    async addStripePrices({products, stripeProducts, stripePrices}) {
+    /**
+     * Add the stripe products / prices
+     */
+    async finalise() {
+        const stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
+        const stripePrices = await this.transaction.select('id', 'stripe_product_id', 'interval').from('stripe_prices');
+
+        const products = await this.transaction.select('id').from('products');
+
         for (const {id} of products) {
             const stripeProduct = stripeProducts.find(p => id === p.product_id);
             if (!stripeProduct) {
@@ -40,7 +53,7 @@ class ProductsImporter extends TableImporter {
             }
 
             if (Object.keys(update).length > 0) {
-                await this.knex('products').update(update).where({
+                await this.transaction('products').update(update).where({
                     id
                 });
             }
@@ -59,7 +72,7 @@ class ProductsImporter extends TableImporter {
         };
         if (count !== 0) {
             tierInfo.type = 'paid';
-            tierInfo.description = `${name} star member`;
+            tierInfo.description = `${name} tier member`;
             tierInfo.currency = 'USD';
             tierInfo.monthly_price = count * 500;
             tierInfo.yearly_price = count * 5000;

ghost/data-generator/lib/importers/RedirectsImporter.js

@@ -1,18 +1,32 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 
 class RedirectsImporter extends TableImporter {
     static table = 'redirects';
+    static dependencies = ['posts'];
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(RedirectsImporter.table, knex);
+        super(RedirectsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model, amount}) {
+    async import(quantity) {
+        const posts = await this.transaction
+            .select('id', 'published_at')
+            .from('posts')
+            .where('type', 'post')
+            .andWhere('status', 'published');
+
+        this.quantity = quantity ? quantity / posts.length : 10;
+        await this.importForEach(posts, this.quantity);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
+
+        // Reset the amount for each model
         this.amount = faker.datatype.number({
-            min: 1,
+            min: 0,
-            max: amount
+            max: this.quantity
         });
     }
 

ghost/data-generator/lib/importers/RolesUsersImporter.js

@@ -1,17 +1,23 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 
 class RolesUsersImporter extends TableImporter {
     static table = 'roles_users';
+    // No roles imorter, since roles are statically defined in database
+    static dependencies = ['users'];
 
-    constructor(knex, {roles}) {
+    constructor(knex, transaction) {
-        super(RolesUsersImporter.table, knex);
+        super(RolesUsersImporter.table, knex, transaction);
-        this.roles = roles;
-        this.sortOrder = 0;
     }
 
-    setImportOptions({model}) {
+    /**
-        this.model = model;
+     * Ignore overriden quantity for 1:1 relationship
+     */
+    async import() {
+        const users = await this.transaction.select('id').from('users').whereNot('id', 1);
+        this.roles = await this.transaction.select('id', 'name').from('roles');
+
+        await this.importForEach(users, 1);
     }
 
     generate() {

ghost/data-generator/lib/importers/StripePricesImporter.js

@@ -1,25 +1,31 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {blogStartDate} = require('../utils/blog-info');
 
+const sixWeeksLater = new Date(blogStartDate);
+sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
+
 class StripePricesImporter extends TableImporter {
     static table = 'stripe_prices';
+    static dependencies = ['products', 'stripe_products'];
 
-    constructor(knex, {products}) {
+    constructor(knex, transaction) {
-        super(StripePricesImporter.table, knex);
+        super(StripePricesImporter.table, knex, transaction);
-        this.products = products;
     }
 
-    setImportOptions({model}) {
+    async import() {
-        this.model = model;
+        const stripeProducts = await this.transaction.select('id', 'stripe_product_id', 'product_id').from('stripe_products');
+        this.products = await this.transaction.select('id', 'monthly_price', 'yearly_price').from('products');
 
+        await this.importForEach(stripeProducts, 2);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
         this.count = 0;
     }
 
     generate() {
-        const sixWeeksLater = new Date(blogStartDate);
-        sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
-
         const count = this.count;
         this.count = this.count + 1;
 

ghost/data-generator/lib/importers/StripeProductsImporter.js

@@ -1,20 +1,24 @@
 const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {blogStartDate} = require('../utils/blog-info');
 
+const sixWeeksLater = new Date(blogStartDate);
+sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
+
 class StripeProductsImporter extends TableImporter {
     static table = 'stripe_products';
-    constructor(knex) {
+    static dependencies = ['products'];
-        super(StripeProductsImporter.table, knex);
+
+    constructor(knex, transaction) {
+        super(StripeProductsImporter.table, knex, transaction);
     }
 
-    setImportOptions({model}) {
+    async import() {
-        this.model = model;
+        const products = await this.transaction.select('id').from('products');
+        await this.importForEach(products, 1);
     }
 
     generate() {
-        const sixWeeksLater = new Date(blogStartDate);
-        sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
         return {
             id: faker.database.mongodbObjectId(),
             product_id: this.model.id,

ghost/data-generator/lib/importers/SubscriptionsImporter.js

@@ -1,20 +1,22 @@
 const {faker} = require('@faker-js/faker');
 const generateEvents = require('../utils/event-generator');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const dateToDatabaseString = require('../utils/database-date');
 
 class SubscriptionsImporter extends TableImporter {
     static table = 'subscriptions';
+    static dependencies = ['members', 'members_products', 'stripe_products', 'stripe_prices'];
 
-    constructor(knex, {members, stripeProducts, stripePrices}) {
+    constructor(knex, transaction) {
-        super(SubscriptionsImporter.table, knex);
+        super(SubscriptionsImporter.table, knex, transaction);
-        this.members = members;
-        this.stripeProducts = stripeProducts;
-        this.stripePrices = stripePrices;
     }
 
-    setImportOptions({model}) {
+    async import() {
-        this.model = model;
+        const membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products');
+        this.members = await this.transaction.select('id', 'status', 'created_at').from('members');
+        this.stripeProducts = await this.transaction.select('product_id', 'stripe_product_id').from('stripe_products');
+        this.stripePrices = await this.transaction.select('stripe_product_id', 'currency', 'amount', 'interval').from('stripe_prices');
+        await this.importForEach(membersProducts, 1);
     }
 
     generate() {
@@ -28,6 +30,7 @@ class SubscriptionsImporter extends TableImporter {
                 return price.stripe_product_id === stripeProduct.stripe_product_id &&
                     (isMonthly ? price.interval === 'month' : price.interval === 'year');
             });
+
             billingInfo.cadence = isMonthly ? 'month' : 'year';
             billingInfo.currency = stripePrice.currency;
             billingInfo.amount = stripePrice.amount;

ghost/data-generator/lib/importers/TableImporter.js

@@ -0,0 +1,108 @@
+class TableImporter {
+    /**
+     * @type {object|undefined} model Referenced model when generating data
+     */
+    model;
+
+    /**
+     * @type {number|undefined} defaultQuantity Default number of records to import
+     */
+    defaultQuantity;
+
+    /**
+     * Transaction and knex need to be separate since we're using the batchInsert helper
+     * @param {string} name Name of the table to be generated
+     * @param {import('knex/types').Knex} knex Database connection
+     * @param {import('knex/types').Knex.Transaction} transaction Transaction to be used for import
+     */
+    constructor(name, knex, transaction) {
+        this.name = name;
+        this.knex = knex;
+        this.transaction = transaction;
+    }
+
+    async import(amount = this.defaultQuantity) {
+        const batchSize = 500;
+        let batch = [];
+
+        for (let i = 0; i < amount; i++) {
+            const model = await this.generate();
+            if (model) {
+                batch.push(model);
+            } else {
+                // After first null assume that there is no more data
+                break;
+            }
+            if (batch.length === batchSize) {
+                await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
+                batch = [];
+            }
+        }
+
+        // Process final batch
+        if (batch.length > 0) {
+            await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
+        }
+    }
+
+    /**
+     * @param {Array<Object>} models List of models to reference
+     * @param {Number|function} amount Number of records to import per model
+     */
+    async importForEach(models = [], amount) {
+        const batchSize = 500;
+        let batch = [];
+
+        for (const model of models) {
+            this.setReferencedModel(model);
+            let currentAmount = (typeof amount === 'function') ? amount() : amount;
+            if (!Number.isInteger(currentAmount)) {
+                currentAmount = Math.floor(currentAmount) + ((Math.random() < currentAmount % 1) ? 1 : 0);
+            }
+            for (let i = 0; i < currentAmount; i++) {
+                const data = await this.generate();
+                if (data) {
+                    batch.push(data);
+                } else {
+                    // After first null assume that there is no more data for this model
+                    break;
+                }
+                if (batch.length === batchSize) {
+                    await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
+                    batch = [];
+                }
+            }
+        }
+
+        // Process final batch
+        if (batch.length > 0) {
+            await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
+        }
+    }
+
+    /**
+     * Finalise the imported data, e.g. adding summary records based on a table's dependents
+     */
+    async finalise() {
+        // No-op by default
+    }
+
+    /**
+     * Sets the model which newly generated data will reference
+     * @param {Object} model Model to reference when generating data
+     */
+    setReferencedModel(model) {
+        this.model = model;
+    }
+
+    /**
+     * Generates the data for a single model to be imported
+     * @returns {Object|null} Data to import, optional
+     */
+    generate() {
+        // Should never be called
+        return false;
+    }
+}
+
+module.exports = TableImporter;

ghost/data-generator/lib/importers/TagsImporter.js

@@ -1,14 +1,23 @@
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const dateToDatabaseString = require('../utils/database-date');
 
 class TagsImporter extends TableImporter {
     static table = 'tags';
+    static dependencies = ['users'];
+    defaultQuantity = faker.datatype.number({
+        min: 16,
+        max: 24
+    });
 
-    constructor(knex, {users}) {
+    constructor(knex, transaction) {
-        super(TagsImporter.table, knex);
+        super(TagsImporter.table, knex, transaction);
-        this.users = users;
+    }
+
+    async import(quantity = this.defaultQuantity) {
+        this.users = await this.transaction.select('id').from('users');
+        await super.import(quantity);
     }
 
     generate() {

ghost/data-generator/lib/importers/UsersImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const {slugify} = require('@tryghost/string');
 const security = require('@tryghost/security');
@@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class UsersImporter extends TableImporter {
     static table = 'users';
+    static dependencies = [];
+    defaultQuantity = 8;
 
-    constructor(knex) {
+    constructor(knex, transaction) {
-        super(UsersImporter.table, knex);
+        super(UsersImporter.table, knex, transaction);
     }
 
     async generate() {

ghost/data-generator/lib/importers/WebMentionsImporter.js

@@ -1,4 +1,4 @@
-const TableImporter = require('./base');
+const TableImporter = require('./TableImporter');
 const {faker} = require('@faker-js/faker');
 const generateEvents = require('../utils/event-generator');
 const {luck} = require('../utils/random');
@@ -6,14 +6,22 @@ const dateToDatabaseString = require('../utils/database-date');
 
 class WebMentionsImporter extends TableImporter {
     static table = 'mentions';
+    static dependencies = ['posts'];
 
-    constructor(knex, {baseUrl}) {
+    constructor(knex, transaction, {baseUrl}) {
-        super(WebMentionsImporter.table, knex);
+        super(WebMentionsImporter.table, knex, transaction);
 
         this.baseUrl = baseUrl;
     }
 
-    setImportOptions({model, amount}) {
+    async import(quantity) {
+        const posts = await this.transaction.select('id', 'slug', 'published_at').from('posts').where('type', 'post');
+
+        this.quantity = quantity ? quantity / posts.length : 4;
+        await this.importForEach(posts, this.quantity);
+    }
+
+    setReferencedModel(model) {
         this.model = model;
 
         // Most web mentions published soon after publication date
@@ -23,7 +31,7 @@ class WebMentionsImporter extends TableImporter {
         this.timestamps = generateEvents({
             shape: 'ease-out',
             trend: 'negative',
-            total: amount,
+            total: this.quantity,
             startTime: startDate,
             endTime: endDate
         }).sort();
@@ -31,7 +39,7 @@ class WebMentionsImporter extends TableImporter {
 
     generate() {
         if (luck(50)) {
-            // 50/50 chance of having a web mention
+            // 50% chance of 1 mention, 25% chance of 2 mentions, etc.
             return null;
         }
 
@@ -39,6 +47,23 @@ class WebMentionsImporter extends TableImporter {
         const timestamp = this.timestamps.shift();
 
         const author = `${faker.name.fullName()}`;
+        /**
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        source: {type: 'string', maxlength: 2000, nullable: false},
+        source_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_site_title: {type: 'string', maxlength: 2000, nullable: true},
+        source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
+        source_author: {type: 'string', maxlength: 2000, nullable: true},
+        source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
+        source_favicon: {type: 'string', maxlength: 2000, nullable: true},
+        target: {type: 'string', maxlength: 2000, nullable: false},
+        resource_id: {type: 'string', maxlength: 24, nullable: true},
+        resource_type: {type: 'string', maxlength: 50, nullable: true},
+        created_at: {type: 'dateTime', nullable: false},
+        payload: {type: 'text', maxlength: 65535, nullable: true},
+        deleted: {type: 'boolean', nullable: false, defaultTo: false},
+        verified: {type: 'boolean', nullable: false, defaultTo: false}
+         */
         return {
             id,
             source: `${faker.internet.url()}/${faker.helpers.slugify(`${faker.word.adjective()} ${faker.word.noun()}`).toLowerCase()}`,

ghost/data-generator/lib/importers/index.js

@@ -0,0 +1,37 @@
+module.exports = [
+    require('./NewslettersImporter'),
+    require('./PostsImporter'),
+    require('./UsersImporter'),
+    require('./TagsImporter'),
+    require('./ProductsImporter'),
+    require('./MembersImporter'),
+    require('./BenefitsImporter'),
+    require('./WebMentionsImporter'),
+    require('./PostsAuthorsImporter'),
+    require('./PostsTagsImporter'),
+    require('./ProductsBenefitsImporter'),
+    require('./MembersProductsImporter'),
+    require('./PostsProductsImporter'),
+    require('./MembersNewslettersImporter'),
+    require('./StripeProductsImporter'),
+    require('./StripePricesImporter'),
+    require('./SubscriptionsImporter'),
+    require('./EmailsImporter'),
+    require('./EmailBatchesImporter'),
+    require('./EmailRecipientsImporter'),
+    require('./RedirectsImporter'),
+    require('./MembersClickEventsImporter'),
+    require('./OffersImporter'),
+    require('./MembersCreatedEventsImporter'),
+    require('./MembersLoginEventsImporter'),
+    require('./MembersStatusEventsImporter'),
+    require('./MembersStripeCustomersImporter'),
+    require('./MembersStripeCustomersSubscriptionsImporter'),
+    require('./MembersPaidSubscriptionEventsImporter'),
+    require('./MembersSubscriptionCreatedEventsImporter'),
+    require('./MembersSubscribeEventsImporter'),
+    require('./LabelsImporter'),
+    require('./MembersLabelsImporter'),
+    require('./RolesUsersImporter'),
+    require('./MembersFeedbackImporter')
+];

ghost/data-generator/lib/tables/base.js

@@ -1,89 +0,0 @@
-class TableImporter {
-    /**
-     * @param {string} name Name of the table to be generated
-     * @param {import('knex/types').Knex} knex Database connection
-     */
-    constructor(name, knex) {
-        this.name = name;
-        this.knex = knex;
-    }
-
-    /**
-     * @typedef {Function} AmountFunction
-     * @returns {number}
-     */
-
-    /**
-     * @typedef {Object.<string,any>} ImportOptions
-     * @property {number|AmountFunction} amount Number of events to generate
-     * @property {Object} [model] Used to reference another object during creation
-     */
-
-    /**
-     * @param {Array<Object>} models List of models to reference
-     * @param {ImportOptions} [options] Import options
-     * @returns {Promise<Array<Object>>}
-     */
-    async importForEach(models = [], options) {
-        const results = [];
-        for (const model of models) {
-            results.push(...await this.import(Object.assign({}, options, {model})));
-        }
-        return results;
-    }
-
-    /**
-     * @param {ImportOptions} options Import options
-     * @returns {Promise<Array<Object>>}
-     */
-    async import(options) {
-        if (options.amount === 0) {
-            return;
-        }
-
-        // Use dynamic amount if faker function given
-        const amount = (typeof options.amount === 'function') ? options.amount() : options.amount;
-
-        this.setImportOptions(Object.assign({}, options, {amount}));
-
-        const data = [];
-        for (let i = 0; i < amount; i++) {
-            const model = await this.generate();
-            if (model) {
-                // Only push models when one is generated successfully
-                data.push(model);
-            } else {
-                // After first null assume that there is no more data
-                break;
-            }
-        }
-
-        const rows = ['id'];
-        if (options && options.rows) {
-            rows.push(...options.rows);
-        }
-        await this.knex.batchInsert(this.name, data, 500);
-        return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(this.name);
-    }
-
-    /**
-     *
-     * @param {ImportOptions} options
-     * @returns {void}
-     */
-    // eslint-disable-next-line no-unused-vars
-    setImportOptions(options) {
-        return;
-    }
-
-    /**
-     * Generates the data for a single model to be imported
-     * @returns {Object|null} Data to import, optional
-     */
-    generate() {
-        // Should never be called
-        return false;
-    }
-}
-
-module.exports = TableImporter;

ghost/data-generator/lib/tables/index.js

@@ -1,38 +0,0 @@
-// Order matters! Ordered so that dependant tables are after their dependencies
-module.exports = {
-    NewslettersImporter: require('./newsletters'),
-    PostsImporter: require('./posts'),
-    UsersImporter: require('./users'),
-    TagsImporter: require('./tags'),
-    ProductsImporter: require('./products'),
-    MembersImporter: require('./members'),
-    BenefitsImporter: require('./benefits'),
-    MentionsImporter: require('./mentions'),
-    PostsAuthorsImporter: require('./posts-authors'),
-    PostsTagsImporter: require('./posts-tags'),
-    ProductsBenefitsImporter: require('./products-benefits'),
-    MembersProductsImporter: require('./members-products'),
-    PostsProductsImporter: require('./posts-products'),
-    MembersNewslettersImporter: require('./members-newsletters'),
-    StripeProductsImporter: require('./stripe-products'),
-    StripePricesImporter: require('./stripe-prices'),
-    SubscriptionsImporter: require('./subscriptions'),
-    EmailsImporter: require('./emails'),
-    EmailBatchesImporter: require('./email-batches'),
-    EmailRecipientsImporter: require('./email-recipients'),
-    RedirectsImporter: require('./redirects'),
-    MembersClickEventsImporter: require('./members-click-events'),
-    OffersImporter: require('./offers'),
-    MembersCreatedEventsImporter: require('./members-created-events'),
-    MembersLoginEventsImporter: require('./members-login-events'),
-    MembersStatusEventsImporter: require('./members-status-events'),
-    MembersStripeCustomersImporter: require('./members-stripe-customers'),
-    MembersStripeCustomersSubscriptionsImporter: require('./members-stripe-customers-subscriptions'),
-    MembersPaidSubscriptionEventsImporter: require('./members-paid-subscription-events'),
-    MembersSubscriptionCreatedEventsImporter: require('./members-subscription-created-events'),
-    MembersSubscribeEventsImporter: require('./members-subscribe-events'),
-    LabelsImporter: require('./labels'),
-    MembersLabelsImporter: require('./members-labels'),
-    RolesUsersImporter: require('./roles-users'),
-    MembersFeedbackImporter: require('./members-feedback')
-};

ghost/data-generator/lib/tables/members-newsletters.js

@@ -1,24 +0,0 @@
-const {faker} = require('@faker-js/faker');
-const TableImporter = require('./base');
-
-class MembersNewslettersImporter extends TableImporter {
-    static table = 'members_newsletters';
-
-    constructor(knex) {
-        super(MembersNewslettersImporter.table, knex);
-    }
-
-    setImportOptions({model}) {
-        this.model = model;
-    }
-
-    generate() {
-        return {
-            id: faker.database.mongodbObjectId(),
-            member_id: this.model.member_id,
-            newsletter_id: this.model.newsletter_id
-        };
-    }
-}
-
-module.exports = MembersNewslettersImporter;

ghost/data-generator/lib/utils/JsonImporter.js

@@ -1,8 +1,9 @@
 const {faker} = require('@faker-js/faker');
 
 class JsonImporter {
-    constructor(knex) {
+    constructor(knex, transaction) {
         this.knex = knex;
+        this.transaction = transaction;
     }
 
     /**
@@ -15,7 +16,7 @@ class JsonImporter {
     /**
      * Import a dataset to the database
      * @param {JsonImportOptions} options
-     * @returns {Promise<Array<Object.<string, any>>>} Set of rows returned from database
+     * @returns {Promise}
      */
     async import({
         name,
@@ -30,8 +31,7 @@ class JsonImporter {
         if (rows.findIndex(row => row === 'id') === -1) {
             rows.unshift('id');
         }
-        await this.knex.batchInsert(name, data, 500);
+        await this.knex.batchInsert(name, data, 500).transacting(this.transaction);
-        return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(name);
     }
 }
 

ghost/data-generator/lib/utils/topological-sort.js

@@ -0,0 +1,33 @@
+/**
+ * This sorting algorithm is used to make sure that dependent tables are imported after their dependencies.
+ * @param {Array<Object>} objects Objects with a name and dependencies properties
+ * @returns Topologically sorted array of objects
+ */
+module.exports = function topologicalSort(objects) {
+    // Create an empty result array to store the ordered objects
+    const result = [];
+    // Create a set to track visited objects during the DFS
+    const visited = new Set();
+
+    // Helper function to perform DFS
+    function dfs(name) {
+        if (visited.has(name)) {
+            return;
+        }
+
+        visited.add(name);
+        const dependencies = objects.find(item => item.name === name)?.dependencies || [];
+        for (const dependency of dependencies) {
+            dfs(dependency);
+        }
+
+        result.push(objects.find(item => item.name === name));
+    }
+
+    // Perform DFS on each object
+    for (const object of objects) {
+        dfs(object.name);
+    }
+
+    return result;
+};

ghost/data-generator/test/data-generator.test.js

@@ -2,11 +2,10 @@
 // const testUtils = require('./utils');
 require('./utils');
 const knex = require('knex');
-const {
+const importers = require('../lib/importers');
-    ProductsImporter,
+const ProductsImporter = importers.find(i => i.table === 'products');
-    StripeProductsImporter,
+const StripeProductsImporter = importers.find(i => i.table === 'stripe_products');
-    StripePricesImporter
+const StripePricesImporter = importers.find(i => i.table === 'stripe_prices');
-} = require('../lib/tables');
 
 const generateEvents = require('../lib/utils/event-generator');
 
@@ -82,11 +81,14 @@ describe('Data Generator', function () {
                 info: () => { },
                 ok: () => { }
             },
-            modelQuantities: {
+            tables: [{
-                members: 10,
+                name: 'members',
-                membersLoginEvents: 5,
+                quantity: 10
-                posts: 2
+            }, {
-            }
+                name: 'posts',
+                quantity: 2
+            }],
+            withDefault: true
         });
         try {
             return await dataGenerator.importData();
@@ -152,27 +154,25 @@ describe('Importer', function () {
     });
 
     it('Should import a single item', async function () {
-        const productsImporter = new ProductsImporter(db);
+        const transaction = await db.transaction();
-        const products = await productsImporter.import({amount: 1, rows: ['name', 'monthly_price', 'yearly_price']});
+        const productsImporter = new ProductsImporter(db, transaction);
+        await productsImporter.import();
+        transaction.commit();
 
-        products.length.should.eql(1);
+        const products = await db.select('id', 'name').from('products');
+
+        products.length.should.eql(4);
         products[0].name.should.eql('Free');
-
-        const results = await db.select('id', 'name').from('products');
-
-        results.length.should.eql(1);
-        results[0].name.should.eql('Free');
     });
 
     it('Should import an item for each entry in an array', async function () {
-        const productsImporter = new ProductsImporter(db);
+        const transaction = await db.transaction();
-        const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
+        const productsImporter = new ProductsImporter(db, transaction);
+        await productsImporter.import();
 
-        const stripeProductsImporter = new StripeProductsImporter(db);
+        const stripeProductsImporter = new StripeProductsImporter(db, transaction);
-        await stripeProductsImporter.importForEach(products, {
+        await stripeProductsImporter.import();
-            amount: 1,
+        transaction.commit();
-            rows: ['product_id', 'stripe_product_id']
-        });
 
         const results = await db.select('id').from('stripe_products');
 
@@ -180,26 +180,20 @@ describe('Importer', function () {
     });
 
     it('Should update products to reference price ids', async function () {
-        const productsImporter = new ProductsImporter(db);
+        const transaction = await db.transaction();
-        const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
+        const productsImporter = new ProductsImporter(db, transaction);
+        await productsImporter.import();
 
-        const stripeProductsImporter = new StripeProductsImporter(db);
+        const stripeProductsImporter = new StripeProductsImporter(db, transaction);
-        const stripeProducts = await stripeProductsImporter.importForEach(products, {
+        await stripeProductsImporter.import();
-            amount: 1,
-            rows: ['product_id', 'stripe_product_id']
-        });
 
-        const stripePricesImporter = new StripePricesImporter(db, {products});
+        const stripePricesImporter = new StripePricesImporter(db, transaction);
-        const stripePrices = await stripePricesImporter.importForEach(stripeProducts, {
+        await stripePricesImporter.import();
-            amount: 2,
-            rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
-        });
 
-        await productsImporter.addStripePrices({
+        await productsImporter.finalise();
-            products,
+        await stripeProductsImporter.finalise();
-            stripeProducts,
+        await stripePricesImporter.finalise();
-            stripePrices
+        transaction.commit();
-        });
 
         const results = await db.select('id', 'name', 'monthly_price_id', 'yearly_price_id').from('products');