TryGhost/Ghost
1
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2024-11-22 11:16:01 +03:00
Code Issues Projects Releases Wiki Activity

Entirely rewrote data generator to simplify codebase

Browse Source 
refs: https://github.com/TryGhost/DevOps/issues/11

This is a pretty huge commit, but the relevant points are:
* Each importer no longer needs to be passed a set of data, it just gets the data it needs
* Each importer specifies its dependencies, so that the order of import can be determined at runtime using a topological sort
* The main data generator function can just tell each importer to import the data it has

This makes working on the data generator much easier.

Some other benefits are:
* Batched importing, massively speeding up the whole process
* `--tables` to set the exact tables you want to import, and specify the quantity of each
This commit is contained in:
Sam Lord 2023-08-02 14:43:26 +01:00 committed by Sam Lord
parent cf947bc4d6
commit 4ff467794f
47 changed files with 865 additions and 893 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
ghost/core/core/cli/generate-data.js
View File

@ -6,10 +6,10 @@ module.exports = class DataGeneratorCommand extends Command {
setup() {
this.help('Generates random data to populate the database for development & testing');
this.argument('--base-data-pack', {type: 'string', defaultValue: '', desc: 'Base data pack file location, imported instead of random content'});
this.argument('--scale', {type: 'string', defaultValue: 'small', desc: 'Scale of the data to generate. `small` for a quick run, `large` for more content'});
this.argument('--single-table', {type: 'string', desc: 'Import a single table'});
this.argument('--quantity', {type: 'number', desc: 'When importing a single table, the quantity to import'});
this.argument('--clear-database', {type: 'boolean', defaultValue: false, desc: 'Clear all entries in the database before importing'});
this.argument('--tables', {type: 'string', desc: 'Only import the specified list of tables, where quantities can be specified by appending a colon followed by the quantity for each table. Example: --tables=members:1000,posts,tags,members_login_events'});
this.argument('--with-default', {type: 'boolean', defaultValue: false, desc: 'Include default tables as well as those specified (simply override quantities)'});
}
initializeContext(context) {
@ -30,22 +30,15 @@ module.exports = class DataGeneratorCommand extends Command {
async handle(argv = {}) {
const knex = require('../server/data/db/connection');
const {tables: schema} = require('../server/data/schema/index');
const modelQuantities = {};
if (argv.scale) {
if (argv.scale === 'small') {
modelQuantities.members = 200;
modelQuantities.membersLoginEvents = 1;
modelQuantities.posts = 10;
}
// Defaults in data-generator package make a large set
}
const tables = (argv.tables ? argv.tables.split(',') : []).map(table => ({
name: table.split(':')[0],
quantity: parseInt(table.split(':')[1]) || undefined
}));
const dataGenerator = new DataGenerator({
baseDataPack: argv['base-data-pack'],
knex,
schema,
logger: {
log: this.log,
ok: this.ok,
@ -55,16 +48,13 @@ module.exports = class DataGeneratorCommand extends Command {
fatal: this.fatal,
debug: this.debug
},
modelQuantities,
baseUrl: config.getSiteUrl(),
clearDatabase: argv['clear-database']
clearDatabase: argv['clear-database'],
tables,
withDefault: argv['with-default']
});
try {
if (argv['single-table']) {
await dataGenerator.importSingleTable(argv['single-table'], argv.quantity ?? undefined);
} else {
await dataGenerator.importData();
}
await dataGenerator.importData();
} catch (error) {
this.fatal('Failed while generating data: ', error);
}

2
ghost/data-generator/index.js
View File

@ -1 +1 @@
module.exports = require('./lib/data-generator');
module.exports = require('./lib/DataGenerator');

190
ghost/data-generator/lib/DataGenerator.js Normal file
View File

@ -0,0 +1,190 @@
const path = require('path');
const fs = require('fs/promises');
const JsonImporter = require('./utils/JsonImporter');
const {getProcessRoot} = require('@tryghost/root-utils');
const topologicalSort = require('./utils/topological-sort');
const importers = require('./importers').reduce((acc, val) => {
acc[val.table] = val;
return acc;
}, {});
const schema = require('../../core/core/server/data/schema').tables;
class DataGenerator {
constructor({
knex,
tables,
clearDatabase = false,
baseDataPack = '',
baseUrl,
logger,
withDefault
}) {
this.knex = knex;
this.tableList = tables || [];
this.willClearData = clearDatabase;
this.useBaseDataPack = baseDataPack !== '';
this.baseDataPack = baseDataPack;
this.baseUrl = baseUrl;
this.logger = logger;
this.withDefault = withDefault;
}
sortTableList() {
// Add missing dependencies
for (const table of this.tableList) {
table.importer = importers[table.name];
// eslint-disable-next-line no-unused-vars
table.dependencies = Object.entries(schema[table.name]).reduce((acc, [_col, data]) => {
if (data.references) {
const referencedTable = data.references.split('.')[0];
if (!acc.includes(referencedTable)) {
acc.push(referencedTable);
}
}
return acc;
}, table.importer.dependencies);
for (const dependency of table.dependencies) {
if (!this.tableList.find(t => t.name === dependency)) {
this.tableList.push({
name: dependency,
importer: importers[dependency]
});
}
}
}
// Order to ensure dependencies are created before dependants
this.tableList = topologicalSort(this.tableList);
}
/**
* TODO: This needs to reverse through all dependency chains to clear data from all tables
* @param {import('knex/types').Knex.Transaction} transaction
*/
async clearData(transaction) {
const tables = this.tableList.map(t => t.name).reverse();
// TODO: Remove this once we import posts_meta
tables.unshift('posts_meta');
// Clear data from any tables that are being imported
for (const table of tables) {
this.logger.debug(`Clearing table ${table}`);
if (table === 'roles_users') {
await transaction(table).del().whereNot('user_id', '1');
} else if (table === 'users') {
// Avoid deleting the admin user
await transaction(table).del().whereNot('id', '1');
} else {
await transaction(table).del();
}
}
}
async importBasePack(transaction) {
let baseDataPack = this.baseDataPack;
if (!path.isAbsolute(this.baseDataPack)) {
baseDataPack = path.join(getProcessRoot(), baseDataPack);
}
let baseData = {};
try {
baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString());
this.logger.info('Read base data pack');
} catch (error) {
this.logger.error('Failed to read data pack: ', error);
throw error;
}
this.logger.info('Starting base data import');
const jsonImporter = new JsonImporter(transaction);
// Clear settings table
await transaction('settings').del();
// Hard-coded for order
const tablesToImport = [
'newsletters',
'posts',
'tags',
'products',
'benefits',
'products_benefits',
'stripe_products',
'stripe_prices',
'settings',
'custom_theme_settings'
];
for (const table of tablesToImport) {
this.logger.info(`Importing content for table ${table} from base data pack`);
await jsonImporter.import({
name: table,
data: baseData[table]
});
const tableIndex = this.tableList.findIndex(t => t.name === table);
if (tableIndex !== -1) {
this.tableList.splice(tableIndex, 1);
}
}
this.logger.info('Completed base data import');
}
async importData() {
const transaction = await this.knex.transaction();
// Add default tables if none are specified
if (this.tableList.length === 0) {
this.tableList = Object.keys(importers).map(name => ({name}));
} else if (this.withDefault) {
// Add default tables to the end of the list
const defaultTables = Object.keys(importers).map(name => ({name}));
for (const table of defaultTables) {
if (!this.tableList.find(t => t.name === table.name)) {
this.tableList.push(table);
}
}
}
// Error if we have an unknown table
for (const table of this.tableList) {
if (importers[table.name] === undefined) {
// eslint-disable-next-line
throw new Error(`Unknown table: ${table.name}`);
}
}
this.sortTableList();
if (this.willClearData) {
await this.clearData(transaction);
}
if (this.useBaseDataPack) {
await this.importBasePack(transaction);
}
for (const table of this.tableList) {
this.logger.info('Importing content for table', table.name);
// Add all common options to every importer, whether they use them or not
const tableImporter = new table.importer(this.knex, transaction, {
baseUrl: this.baseUrl
});
await tableImporter.import(table.quantity ?? undefined);
}
// Finalise all tables - uses new table importer objects to avoid keeping all data in memory
for (const table of this.tableList) {
const tableImporter = new table.importer(this.knex, transaction, {
baseUrl: this.baseUrl
});
await tableImporter.finalise();
}
await transaction.commit();
}
}
module.exports = DataGenerator;

461
ghost/data-generator/lib/data-generator.js
View File

@ -1,461 +0,0 @@
const tables = require('./tables');
// Order here does not matter
const {
NewslettersImporter,
PostsImporter,
UsersImporter,
TagsImporter,
ProductsImporter,
MembersImporter,
BenefitsImporter,
MentionsImporter,
PostsAuthorsImporter,
PostsTagsImporter,
ProductsBenefitsImporter,
MembersProductsImporter,
PostsProductsImporter,
MembersNewslettersImporter,
StripeProductsImporter,
StripePricesImporter,
SubscriptionsImporter,
EmailsImporter,
MembersCreatedEventsImporter,
MembersLoginEventsImporter,
MembersStatusEventsImporter,
MembersSubscribeEventsImporter,
MembersStripeCustomersImporter,
MembersPaidSubscriptionEventsImporter,
EmailBatchesImporter,
EmailRecipientsImporter,
RedirectsImporter,
MembersClickEventsImporter,
OffersImporter,
MembersStripeCustomersSubscriptionsImporter,
MembersSubscriptionCreatedEventsImporter,
LabelsImporter,
MembersLabelsImporter,
RolesUsersImporter,
MembersFeedbackImporter
} = tables;
const path = require('path');
const fs = require('fs/promises');
const {faker} = require('@faker-js/faker');
const JsonImporter = require('./utils/json-importer');
const {getProcessRoot} = require('@tryghost/root-utils');
/**
* @typedef {Object} DataGeneratorOptions
* @property {string} baseDataPack
* @property {import('knex/types').Knex} knex
* @property {Object} schema
* @property {Object} logger
* @property {Object} modelQuantities
* @property {string} baseUrl
* @property {boolean} clearDatabase
*/
const defaultQuantities = {
members: () => faker.datatype.number({
min: 7000,
max: 8000
}),
// This will generate n * <members> events, is not worth being very high
membersLoginEvents: 5,
posts: () => faker.datatype.number({
min: 80,
max: 120
})
};
class DataGenerator {
/**
*
* @param {DataGeneratorOptions} options
*/
constructor({
baseDataPack = '',
knex,
schema,
logger,
modelQuantities = {},
baseUrl,
clearDatabase
}) {
this.useBaseData = baseDataPack !== '';
this.baseDataPack = baseDataPack;
this.knex = knex;
this.schema = schema;
this.logger = logger;
this.modelQuantities = Object.assign({}, defaultQuantities, modelQuantities);
this.baseUrl = baseUrl;
this.clearDatabase = clearDatabase;
}
async importData() {
const transaction = await this.knex.transaction();
if (this.clearDatabase) {
this.logger.info('Clearing existing database');
// List of tables ordered to avoid dependencies when deleting
const tableNames = Object.values(tables).map(importer => importer.table).reverse();
// We don't currently generate posts_meta, but we need to clear it to ensure posts can be removed
tableNames.unshift('posts_meta');
for (const table of tableNames) {
this.logger.debug(`Clearing table ${table}`);
if (table === 'roles_users') {
await transaction(table).del().whereNot('user_id', '1');
continue;
}
if (table === 'users') {
// Avoid deleting the admin user
await transaction(table).del().whereNot('id', '1');
continue;
}
await transaction(table).del();
}
this.logger.info('Finished clearing database');
}
this.logger.info('Starting import process, this has two parts: base data and member data. It can take a while...');
const usersImporter = new UsersImporter(transaction);
const users = await usersImporter.import({amount: 8});
let newsletters;
let posts;
let tags;
let products;
let stripeProducts;
let stripePrices;
let benefits;
let labels;
// Use an existant set of data for a more realisitic looking site
if (this.useBaseData) {
let baseDataPack = this.baseDataPack;
if (!path.isAbsolute(this.baseDataPack)) {
baseDataPack = path.join(getProcessRoot(), baseDataPack);
}
let baseData = {};
try {
baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString());
this.logger.info('Read base data pack');
} catch (error) {
this.logger.error('Failed to read data pack: ', error);
throw error;
}
this.logger.info('Starting base data import');
const jsonImporter = new JsonImporter(transaction);
// Must have at least 2 in base data set
newsletters = await jsonImporter.import({
name: 'newsletters',
data: baseData.newsletters,
rows: ['sort_order']
});
newsletters.sort((a, b) => a.sort_order - b.sort_order);
const postsImporter = new PostsImporter(transaction, {
newsletters
});
posts = await jsonImporter.import({
name: 'posts',
data: baseData.posts
});
await postsImporter.addNewsletters({posts});
posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title').from('posts');
tags = await jsonImporter.import({
name: 'tags',
data: baseData.tags
});
products = await jsonImporter.import({
name: 'products',
data: baseData.products,
rows: ['name', 'monthly_price', 'yearly_price']
});
benefits = await jsonImporter.import({
name: 'benefits',
data: baseData.benefits
});
await jsonImporter.import({
name: 'products_benefits',
data: baseData.products_benefits
});
stripeProducts = await jsonImporter.import({
name: 'stripe_products',
data: baseData.stripe_products,
rows: ['product_id', 'stripe_product_id']
});
stripePrices = await jsonImporter.import({
name: 'stripe_prices',
data: baseData.stripe_prices,
rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
});
labels = await jsonImporter.import({
name: 'labels',
data: baseData.labels
});
// Import settings
await transaction('settings').del();
await jsonImporter.import({
name: 'settings',
data: baseData.settings
});
await jsonImporter.import({
name: 'custom_theme_settings',
data: baseData.custom_theme_settings
});
this.logger.info('Completed base data import');
} else {
this.logger.info('No base data pack specified, starting random base data generation');
const newslettersImporter = new NewslettersImporter(transaction);
// First newsletter is paid, second is free
newsletters = await newslettersImporter.import({amount: 2, rows: ['sort_order']});
newsletters.sort((a, b) => a.sort_order - b.sort_order);
const postsImporter = new PostsImporter(transaction, {
newsletters
});
posts = await postsImporter.import({
amount: this.modelQuantities.posts,
rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type']
});
posts.push(...await postsImporter.import({
amount: 3,
type: 'page',
rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type']
}));
const tagsImporter = new TagsImporter(transaction, {
users
});
tags = await tagsImporter.import({amount: faker.datatype.number({
min: 16,
max: 24
})});
const productsImporter = new ProductsImporter(transaction);
products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
const stripeProductsImporter = new StripeProductsImporter(transaction);
stripeProducts = await stripeProductsImporter.importForEach(products.filter(product => product.name !== 'Free'), {
amount: 1,
rows: ['product_id', 'stripe_product_id']
});
const stripePricesImporter = new StripePricesImporter(transaction, {products});
stripePrices = await stripePricesImporter.importForEach(stripeProducts, {
amount: 2,
rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
});
await productsImporter.addStripePrices({
products,
stripeProducts,
stripePrices
});
const benefitsImporter = new BenefitsImporter(transaction);
benefits = await benefitsImporter.import({amount: 5});
const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits});
// Up to 5 benefits for each product
await productsBenefitsImporter.importForEach(products, {amount: 5});
const labelsImporter = new LabelsImporter(transaction);
labels = await labelsImporter.import({amount: 10});
this.logger.info('Completed random base data generation');
}
this.logger.info('Started member data generation');
const postsTagsImporter = new PostsTagsImporter(transaction, {
tags
});
await postsTagsImporter.importForEach(posts, {
amount: () => faker.datatype.number({
min: 0,
max: 3
})
});
const membersImporter = new MembersImporter(transaction);
const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email', 'uuid']});
const postsAuthorsImporter = new PostsAuthorsImporter(transaction, {
users
});
await postsAuthorsImporter.importForEach(posts, {amount: 1});
// TODO: Use subscriptions to generate members_products table?
const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name !== 'Free')});
const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), {
amount: 1,
rows: ['product_id', 'member_id']
});
const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name === 'Free')});
await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), {
amount: 1,
rows: ['product_id', 'member_id']
});
const postsProductsImporter = new PostsProductsImporter(transaction, {products: products.slice(1)});
// Paid newsletters
await postsProductsImporter.importForEach(posts.filter(post => newsletters.findIndex(newsletter => newsletter.id === post.newsletter_id) === 0), {
// Each post is available on all 3 premium products
amount: 3
});
const membersCreatedEventsImporter = new MembersCreatedEventsImporter(transaction, {posts});
await membersCreatedEventsImporter.importForEach(members, {amount: 1});
const membersLoginEventsImporter = new MembersLoginEventsImporter(transaction);
// Will create roughly 1 login event for every 3 days, up to a maximum of 100.
await membersLoginEventsImporter.importForEach(members, {amount: this.modelQuantities.membersLoginEvents});
const membersStatusEventsImporter = new MembersStatusEventsImporter(transaction);
// Up to 2 events per member - 1 from null -> free, 1 from free -> {paid, comped}
await membersStatusEventsImporter.importForEach(members, {amount: 2});
const subscriptionsImporter = new SubscriptionsImporter(transaction, {members, stripeProducts, stripePrices});
const subscriptions = await subscriptionsImporter.importForEach(membersProducts, {
amount: 1,
rows: ['cadence', 'tier_id', 'expires_at', 'created_at', 'member_id', 'currency']
});
const membersStripeCustomersImporter = new MembersStripeCustomersImporter(transaction);
const membersStripeCustomers = await membersStripeCustomersImporter.importForEach(members, {
amount: 1,
rows: ['customer_id', 'member_id']
});
const membersStripeCustomersSubscriptionsImporter = new MembersStripeCustomersSubscriptionsImporter(transaction, {
membersStripeCustomers,
products,
stripeProducts,
stripePrices
});
const membersStripeCustomersSubscriptions = await membersStripeCustomersSubscriptionsImporter.importForEach(subscriptions, {
amount: 1,
rows: ['mrr', 'plan_id', 'ghost_subscription_id']
});
const membersSubscribeEventsImporter = new MembersSubscribeEventsImporter(transaction, {newsletters, subscriptions});
const membersSubscribeEvents = await membersSubscribeEventsImporter.importForEach(members, {
amount: 2,
rows: ['member_id', 'newsletter_id', 'created_at']
});
const membersNewslettersImporter = new MembersNewslettersImporter(transaction);
await membersNewslettersImporter.importForEach(membersSubscribeEvents, {amount: 1});
const membersPaidSubscriptionEventsImporter = new MembersPaidSubscriptionEventsImporter(transaction, {
membersStripeCustomersSubscriptions
});
await membersPaidSubscriptionEventsImporter.importForEach(subscriptions, {amount: 1});
const membersSubscriptionCreatedEventsImporter = new MembersSubscriptionCreatedEventsImporter(transaction, {subscriptions, posts});
await membersSubscriptionCreatedEventsImporter.importForEach(membersStripeCustomersSubscriptions, {amount: 1});
const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl});
// Generate up to 4 webmentions per post
await mentionsImporter.importForEach(posts, {amount: 4});
const emailsImporter = new EmailsImporter(transaction, {newsletters, members, membersSubscribeEvents});
const emails = await emailsImporter.importForEach(posts.filter(post => post.newsletter_id), {
amount: 1,
rows: ['created_at', 'email_count', 'delivered_count', 'opened_count', 'failed_count', 'newsletter_id', 'post_id']
});
const emailBatchesImporter = new EmailBatchesImporter(transaction);
const emailBatches = await emailBatchesImporter.importForEach(emails, {
amount: 1,
rows: ['email_id', 'updated_at']
});
const emailRecipientsImporter = new EmailRecipientsImporter(transaction, {emailBatches, members, membersSubscribeEvents});
const emailRecipients = await emailRecipientsImporter.importForEach(emails, {
amount: this.modelQuantities.members,
rows: ['opened_at', 'email_id', 'member_id']
});
await membersImporter.addOpenRate({emailRecipients});
const redirectsImporter = new RedirectsImporter(transaction);
const redirects = await redirectsImporter.importForEach(posts.filter(post => post.newsletter_id), {
amount: 10,
rows: ['post_id']
});
const membersClickEventsImporter = new MembersClickEventsImporter(transaction, {redirects, emails});
await membersClickEventsImporter.importForEach(emailRecipients, {amount: 2});
const offersImporter = new OffersImporter(transaction, {products: products.filter(product => product.name !== 'Free')});
await offersImporter.import({amount: 2});
const membersLabelsImporter = new MembersLabelsImporter(transaction, {labels});
await membersLabelsImporter.importForEach(members, {
amount: 1
});
const roles = await transaction.select('id', 'name').from('roles');
const rolesUsersImporter = new RolesUsersImporter(transaction, {roles});
await rolesUsersImporter.importForEach(users, {amount: 1});
const membersFeedbackImporter = new MembersFeedbackImporter(transaction, {emails});
await membersFeedbackImporter.importForEach(emailRecipients, {amount: 1});
await transaction.commit();
this.logger.info('Completed member data generation');
this.logger.ok('Completed import process.');
}
async importSingleTable(tableName, quantity) {
this.logger.info('Importing a single table');
const transaction = await this.knex.transaction();
const importMembers = async () => {
this.logger.info(`Importing ${quantity ?? this.modelQuantities.members} members`);
const membersImporter = new MembersImporter(transaction);
await membersImporter.import({amount: quantity ?? this.modelQuantities.members});
};
const importMentions = async () => {
const posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility').from('posts');
this.logger.info(`Importing up to ${posts.length * 4} mentions`);
const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl});
await mentionsImporter.importForEach(posts, {amount: 4});
};
switch (tableName) {
case 'members':
await importMembers();
break;
case 'mentions':
await importMentions();
break;
default:
this.logger.warn(`Cannot import single table '${tableName}'`);
await transaction.commit(); // no-op, just close the transaction
return;
}
this.logger.ok('Completed import process.');
await transaction.commit();
}
}
module.exports = DataGenerator;
module.exports.tables = tables;

8
ghost/data-generator/lib/tables/benefits.js → ghost/data-generator/lib/importers/BenefitsImporter.js
View File

@ -1,13 +1,15 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const {blogStartDate} = require('../utils/blog-info');
class BenefitsImporter extends TableImporter {
static table = 'benefits';
static dependencies = [];
defaultQuantity = 5;
constructor(knex) {
super(BenefitsImporter.table, knex);
constructor(knex, transaction) {
super(BenefitsImporter.table, knex, transaction);
}
generate() {

14
ghost/data-generator/lib/tables/email-batches.js → ghost/data-generator/lib/importers/EmailBatchesImporter.js
View File

@ -1,16 +1,20 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const dateToDatabaseString = require('../utils/database-date');
class EmailBatchesImporter extends TableImporter {
static table = 'email_batches';
static dependencies = ['emails'];
constructor(knex) {
super(EmailBatchesImporter.table, knex);
constructor(knex, transaction) {
super(EmailBatchesImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const emails = await this.transaction.select('id', 'created_at').from('emails');
// TODO: Generate >1 batch per email
await this.importForEach(emails, quantity ?? emails.length);
}
generate() {

29
ghost/data-generator/lib/tables/email-recipients.js → ghost/data-generator/lib/importers/EmailRecipientsImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const generateEvents = require('../utils/event-generator');
const dateToDatabaseString = require('../utils/database-date');
@ -12,15 +12,30 @@ const emailStatus = {
class EmailRecipientsImporter extends TableImporter {
static table = 'email_recipients';
static dependencies = ['emails', 'email_batches', 'members', 'members_subscribe_events'];
constructor(knex, {emailBatches, members, membersSubscribeEvents}) {
super(EmailRecipientsImporter.table, knex);
this.emailBatches = emailBatches;
this.members = members;
this.membersSubscribeEvents = membersSubscribeEvents;
constructor(knex, transaction) {
super(EmailRecipientsImporter.table, knex, transaction);
}
setImportOptions({model}) {
async import(quantity) {
const emails = await this.transaction
.select(
'id',
'newsletter_id',
'email_count',
'delivered_count',
'opened_count',
'failed_count')
.from('emails');
this.emailBatches = await this.transaction.select('id', 'email_id', 'updated_at').from('email_batches');
this.members = await this.transaction.select('id', 'uuid', 'email', 'name').from('members');
this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at', 'member_id').from('members_subscribe_events');
await this.importForEach(emails, quantity ? quantity / emails.length : this.members.length);
}
setReferencedModel(model) {
this.model = model;
this.batch = this.emailBatches.find(batch => batch.email_id === model.id);
// Shallow clone members list so we can shuffle and modify it

18
ghost/data-generator/lib/tables/emails.js → ghost/data-generator/lib/importers/EmailsImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const generateEvents = require('../utils/event-generator');
const {luck} = require('../utils/random');
@ -6,16 +6,18 @@ const dateToDatabaseString = require('../utils/database-date');
class EmailsImporter extends TableImporter {
static table = 'emails';
static dependencies = ['posts', 'newsletters', 'members_subscribe_events'];
constructor(knex, {newsletters, members, membersSubscribeEvents}) {
super(EmailsImporter.table, knex);
this.newsletters = newsletters;
this.members = members;
this.membersSubscribeEvents = membersSubscribeEvents;
constructor(knex, transaction) {
super(EmailsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const posts = await this.transaction.select('id', 'title', 'published_at').from('posts').where('type', 'post');
this.newsletters = await this.transaction.select('id').from('newsletters');
this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at').from('members_subscribe_events');
await this.importForEach(posts, quantity ? quantity / posts.length : 1);
}
generate() {

8
ghost/data-generator/lib/tables/labels.js → ghost/data-generator/lib/importers/LabelsImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const {blogStartDate} = require('../utils/blog-info');
@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date');
class LabelsImporter extends TableImporter {
static table = 'labels';
static dependencies = [];
defaultQuantity = 10;
constructor(knex) {
super(LabelsImporter.table, knex);
constructor(knex, transaction) {
super(LabelsImporter.table, knex, transaction);
this.generatedNames = new Set();
}

25
ghost/data-generator/lib/tables/members-click-events.js → ghost/data-generator/lib/importers/MembersClickEventsImporter.js
View File

@ -1,30 +1,37 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');
class MembersClickEventsImporter extends TableImporter {
static table = 'members_click_events';
static dependencies = ['email_recipients', 'redirects', 'emails'];
constructor(knex, {redirects, emails}) {
super(MembersClickEventsImporter.table, knex);
this.redirects = redirects;
this.emails = emails;
constructor(knex, transaction) {
super(MembersClickEventsImporter.table, knex, transaction);
}
setImportOptions({model, amount}) {
async import(quantity) {
const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients');
this.redirects = await this.transaction.select('id', 'post_id').from('redirects');
this.emails = await this.transaction.select('id', 'post_id').from('emails');
this.quantity = quantity ? quantity / emailRecipients.length : 2;
await this.importForEach(emailRecipients, this.quantity);
}
setReferencedModel(model) {
this.model = model;
this.amount = model.opened_at === null ? 0 : luck(40) ? faker.datatype.number({
min: 0,
max: amount
max: this.quantity
}) : 0;
const email = this.emails.find(e => e.id === this.model.email_id);
this.redirectList = this.redirects.filter(redirect => redirect.post_id === email.post_id);
}
generate() {
if (this.amount <= 0) {
if (this.amount <= 0 || this.redirectList.length === 0) {
return;
}
this.amount -= 1;

18
ghost/data-generator/lib/tables/members-created-events.js → ghost/data-generator/lib/importers/MembersCreatedEventsImporter.js
View File

@ -1,20 +1,20 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
class MembersCreatedEventsImporter extends TableImporter {
static table = 'members_created_events';
static dependencies = ['members', 'posts'];
constructor(knex, {posts}) {
super(MembersCreatedEventsImporter.table, knex);
this.posts = [...posts];
// Sort posts in reverse chronologoical order
this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf());
constructor(knex, transaction) {
super(MembersCreatedEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const members = await this.transaction.select('id', 'created_at').from('members');
this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
await this.importForEach(members, quantity ? quantity / members.length : 1);
}
generateSource() {

14
ghost/data-generator/lib/tables/members-feedback.js → ghost/data-generator/lib/importers/MembersFeedbackImporter.js
View File

@ -1,18 +1,22 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');
class MembersFeedbackImporter extends TableImporter {
static table = 'members_feedback';
static dependencies = ['emails', 'email_recipients'];
constructor(knex, {emails}) {
super(MembersFeedbackImporter.table, knex);
constructor(knex, transaction, {emails}) {
super(MembersFeedbackImporter.table, knex, transaction);
this.emails = emails;
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients');
this.emails = await this.transaction.select('id', 'post_id').from('emails');
await this.importForEach(emailRecipients, quantity ? quantity / emailRecipients.length : 1);
}
generate() {

26
ghost/data-generator/lib/tables/members.js → ghost/data-generator/lib/importers/MembersImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {faker: americanFaker} = require('@faker-js/faker/locale/en_US');
const {blogStartDate: startTime} = require('../utils/blog-info');
@ -8,22 +8,34 @@ const dateToDatabaseString = require('../utils/database-date');
class MembersImporter extends TableImporter {
static table = 'members';
static dependencies = [];
defaultQuantity = faker.datatype.number({
min: 7000,
max: 8000
});
constructor(knex) {
super(MembersImporter.table, knex);
constructor(knex, transaction) {
super(MembersImporter.table, knex, transaction);
}
setImportOptions({amount}) {
async import(quantity = this.defaultQuantity) {
this.timestamps = generateEvents({
shape: 'ease-in',
trend: 'positive',
total: amount,
total: quantity,
startTime,
endTime: new Date()
}).sort();
await super.import(quantity);
}
async addOpenRate({emailRecipients}) {
/**
* Add open rate data to members table
*/
async finalise() {
const emailRecipients = await this.transaction.select('id', 'member_id', 'opened_at').from('email_recipients');
const memberData = {};
for (const emailRecipient of emailRecipients) {
if (!(emailRecipient.member_id in memberData)) {
@ -41,7 +53,7 @@ class MembersImporter extends TableImporter {
for (const [memberId, emailInfo] of Object.entries(memberData)) {
const openRate = Math.round(100 * (emailInfo.openedCount / emailInfo.emailCount));
await this.knex('members').update({
await this.transaction('members').update({
email_count: emailInfo.emailCount,
email_opened_count: emailInfo.openedCount,
email_open_rate: emailInfo.emailCount >= 5 ? openRate : null

15
ghost/data-generator/lib/tables/members-labels.js → ghost/data-generator/lib/importers/MembersLabelsImporter.js
View File

@ -1,17 +1,21 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
class MembersLabelsImporter extends TableImporter {
static table = 'members_labels';
static dependencies = ['labels', 'members'];
constructor(knex, {labels}) {
super(MembersLabelsImporter.table, knex);
constructor(knex, transaction, {labels}) {
super(MembersLabelsImporter.table, knex, transaction);
this.labels = labels;
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const members = await this.transaction.select('id').from('members');
this.labels = await this.transaction.select('id').from('labels');
await this.importForEach(members, quantity ? quantity / members.length : 1);
}
generate() {
@ -19,6 +23,7 @@ class MembersLabelsImporter extends TableImporter {
// 90% of members don't have labels
return;
}
// TODO: Ensure we don't generate the same member label twice
return {
id: faker.database.mongodbObjectId(),
member_id: this.model.id,

16
ghost/data-generator/lib/tables/members-login-events.js → ghost/data-generator/lib/importers/MembersLoginEventsImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const generateEvents = require('../utils/event-generator');
@ -6,13 +6,21 @@ const dateToDatabaseString = require('../utils/database-date');
class MembersLoginEventsImporter extends TableImporter {
static table = 'members_login_events';
static dependencies = ['members'];
constructor(knex) {
super(MembersLoginEventsImporter.table, knex);
constructor(knex, transaction) {
super(MembersLoginEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
async import(quantity) {
const members = await this.transaction.select('id', 'created_at').from('members');
await this.importForEach(members, quantity ? quantity / members.length : 5);
}
setReferencedModel(model) {
this.model = model;
const endDate = new Date();
const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24));

27
ghost/data-generator/lib/importers/MembersNewslettersImporter.js Normal file
View File

@ -0,0 +1,27 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./TableImporter');
class MembersNewslettersImporter extends TableImporter {
static table = 'members_newsletters';
static dependencies = ['members_subscribe_events'];
constructor(knex, transaction) {
super(MembersNewslettersImporter.table, knex, transaction);
}
async import(quantity) {
const membersSubscribeEvents = await this.transaction.select('member_id', 'newsletter_id').from('members_subscribe_events');
await this.importForEach(membersSubscribeEvents, quantity ? quantity / membersSubscribeEvents.length : 1);
}
generate() {
return {
id: faker.database.mongodbObjectId(),
member_id: this.model.member_id,
newsletter_id: this.model.newsletter_id
};
}
}
module.exports = MembersNewslettersImporter;

15
ghost/data-generator/lib/tables/members-paid-subscription-events.js → ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js
View File

@ -1,16 +1,19 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
class MembersPaidSubscriptionEventsImporter extends TableImporter {
static table = 'members_paid_subscription_events';
static dependencies = ['subscriptions', 'members_stripe_customers_subscriptions'];
constructor(knex, {membersStripeCustomersSubscriptions}) {
super(MembersPaidSubscriptionEventsImporter.table, knex);
this.membersStripeCustomersSubscriptions = membersStripeCustomersSubscriptions;
constructor(knex, transaction) {
super(MembersPaidSubscriptionEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const subscriptions = await this.transaction.select('id', 'member_id', 'currency', 'created_at').from('subscriptions');
this.membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id', 'plan_id', 'mrr').from('members_stripe_customers_subscriptions');
await this.importForEach(subscriptions, quantity ? quantity / subscriptions.length : 1);
}
generate() {

16
ghost/data-generator/lib/tables/members-products.js → ghost/data-generator/lib/importers/MembersProductsImporter.js
View File

@ -1,16 +1,20 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {luck} = require('../utils/random');
class MembersProductsImporter extends TableImporter {
static table = 'members_products';
constructor(knex, {products}) {
super(MembersProductsImporter.table, knex);
this.products = products;
static dependencies = ['products', 'members'];
constructor(knex, transaction) {
super(MembersProductsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const members = await this.transaction.select('id').from('members').whereNot('status', 'free');
this.products = await this.transaction.select('id').from('products').whereNot('name', 'Free');
await this.importForEach(members, quantity ? quantity / members.length : 1);
}
getProduct() {

15
ghost/data-generator/lib/tables/members-status-events.js → ghost/data-generator/lib/importers/MembersStatusEventsImporter.js
View File

@ -1,15 +1,22 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const dateToDatabaseString = require('../utils/database-date');
class MembersStatusEventsImporter extends TableImporter {
static table = 'members_status_events';
static dependencies = ['members'];
constructor(knex) {
super(MembersStatusEventsImporter.table, knex);
constructor(knex, transaction) {
super(MembersStatusEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
async import(quantity) {
const members = await this.transaction.select('id', 'created_at', 'status').from('members');
await this.importForEach(members, quantity ? quantity / members.length : 2);
}
setReferencedModel(model) {
this.events = [{
id: faker.database.mongodbObjectId(),
member_id: model.id,

13
ghost/data-generator/lib/tables/members-stripe-customers.js → ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js
View File

@ -1,15 +1,18 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class MembersStripeCustomersImporter extends TableImporter {
static table = 'members_stripe_customers';
static dependencies = ['members'];
constructor(knex) {
super(MembersStripeCustomersImporter.table, knex);
constructor(knex, transaction) {
super(MembersStripeCustomersImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const members = await this.transaction.select('id', 'name', 'email', 'created_at').from('members');
await this.importForEach(members, quantity ? quantity / members.length : 1);
}
generate() {

21
ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js → ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js
View File

@ -1,19 +1,22 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
static table = 'members_stripe_customers_subscriptions';
static dependencies = ['subscriptions', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices'];
constructor(knex, {membersStripeCustomers, products, stripeProducts, stripePrices}) {
super(MembersStripeCustomersSubscriptionsImporter.table, knex);
this.membersStripeCustomers = membersStripeCustomers;
this.products = products;
this.stripeProducts = stripeProducts;
this.stripePrices = stripePrices;
constructor(knex, transaction) {
super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import() {
const subscriptions = await this.transaction.select('id', 'member_id', 'tier_id', 'cadence', 'created_at', 'expires_at').from('subscriptions');
this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers');
this.products = await this.transaction.select('id', 'name').from('products');
this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices');
await this.importForEach(subscriptions, 1);
}
generate() {

20
ghost/data-generator/lib/tables/members-subscribe-events.js → ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js
View File

@ -1,17 +1,25 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
const dateToDatabaseString = require('../utils/database-date');
class MembersSubscribeEventsImporter extends TableImporter {
static table = 'members_subscribe_events';
constructor(knex, {newsletters, subscriptions}) {
super(MembersSubscribeEventsImporter.table, knex);
this.newsletters = newsletters;
this.subscriptions = subscriptions;
static dependencies = ['members', 'newsletters', 'subscriptions'];
constructor(knex, transaction) {
super(MembersSubscribeEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
async import(quantity) {
const members = await this.transaction.select('id', 'created_at', 'status').from('members');
this.newsletters = await this.transaction.select('id').from('newsletters');
this.subscriptions = await this.transaction.select('member_id', 'created_at').from('subscriptions');
await this.importForEach(members, quantity ? quantity / members.length : 2);
}
setReferencedModel(model) {
this.model = model;
this.count = 0;
}

20
ghost/data-generator/lib/tables/members-subscription-created-events.js → ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js
View File

@ -1,21 +1,21 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {luck} = require('../utils/random');
class MembersSubscriptionCreatedEventsImporter extends TableImporter {
static table = 'members_subscription_created_events';
static dependencies = ['members_stripe_customers_subscriptions', 'subscriptions', 'posts'];
constructor(knex, {subscriptions, posts}) {
super(MembersSubscriptionCreatedEventsImporter.table, knex);
this.subscriptions = subscriptions;
this.posts = [...posts];
// Sort posts in reverse chronologoical order
this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf());
constructor(knex, transaction) {
super(MembersSubscriptionCreatedEventsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id').from('members_stripe_customers_subscriptions');
this.subscriptions = await this.transaction.select('id', 'created_at', 'member_id').from('subscriptions');
this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
await this.importForEach(membersStripeCustomersSubscriptions, quantity ? quantity / membersStripeCustomersSubscriptions.length : 1);
}
generate() {

9
ghost/data-generator/lib/tables/newsletters.js → ghost/data-generator/lib/importers/NewslettersImporter.js
View File

@ -1,14 +1,17 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {blogStartDate} = require('../utils/blog-info');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
class NewslettersImporter extends TableImporter {
static table = 'newsletters';
static dependencies = [];
defaultQuantity = 2;
constructor(knex) {
super(NewslettersImporter.table, knex);
constructor(knex, transaction) {
super(NewslettersImporter.table, knex, transaction);
this.sortOrder = 0;
// TODO: Use random names if we ever need more than 2 newsletters
this.names = ['Regular premium', 'Occasional freebie'];
}

14
ghost/data-generator/lib/tables/offers.js → ghost/data-generator/lib/importers/OffersImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const {blogStartDate} = require('../utils/blog-info');
@ -6,15 +6,19 @@ const dateToDatabaseString = require('../utils/database-date');
class OffersImporter extends TableImporter {
static table = 'offers';
static dependencies = ['products'];
defaultQuantity = 2;
constructor(knex, {products}) {
super(OffersImporter.table, knex);
this.products = products;
constructor(knex, transaction) {
super(OffersImporter.table, knex, transaction);
}
setImportOptions() {
async import(quantity = this.defaultQuantity) {
this.products = await this.transaction.select('id', 'currency').from('products');
this.names = ['Black Friday', 'Free Trial'];
this.count = 0;
await super.import(quantity);
}
generate() {

15
ghost/data-generator/lib/tables/posts-authors.js → ghost/data-generator/lib/importers/PostsAuthorsImporter.js
View File

@ -1,17 +1,20 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class PostsAuthorsImporter extends TableImporter {
static table = 'posts_authors';
static dependencies = ['posts', 'users'];
constructor(knex, {users}) {
super(PostsAuthorsImporter.table, knex);
this.users = users;
constructor(knex, transaction) {
super(PostsAuthorsImporter.table, knex, transaction);
this.sortOrder = 0;
}
setImportOptions({model}) {
this.model = model;
async import(quantity) {
const posts = await this.transaction.select('id').from('posts');
this.users = await this.transaction.select('id').from('users');
await this.importForEach(posts, quantity ? quantity / posts.length : 1);
}
generate() {

26
ghost/data-generator/lib/tables/posts.js → ghost/data-generator/lib/importers/PostsImporter.js
View File

@ -1,27 +1,27 @@
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const {luck} = require('../utils/random');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const dateToDatabaseString = require('../utils/database-date');
class PostsImporter extends TableImporter {
static table = 'posts';
static dependencies = ['newsletters'];
defaultQuantity = faker.datatype.number({
min: 80,
max: 120
});
constructor(knex, {newsletters}) {
super(PostsImporter.table, knex);
this.newsletters = newsletters;
type = 'post';
constructor(knex, transaction) {
super(PostsImporter.table, knex, transaction);
}
setImportOptions({type = 'post'}) {
this.type = type;
}
async import(quantity = this.defaultQuantity) {
this.newsletters = await this.transaction.select('id').from('newsletters');
async addNewsletters({posts}) {
for (const {id, visibility} of posts) {
await this.knex('posts').update({
newsletter_id: luck(90) ? (visibility === 'paid' ? this.newsletters[0].id : this.newsletters[1].id) : null
}).where({id, type: 'post', status: 'published'});
}
await super.import(quantity);
}
generate() {

19
ghost/data-generator/lib/tables/posts-products.js → ghost/data-generator/lib/importers/PostsProductsImporter.js
View File

@ -1,17 +1,24 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class PostsProductsImporter extends TableImporter {
static table = 'posts_products';
static dependencies = ['posts', 'products'];
constructor(knex, {products}) {
super(PostsProductsImporter.table, knex);
this.products = products;
constructor(knex, transaction) {
super(PostsProductsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.sortOrder = 0;
async import(quantity) {
const posts = await this.transaction.select('id').from('posts').where('type', 'post');
this.products = await this.transaction.select('id').from('products');
await this.importForEach(posts, quantity ? quantity / posts.length : 1);
}
setReferencedModel(model) {
this.model = model;
this.sortOrder = 0;
}
generate() {

24
ghost/data-generator/lib/tables/posts-tags.js → ghost/data-generator/lib/importers/PostsTagsImporter.js
View File

@ -1,18 +1,28 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class PostsTagsImporter extends TableImporter {
static table = 'posts_tags';
constructor(knex, {tags}) {
super(PostsTagsImporter.table, knex);
this.tags = tags;
static dependencies = ['posts', 'tags'];
constructor(knex, transaction) {
super(PostsTagsImporter.table, knex, transaction);
this.sortOrder = 0;
}
setImportOptions({model}) {
this.notIndex = [];
this.sortOrder = 0;
async import(quantity) {
const posts = await this.transaction.select('id').from('posts').where('type', 'post');
this.tags = await this.transaction.select('id').from('tags');
await this.importForEach(posts, quantity ? quantity / posts.length : () => faker.datatype.number({
min: 0,
max: 3
}));
}
setReferencedModel(model) {
this.model = model;
this.notIndex = [];
}
generate() {

19
ghost/data-generator/lib/tables/products-benefits.js → ghost/data-generator/lib/importers/ProductsBenefitsImporter.js
View File

@ -1,17 +1,24 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class ProductsBenefitsImporter extends TableImporter {
static table = 'products_benefits';
static dependencies = ['benefits', 'products'];
constructor(knex, {benefits}) {
super(ProductsBenefitsImporter.table, knex);
this.benefits = benefits;
this.sortOrder = 0;
constructor(knex, transaction) {
super(ProductsBenefitsImporter.table, knex, transaction);
}
setImportOptions({model}) {
async import(quantity) {
const products = await this.transaction.select('id', 'name').from('products');
this.benefits = await this.transaction.select('id').from('benefits');
await this.importForEach(products, quantity ? quantity / products.length : 5);
}
setReferencedModel(model) {
this.model = model;
this.sortOrder = 0;
switch (this.model.name) {
case 'Bronze':

27
ghost/data-generator/lib/tables/products.js → ghost/data-generator/lib/importers/ProductsImporter.js
View File

@ -1,21 +1,34 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const {blogStartDate} = require('../utils/blog-info');
class ProductsImporter extends TableImporter {
static table = 'products';
static dependencies = [];
defaultQuantity = 4;
constructor(knex) {
super(ProductsImporter.table, knex);
constructor(knex, transaction) {
super(ProductsImporter.table, knex, transaction);
}
setImportOptions() {
async import(quantity = this.defaultQuantity) {
// TODO: Add random products if quantity != 4
this.names = ['Free', 'Bronze', 'Silver', 'Gold'];
this.count = 0;
await super.import(quantity);
}
async addStripePrices({products, stripeProducts, stripePrices}) {
/**
* Add the stripe products / prices
*/
async finalise() {
const stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
const stripePrices = await this.transaction.select('id', 'stripe_product_id', 'interval').from('stripe_prices');
const products = await this.transaction.select('id').from('products');
for (const {id} of products) {
const stripeProduct = stripeProducts.find(p => id === p.product_id);
if (!stripeProduct) {
@ -40,7 +53,7 @@ class ProductsImporter extends TableImporter {
}
if (Object.keys(update).length > 0) {
await this.knex('products').update(update).where({
await this.transaction('products').update(update).where({
id
});
}
@ -59,7 +72,7 @@ class ProductsImporter extends TableImporter {
};
if (count !== 0) {
tierInfo.type = 'paid';
tierInfo.description = `${name} star member`;
tierInfo.description = `${name} tier member`;
tierInfo.currency = 'USD';
tierInfo.monthly_price = count * 500;
tierInfo.yearly_price = count * 5000;

26
ghost/data-generator/lib/tables/redirects.js → ghost/data-generator/lib/importers/RedirectsImporter.js
View File

@ -1,18 +1,32 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
class RedirectsImporter extends TableImporter {
static table = 'redirects';
static dependencies = ['posts'];
constructor(knex) {
super(RedirectsImporter.table, knex);
constructor(knex, transaction) {
super(RedirectsImporter.table, knex, transaction);
}
setImportOptions({model, amount}) {
async import(quantity) {
const posts = await this.transaction
.select('id', 'published_at')
.from('posts')
.where('type', 'post')
.andWhere('status', 'published');
this.quantity = quantity ? quantity / posts.length : 10;
await this.importForEach(posts, this.quantity);
}
setReferencedModel(model) {
this.model = model;
// Reset the amount for each model
this.amount = faker.datatype.number({
min: 1,
max: amount
min: 0,
max: this.quantity
});
}

20
ghost/data-generator/lib/tables/roles-users.js → ghost/data-generator/lib/importers/RolesUsersImporter.js
View File

@ -1,17 +1,23 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
class RolesUsersImporter extends TableImporter {
static table = 'roles_users';
// No roles imorter, since roles are statically defined in database
static dependencies = ['users'];
constructor(knex, {roles}) {
super(RolesUsersImporter.table, knex);
this.roles = roles;
this.sortOrder = 0;
constructor(knex, transaction) {
super(RolesUsersImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
/**
* Ignore overriden quantity for 1:1 relationship
*/
async import() {
const users = await this.transaction.select('id').from('users').whereNot('id', 1);
this.roles = await this.transaction.select('id', 'name').from('roles');
await this.importForEach(users, 1);
}
generate() {

24
ghost/data-generator/lib/tables/stripe-prices.js → ghost/data-generator/lib/importers/StripePricesImporter.js
View File

@ -1,25 +1,31 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {blogStartDate} = require('../utils/blog-info');
const sixWeeksLater = new Date(blogStartDate);
sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
class StripePricesImporter extends TableImporter {
static table = 'stripe_prices';
static dependencies = ['products', 'stripe_products'];
constructor(knex, {products}) {
super(StripePricesImporter.table, knex);
this.products = products;
constructor(knex, transaction) {
super(StripePricesImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import() {
const stripeProducts = await this.transaction.select('id', 'stripe_product_id', 'product_id').from('stripe_products');
this.products = await this.transaction.select('id', 'monthly_price', 'yearly_price').from('products');
await this.importForEach(stripeProducts, 2);
}
setReferencedModel(model) {
this.model = model;
this.count = 0;
}
generate() {
const sixWeeksLater = new Date(blogStartDate);
sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
const count = this.count;
this.count = this.count + 1;

18
ghost/data-generator/lib/tables/stripe-products.js → ghost/data-generator/lib/importers/StripeProductsImporter.js
View File

@ -1,20 +1,24 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {blogStartDate} = require('../utils/blog-info');
const sixWeeksLater = new Date(blogStartDate);
sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
class StripeProductsImporter extends TableImporter {
static table = 'stripe_products';
constructor(knex) {
super(StripeProductsImporter.table, knex);
static dependencies = ['products'];
constructor(knex, transaction) {
super(StripeProductsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import() {
const products = await this.transaction.select('id').from('products');
await this.importForEach(products, 1);
}
generate() {
const sixWeeksLater = new Date(blogStartDate);
sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
return {
id: faker.database.mongodbObjectId(),
product_id: this.model.id,

19
ghost/data-generator/lib/tables/subscriptions.js → ghost/data-generator/lib/importers/SubscriptionsImporter.js
View File

@ -1,20 +1,22 @@
const {faker} = require('@faker-js/faker');
const generateEvents = require('../utils/event-generator');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const dateToDatabaseString = require('../utils/database-date');
class SubscriptionsImporter extends TableImporter {
static table = 'subscriptions';
static dependencies = ['members', 'members_products', 'stripe_products', 'stripe_prices'];
constructor(knex, {members, stripeProducts, stripePrices}) {
super(SubscriptionsImporter.table, knex);
this.members = members;
this.stripeProducts = stripeProducts;
this.stripePrices = stripePrices;
constructor(knex, transaction) {
super(SubscriptionsImporter.table, knex, transaction);
}
setImportOptions({model}) {
this.model = model;
async import() {
const membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products');
this.members = await this.transaction.select('id', 'status', 'created_at').from('members');
this.stripeProducts = await this.transaction.select('product_id', 'stripe_product_id').from('stripe_products');
this.stripePrices = await this.transaction.select('stripe_product_id', 'currency', 'amount', 'interval').from('stripe_prices');
await this.importForEach(membersProducts, 1);
}
generate() {
@ -28,6 +30,7 @@ class SubscriptionsImporter extends TableImporter {
return price.stripe_product_id === stripeProduct.stripe_product_id &&
(isMonthly ? price.interval === 'month' : price.interval === 'year');
});
billingInfo.cadence = isMonthly ? 'month' : 'year';
billingInfo.currency = stripePrice.currency;
billingInfo.amount = stripePrice.amount;

108
ghost/data-generator/lib/importers/TableImporter.js Normal file
View File

@ -0,0 +1,108 @@
class TableImporter {
/**
* @type {object|undefined} model Referenced model when generating data
*/
model;
/**
* @type {number|undefined} defaultQuantity Default number of records to import
*/
defaultQuantity;
/**
* Transaction and knex need to be separate since we're using the batchInsert helper
* @param {string} name Name of the table to be generated
* @param {import('knex/types').Knex} knex Database connection
* @param {import('knex/types').Knex.Transaction} transaction Transaction to be used for import
*/
constructor(name, knex, transaction) {
this.name = name;
this.knex = knex;
this.transaction = transaction;
}
async import(amount = this.defaultQuantity) {
const batchSize = 500;
let batch = [];
for (let i = 0; i < amount; i++) {
const model = await this.generate();
if (model) {
batch.push(model);
} else {
// After first null assume that there is no more data
break;
}
if (batch.length === batchSize) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
batch = [];
}
}
// Process final batch
if (batch.length > 0) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
}
}
/**
* @param {Array<Object>} models List of models to reference
* @param {Number|function} amount Number of records to import per model
*/
async importForEach(models = [], amount) {
const batchSize = 500;
let batch = [];
for (const model of models) {
this.setReferencedModel(model);
let currentAmount = (typeof amount === 'function') ? amount() : amount;
if (!Number.isInteger(currentAmount)) {
currentAmount = Math.floor(currentAmount) + ((Math.random() < currentAmount % 1) ? 1 : 0);
}
for (let i = 0; i < currentAmount; i++) {
const data = await this.generate();
if (data) {
batch.push(data);
} else {
// After first null assume that there is no more data for this model
break;
}
if (batch.length === batchSize) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
batch = [];
}
}
}
// Process final batch
if (batch.length > 0) {
await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction);
}
}
/**
* Finalise the imported data, e.g. adding summary records based on a table's dependents
*/
async finalise() {
// No-op by default
}
/**
* Sets the model which newly generated data will reference
* @param {Object} model Model to reference when generating data
*/
setReferencedModel(model) {
this.model = model;
}
/**
* Generates the data for a single model to be imported
* @returns {Object|null} Data to import, optional
*/
generate() {
// Should never be called
return false;
}
}
module.exports = TableImporter;

17
ghost/data-generator/lib/tables/tags.js → ghost/data-generator/lib/importers/TagsImporter.js
View File

@ -1,14 +1,23 @@
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const dateToDatabaseString = require('../utils/database-date');
class TagsImporter extends TableImporter {
static table = 'tags';
static dependencies = ['users'];
defaultQuantity = faker.datatype.number({
min: 16,
max: 24
});
constructor(knex, {users}) {
super(TagsImporter.table, knex);
this.users = users;
constructor(knex, transaction) {
super(TagsImporter.table, knex, transaction);
}
async import(quantity = this.defaultQuantity) {
this.users = await this.transaction.select('id').from('users');
await super.import(quantity);
}
generate() {

8
ghost/data-generator/lib/tables/users.js → ghost/data-generator/lib/importers/UsersImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const {slugify} = require('@tryghost/string');
const security = require('@tryghost/security');
@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date');
class UsersImporter extends TableImporter {
static table = 'users';
static dependencies = [];
defaultQuantity = 8;
constructor(knex) {
super(UsersImporter.table, knex);
constructor(knex, transaction) {
super(UsersImporter.table, knex, transaction);
}
async generate() {

37
ghost/data-generator/lib/tables/mentions.js → ghost/data-generator/lib/importers/WebMentionsImporter.js
View File

@ -1,4 +1,4 @@
const TableImporter = require('./base');
const TableImporter = require('./TableImporter');
const {faker} = require('@faker-js/faker');
const generateEvents = require('../utils/event-generator');
const {luck} = require('../utils/random');
@ -6,14 +6,22 @@ const dateToDatabaseString = require('../utils/database-date');
class WebMentionsImporter extends TableImporter {
static table = 'mentions';
static dependencies = ['posts'];
constructor(knex, {baseUrl}) {
super(WebMentionsImporter.table, knex);
constructor(knex, transaction, {baseUrl}) {
super(WebMentionsImporter.table, knex, transaction);
this.baseUrl = baseUrl;
}
setImportOptions({model, amount}) {
async import(quantity) {
const posts = await this.transaction.select('id', 'slug', 'published_at').from('posts').where('type', 'post');
this.quantity = quantity ? quantity / posts.length : 4;
await this.importForEach(posts, this.quantity);
}
setReferencedModel(model) {
this.model = model;
// Most web mentions published soon after publication date
@ -23,7 +31,7 @@ class WebMentionsImporter extends TableImporter {
this.timestamps = generateEvents({
shape: 'ease-out',
trend: 'negative',
total: amount,
total: this.quantity,
startTime: startDate,
endTime: endDate
}).sort();
@ -31,7 +39,7 @@ class WebMentionsImporter extends TableImporter {
generate() {
if (luck(50)) {
// 50/50 chance of having a web mention
// 50% chance of 1 mention, 25% chance of 2 mentions, etc.
return null;
}
@ -39,6 +47,23 @@ class WebMentionsImporter extends TableImporter {
const timestamp = this.timestamps.shift();
const author = `${faker.name.fullName()}`;
/**
id: {type: 'string', maxlength: 24, nullable: false, primary: true},
source: {type: 'string', maxlength: 2000, nullable: false},
source_title: {type: 'string', maxlength: 2000, nullable: true},
source_site_title: {type: 'string', maxlength: 2000, nullable: true},
source_excerpt: {type: 'string', maxlength: 2000, nullable: true},
source_author: {type: 'string', maxlength: 2000, nullable: true},
source_featured_image: {type: 'string', maxlength: 2000, nullable: true},
source_favicon: {type: 'string', maxlength: 2000, nullable: true},
target: {type: 'string', maxlength: 2000, nullable: false},
resource_id: {type: 'string', maxlength: 24, nullable: true},
resource_type: {type: 'string', maxlength: 50, nullable: true},
created_at: {type: 'dateTime', nullable: false},
payload: {type: 'text', maxlength: 65535, nullable: true},
deleted: {type: 'boolean', nullable: false, defaultTo: false},
verified: {type: 'boolean', nullable: false, defaultTo: false}
*/
return {
id,
source: `${faker.internet.url()}/${faker.helpers.slugify(`${faker.word.adjective()} ${faker.word.noun()}`).toLowerCase()}`,

37
ghost/data-generator/lib/importers/index.js Normal file
View File

@ -0,0 +1,37 @@
module.exports = [
require('./NewslettersImporter'),
require('./PostsImporter'),
require('./UsersImporter'),
require('./TagsImporter'),
require('./ProductsImporter'),
require('./MembersImporter'),
require('./BenefitsImporter'),
require('./WebMentionsImporter'),
require('./PostsAuthorsImporter'),
require('./PostsTagsImporter'),
require('./ProductsBenefitsImporter'),
require('./MembersProductsImporter'),
require('./PostsProductsImporter'),
require('./MembersNewslettersImporter'),
require('./StripeProductsImporter'),
require('./StripePricesImporter'),
require('./SubscriptionsImporter'),
require('./EmailsImporter'),
require('./EmailBatchesImporter'),
require('./EmailRecipientsImporter'),
require('./RedirectsImporter'),
require('./MembersClickEventsImporter'),
require('./OffersImporter'),
require('./MembersCreatedEventsImporter'),
require('./MembersLoginEventsImporter'),
require('./MembersStatusEventsImporter'),
require('./MembersStripeCustomersImporter'),
require('./MembersStripeCustomersSubscriptionsImporter'),
require('./MembersPaidSubscriptionEventsImporter'),
require('./MembersSubscriptionCreatedEventsImporter'),
require('./MembersSubscribeEventsImporter'),
require('./LabelsImporter'),
require('./MembersLabelsImporter'),
require('./RolesUsersImporter'),
require('./MembersFeedbackImporter')
];

89
ghost/data-generator/lib/tables/base.js
View File

@ -1,89 +0,0 @@
class TableImporter {
/**
* @param {string} name Name of the table to be generated
* @param {import('knex/types').Knex} knex Database connection
*/
constructor(name, knex) {
this.name = name;
this.knex = knex;
}
/**
* @typedef {Function} AmountFunction
* @returns {number}
*/
/**
* @typedef {Object.<string,any>} ImportOptions
* @property {number|AmountFunction} amount Number of events to generate
* @property {Object} [model] Used to reference another object during creation
*/
/**
* @param {Array<Object>} models List of models to reference
* @param {ImportOptions} [options] Import options
* @returns {Promise<Array<Object>>}
*/
async importForEach(models = [], options) {
const results = [];
for (const model of models) {
results.push(...await this.import(Object.assign({}, options, {model})));
}
return results;
}
/**
* @param {ImportOptions} options Import options
* @returns {Promise<Array<Object>>}
*/
async import(options) {
if (options.amount === 0) {
return;
}
// Use dynamic amount if faker function given
const amount = (typeof options.amount === 'function') ? options.amount() : options.amount;
this.setImportOptions(Object.assign({}, options, {amount}));
const data = [];
for (let i = 0; i < amount; i++) {
const model = await this.generate();
if (model) {
// Only push models when one is generated successfully
data.push(model);
} else {
// After first null assume that there is no more data
break;
}
}
const rows = ['id'];
if (options && options.rows) {
rows.push(...options.rows);
}
await this.knex.batchInsert(this.name, data, 500);
return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(this.name);
}
/**
*
* @param {ImportOptions} options
* @returns {void}
*/
// eslint-disable-next-line no-unused-vars
setImportOptions(options) {
return;
}
/**
* Generates the data for a single model to be imported
* @returns {Object|null} Data to import, optional
*/
generate() {
// Should never be called
return false;
}
}
module.exports = TableImporter;

38
ghost/data-generator/lib/tables/index.js
View File

@ -1,38 +0,0 @@
// Order matters! Ordered so that dependant tables are after their dependencies
module.exports = {
NewslettersImporter: require('./newsletters'),
PostsImporter: require('./posts'),
UsersImporter: require('./users'),
TagsImporter: require('./tags'),
ProductsImporter: require('./products'),
MembersImporter: require('./members'),
BenefitsImporter: require('./benefits'),
MentionsImporter: require('./mentions'),
PostsAuthorsImporter: require('./posts-authors'),
PostsTagsImporter: require('./posts-tags'),
ProductsBenefitsImporter: require('./products-benefits'),
MembersProductsImporter: require('./members-products'),
PostsProductsImporter: require('./posts-products'),
MembersNewslettersImporter: require('./members-newsletters'),
StripeProductsImporter: require('./stripe-products'),
StripePricesImporter: require('./stripe-prices'),
SubscriptionsImporter: require('./subscriptions'),
EmailsImporter: require('./emails'),
EmailBatchesImporter: require('./email-batches'),
EmailRecipientsImporter: require('./email-recipients'),
RedirectsImporter: require('./redirects'),
MembersClickEventsImporter: require('./members-click-events'),
OffersImporter: require('./offers'),
MembersCreatedEventsImporter: require('./members-created-events'),
MembersLoginEventsImporter: require('./members-login-events'),
MembersStatusEventsImporter: require('./members-status-events'),
MembersStripeCustomersImporter: require('./members-stripe-customers'),
MembersStripeCustomersSubscriptionsImporter: require('./members-stripe-customers-subscriptions'),
MembersPaidSubscriptionEventsImporter: require('./members-paid-subscription-events'),
MembersSubscriptionCreatedEventsImporter: require('./members-subscription-created-events'),
MembersSubscribeEventsImporter: require('./members-subscribe-events'),
LabelsImporter: require('./labels'),
MembersLabelsImporter: require('./members-labels'),
RolesUsersImporter: require('./roles-users'),
MembersFeedbackImporter: require('./members-feedback')
};

24
ghost/data-generator/lib/tables/members-newsletters.js
View File

@ -1,24 +0,0 @@
const {faker} = require('@faker-js/faker');
const TableImporter = require('./base');
class MembersNewslettersImporter extends TableImporter {
static table = 'members_newsletters';
constructor(knex) {
super(MembersNewslettersImporter.table, knex);
}
setImportOptions({model}) {
this.model = model;
}
generate() {
return {
id: faker.database.mongodbObjectId(),
member_id: this.model.member_id,
newsletter_id: this.model.newsletter_id
};
}
}
module.exports = MembersNewslettersImporter;

8
ghost/data-generator/lib/utils/json-importer.js → ghost/data-generator/lib/utils/JsonImporter.js
View File

@ -1,8 +1,9 @@
const {faker} = require('@faker-js/faker');
class JsonImporter {
constructor(knex) {
constructor(knex, transaction) {
this.knex = knex;
this.transaction = transaction;
}
/**
@ -15,7 +16,7 @@ class JsonImporter {
/**
* Import a dataset to the database
* @param {JsonImportOptions} options
* @returns {Promise<Array<Object.<string, any>>>} Set of rows returned from database
* @returns {Promise}
*/
async import({
name,
@ -30,8 +31,7 @@ class JsonImporter {
if (rows.findIndex(row => row === 'id') === -1) {
rows.unshift('id');
}
await this.knex.batchInsert(name, data, 500);
return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(name);
await this.knex.batchInsert(name, data, 500).transacting(this.transaction);
}
}

33
ghost/data-generator/lib/utils/topological-sort.js Normal file
View File

@ -0,0 +1,33 @@
/**
* This sorting algorithm is used to make sure that dependent tables are imported after their dependencies.
* @param {Array<Object>} objects Objects with a name and dependencies properties
* @returns Topologically sorted array of objects
*/
module.exports = function topologicalSort(objects) {
// Create an empty result array to store the ordered objects
const result = [];
// Create a set to track visited objects during the DFS
const visited = new Set();
// Helper function to perform DFS
function dfs(name) {
if (visited.has(name)) {
return;
}
visited.add(name);
const dependencies = objects.find(item => item.name === name)?.dependencies || [];
for (const dependency of dependencies) {
dfs(dependency);
}
result.push(objects.find(item => item.name === name));
}
// Perform DFS on each object
for (const object of objects) {
dfs(object.name);
}
return result;
};

78
ghost/data-generator/test/data-generator.test.js
View File

@ -2,11 +2,10 @@
// const testUtils = require('./utils');
require('./utils');
const knex = require('knex');
const {
ProductsImporter,
StripeProductsImporter,
StripePricesImporter
} = require('../lib/tables');
const importers = require('../lib/importers');
const ProductsImporter = importers.find(i => i.table === 'products');
const StripeProductsImporter = importers.find(i => i.table === 'stripe_products');
const StripePricesImporter = importers.find(i => i.table === 'stripe_prices');
const generateEvents = require('../lib/utils/event-generator');
@ -82,11 +81,14 @@ describe('Data Generator', function () {
info: () => { },
ok: () => { }
},
modelQuantities: {
members: 10,
membersLoginEvents: 5,
posts: 2
}
tables: [{
name: 'members',
quantity: 10
}, {
name: 'posts',
quantity: 2
}],
withDefault: true
});
try {
return await dataGenerator.importData();
@ -152,27 +154,25 @@ describe('Importer', function () {
});
it('Should import a single item', async function () {
const productsImporter = new ProductsImporter(db);
const products = await productsImporter.import({amount: 1, rows: ['name', 'monthly_price', 'yearly_price']});
const transaction = await db.transaction();
const productsImporter = new ProductsImporter(db, transaction);
await productsImporter.import();
transaction.commit();
products.length.should.eql(1);
const products = await db.select('id', 'name').from('products');
products.length.should.eql(4);
products[0].name.should.eql('Free');
const results = await db.select('id', 'name').from('products');
results.length.should.eql(1);
results[0].name.should.eql('Free');
});
it('Should import an item for each entry in an array', async function () {
const productsImporter = new ProductsImporter(db);
const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
const transaction = await db.transaction();
const productsImporter = new ProductsImporter(db, transaction);
await productsImporter.import();
const stripeProductsImporter = new StripeProductsImporter(db);
await stripeProductsImporter.importForEach(products, {
amount: 1,
rows: ['product_id', 'stripe_product_id']
});
const stripeProductsImporter = new StripeProductsImporter(db, transaction);
await stripeProductsImporter.import();
transaction.commit();
const results = await db.select('id').from('stripe_products');
@ -180,26 +180,20 @@ describe('Importer', function () {
});
it('Should update products to reference price ids', async function () {
const productsImporter = new ProductsImporter(db);
const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']});
const transaction = await db.transaction();
const productsImporter = new ProductsImporter(db, transaction);
await productsImporter.import();
const stripeProductsImporter = new StripeProductsImporter(db);
const stripeProducts = await stripeProductsImporter.importForEach(products, {
amount: 1,
rows: ['product_id', 'stripe_product_id']
});
const stripeProductsImporter = new StripeProductsImporter(db, transaction);
await stripeProductsImporter.import();
const stripePricesImporter = new StripePricesImporter(db, {products});
const stripePrices = await stripePricesImporter.importForEach(stripeProducts, {
amount: 2,
rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname']
});
const stripePricesImporter = new StripePricesImporter(db, transaction);
await stripePricesImporter.import();
await productsImporter.addStripePrices({
products,
stripeProducts,
stripePrices
});
await productsImporter.finalise();
await stripeProductsImporter.finalise();
await stripePricesImporter.finalise();
transaction.commit();
const results = await db.select('id', 'name', 'monthly_price_id', 'yearly_price_id').from('products');