Added products and Stripe data to exports and imports (#14873)

- The migration path from 4.x on SQLite to 5.0 on MySQL requires an export/import
- Exports don't include the Stripe info required to map members to tiers correctly on import. This change fixes that.

Co-authored-by: Simon Backx <simon@ghost.org>
Co-authored-by: Hannah Wolfe <github.erisds@gmail.com>
This commit is contained in:
Matt Hanley 2022-05-20 21:13:58 +01:00 committed by GitHub
parent d6d6841186
commit eae0a6a3b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 506 additions and 26 deletions

View File

@ -304,6 +304,19 @@ class Base {
});
}
/**
* @returns {Object}
*/
mapImportedData(originalObject, importedObject) {
return {
id: importedObject.id,
originalId: this.originalIdMap[importedObject.id],
slug: importedObject.get('slug'),
originalSlug: originalObject.slug,
email: importedObject.get('email')
};
}
doImport(options, importOptions) {
debug('doImport', this.modelName, this.dataToImport.length);
@ -322,13 +335,9 @@ class Base {
}
// for identifier lookup
this.importedData.push({
id: importedModel.id,
originalId: this.originalIdMap[importedModel.id],
slug: importedModel.get('slug'),
originalSlug: obj.slug,
email: importedModel.get('email')
});
this.importedData.push(
this.mapImportedData(obj, importedModel)
);
importedModel = null;
this.dataToImport.splice(index, 1);

View File

@ -10,6 +10,9 @@ const TagsImporter = require('./tags');
const SettingsImporter = require('./settings');
const UsersImporter = require('./users');
const NewslettersImporter = require('./newsletters');
const ProductsImporter = require('./products');
const StripeProductsImporter = require('./stripe-products');
const StripePricesImporter = require('./stripe-prices');
const RolesImporter = require('./roles');
let importers = {};
let DataImporter;
@ -26,9 +29,12 @@ DataImporter = {
importers.users = new UsersImporter(importData.data);
importers.roles = new RolesImporter(importData.data);
importers.tags = new TagsImporter(importData.data);
importers.posts = new PostsImporter(importData.data);
importers.newsletters = new NewslettersImporter(importData.data);
importers.settings = new SettingsImporter(importData.data);
importers.products = new ProductsImporter(importData.data);
importers.stripe_products = new StripeProductsImporter(importData.data);
importers.stripe_prices = new StripePricesImporter(importData.data);
importers.posts = new PostsImporter(importData.data);
return importData;
},
@ -114,6 +120,35 @@ DataImporter = {
});
});
/**
* @TODO: figure out how to fix this properly
* fixup the circular reference from
* stripe_prices -> stripe_products -> products -> stripe_prices
*
* Note: the product importer validates that all values are either
* - being imported, or
* - already exist in the db
* so we only need to map imported products
*/
ops.push(() => {
const importedStripePrices = importers.stripe_prices.importedData;
const importedProducts = importers.products.importedData;
const productOps = [];
_.forEach(importedProducts, (importedProduct) => {
return _.forEach(['monthly_price_id', 'yearly_price_id'], (field) => {
const mappedPrice = _.find(importedStripePrices, {originalId: importedProduct[field]});
if (mappedPrice) {
productOps.push(() => {
return models.Product.edit({[field]: mappedPrice.id}, {id: importedProduct.id, transacting});
});
}
});
});
return sequence(productOps);
});
return sequence(ops)
.then(function () {
results.forEach(function (promise) {

View File

@ -1,6 +1,7 @@
const debug = require('@tryghost/debug')('importer:newsletters');
const _ = require('lodash');
const BaseImporter = require('./base');
const models = require('../../../../models');
const ignoredColumns = ['sender_email'];
@ -23,6 +24,13 @@ class NewslettersImporter extends BaseImporter {
});
}
fetchExisting(modelOptions) {
return models.Newsletter.findAll(_.merge({columns: ['id']}, modelOptions))
.then((existingData) => {
this.existingData = existingData.toJSON();
});
}
beforeImport() {
debug('beforeImport');
this.sanitizeValues();

View File

@ -6,16 +6,23 @@ const mobiledocLib = require('../../../../lib/mobiledoc');
const validator = require('@tryghost/validator');
const postsMetaSchema = require('../../../schema').tables.posts_meta;
const metaAttrs = _.keys(_.omit(postsMetaSchema, ['id']));
const ignoredColumns = ['newsletter_id'];
class PostsImporter extends BaseImporter {
constructor(allDataFromFile) {
super(allDataFromFile, {
modelName: 'Post',
dataKeyToImport: 'posts',
requiredFromFile: ['posts', 'tags', 'posts_tags', 'posts_authors', 'posts_meta'],
requiredImportedData: ['tags'],
requiredExistingData: ['tags']
requiredFromFile: [
'posts',
'tags',
'posts_tags',
'posts_authors',
'posts_meta',
'products',
'posts_products'
],
requiredImportedData: ['tags', 'products', 'newsletters'],
requiredExistingData: ['tags', 'products', 'newsletters']
});
}
@ -42,10 +49,6 @@ class PostsImporter extends BaseImporter {
}
delete obj.send_email_when_published;
}
ignoredColumns.forEach((column) => {
delete obj[column];
});
});
}
@ -64,15 +67,17 @@ class PostsImporter extends BaseImporter {
}
/**
* Naive function to attach related tags and authors.
* Naive function to attach related tags, authors, and products.
*/
addNestedRelations() {
this.requiredFromFile.posts_tags = _.orderBy(this.requiredFromFile.posts_tags, ['post_id', 'sort_order'], ['asc', 'asc']);
this.requiredFromFile.posts_authors = _.orderBy(this.requiredFromFile.posts_authors, ['post_id', 'sort_order'], ['asc', 'asc']);
this.requiredFromFile.posts_products = _.orderBy(this.requiredFromFile.posts_products, ['post_id', 'sort_order'], ['asc', 'asc']);
/**
* from {post_id: 1, tag_id: 2} to post.tags=[{id:id}]
* from {post_id: 1, author_id: 2} post.authors=[{id:id}]
* from {post_id: 1, product_id: 2} post.products=[{id:id}]
*/
const run = (relations, target, fk) => {
_.each(relations, (relation) => {
@ -102,6 +107,7 @@ class PostsImporter extends BaseImporter {
run(this.requiredFromFile.posts_tags, 'tags', 'tag_id');
run(this.requiredFromFile.posts_authors, 'authors', 'author_id');
run(this.requiredFromFile.posts_products, 'tiers', 'product_id');
}
/**
@ -179,6 +185,29 @@ class PostsImporter extends BaseImporter {
_.each(this.dataToImport, (postToImport, postIndex) => {
run(postToImport, postIndex, 'tags', 'tags');
run(postToImport, postIndex, 'authors', 'users');
run(postToImport, postIndex, 'tiers', 'products');
});
// map newsletter_id -> newsletters.id
_.each(this.dataToImport, (objectInFile) => {
if (!objectInFile.newsletter_id) {
return;
}
const importedObject = _.find(this.requiredImportedData.newsletters, {originalId: objectInFile.newsletter_id});
// CASE: we've imported the newsletter
if (importedObject) {
debug(`replaced newsletter_id ${objectInFile.newsletter_id} with ${importedObject.id}`);
objectInFile.newsletter_id = importedObject.id;
return;
}
const existingObject = _.find(this.requiredExistingData.newsletters, {id: objectInFile.newsletter_id});
// CASE: newsletter already exists in the db
if (existingObject) {
return;
}
// CASE: newsletter doesn't exist; ignore it
debug(`newsletter ${objectInFile.newsletter_id} not found; ignoring`);
delete objectInFile.newsletter_id;
});
return super.replaceIdentifiers();

View File

@ -0,0 +1,68 @@
const _ = require('lodash');
const BaseImporter = require('./base');
const models = require('../../../../models');
class ProductsImporter extends BaseImporter {
constructor(allDataFromFile) {
super(allDataFromFile, {
modelName: 'Product',
dataKeyToImport: 'products',
requiredFromFile: ['stripe_prices'],
requiredExistingData: ['stripe_prices']
});
}
fetchExisting(modelOptions) {
return models.Product.findAll(_.merge({columns: ['products.id as id']}, modelOptions))
.then((existingData) => {
this.existingData = existingData.toJSON();
});
}
mapImportedData(originalObject, importedObject) {
return {
id: importedObject.id,
originalId: this.originalIdMap[importedObject.id],
monthly_price_id: originalObject.monthly_price_id,
yearly_price_id: originalObject.yearly_price_id
};
}
validateStripePrice() {
// the stripe price either needs to exist in the current db,
// or be imported as part of the same import
let invalidProducts = [];
_.each(['monthly_price_id', 'yearly_price_id'], (field) => {
_.each(this.dataToImport, (objectInFile) => {
const importedObject = _.find(
this.requiredFromFile.stripe_prices,
{id: objectInFile[field]}
);
// CASE: we'll import the stripe price later
if (importedObject) {
return;
}
const existingObject = _.find(
this.requiredExistingData.stripe_prices,
{id: objectInFile[field]}
);
// CASE: stripe price already exists in the DB
if (existingObject) {
return;
}
// CASE: we don't know what stripe price this is for
invalidProducts.push(objectInFile.id);
});
});
// ignore prices with invalid products
this.dataToImport = this.dataToImport.filter(item => !invalidProducts.includes(item.id));
}
replaceIdentifiers() {
// this has to be in replaceIdentifiers because it's after required* fields are set
this.validateStripePrice();
return super.replaceIdentifiers();
}
}
module.exports = ProductsImporter;

View File

@ -10,7 +10,7 @@ const keyTypeMapper = require('../../../../api/shared/serializers/input/utils/se
const {WRITABLE_KEYS_ALLOWLIST} = require('../../../../../shared/labs');
const labsDefaults = JSON.parse(defaultSettings.labs.labs.defaultValue);
const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address'];
const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address', 'portal_products'];
// Importer maintains as much backwards compatibility as possible
const renamedSettingsMap = {

View File

@ -0,0 +1,59 @@
const _ = require('lodash');
const debug = require('@tryghost/debug')('importer:stripeprices');
const BaseImporter = require('./base');
const models = require('../../../../models');
class StripePricesImporter extends BaseImporter {
constructor(allDataFromFile) {
super(allDataFromFile, {
modelName: 'StripePrice',
dataKeyToImport: 'stripe_prices',
requiredImportedData: ['stripe_products'],
requiredExistingData: ['stripe_products']
});
}
fetchExisting(modelOptions) {
return models.StripePrice.findAll(_.merge({columns: ['id', 'stripe_product_id']}, modelOptions))
.then((existingData) => {
this.existingData = existingData.toJSON();
});
}
validateStripeProduct() {
// ensure we have a valid stripe_product_id in the stripe_products table
let invalidPrices = [];
_.each(this.dataToImport, (objectInFile) => {
const importedObject = _.find(
this.requiredImportedData.stripe_products,
{stripe_product_id: objectInFile.stripe_product_id}
);
// CASE: we've imported the stripe_product
if (importedObject) {
return;
}
const existingObject = _.find(
this.requiredExistingData.stripe_products,
{stripe_product_id: objectInFile.stripe_product_id}
);
// CASE: stripe product already exists in the DB
if (existingObject) {
return;
}
// CASE: we don't know what stripe product this is for
debug(`ignoring invalid product ${objectInFile.stripe_product_id}`);
invalidPrices.push(objectInFile.id);
});
// ignore prices with invalid products
debug(`ignoring ${invalidPrices.length} products`);
this.dataToImport = this.dataToImport.filter(item => !invalidPrices.includes(item.id));
}
replaceIdentifiers() {
// this has to be in replaceIdentifiers because it's after required* fields are set
this.validateStripeProduct();
return super.replaceIdentifiers();
}
}
module.exports = StripePricesImporter;

View File

@ -0,0 +1,61 @@
const _ = require('lodash');
const debug = require('@tryghost/debug')('importer:stripeproducts');
const BaseImporter = require('./base');
const models = require('../../../../models');
class StripeProductsImporter extends BaseImporter {
constructor(allDataFromFile) {
super(allDataFromFile, {
modelName: 'StripeProduct',
dataKeyToImport: 'stripe_products',
requiredImportedData: ['products'],
requiredExistingData: ['products']
});
}
fetchExisting(modelOptions) {
return models.StripeProduct.findAll(_.merge({columns: ['id', 'stripe_product_id']}, modelOptions))
.then((existingData) => {
this.existingData = existingData.toJSON();
});
}
mapImportedData(originalObject, importedObject) {
return {
id: importedObject.id,
originalId: this.originalIdMap[importedObject.id],
stripe_product_id: originalObject.stripe_product_id
};
}
replaceIdentifiers() {
debug('replaceIdentifiers');
// map product_id -> product.id
let invalidProducts = [];
_.each(this.dataToImport, (objectInFile) => {
const importedObject = _.find(this.requiredImportedData.products, {originalId: objectInFile.product_id});
// CASE: we've imported the product and need to map the ID
if (importedObject) {
debug(`replaced product_id ${objectInFile.product_id} with ${importedObject.id}`);
objectInFile.product_id = importedObject.id;
return;
}
const existingObject = _.find(this.requiredExistingData.products, {id: objectInFile.product_id});
// CASE: the product exists in the db already
if (existingObject) {
return;
}
// CASE: we don't know what product this is for
debug(`ignoring stripe product ${objectInFile.stripe_product_id}`);
invalidProducts.push(objectInFile.id);
});
// ignore Stripe products without Ghost products
debug(`ignoring ${invalidProducts.length} products`);
this.dataToImport = this.dataToImport.filter(item => !invalidProducts.includes(item.id));
return super.replaceIdentifiers();
}
}
module.exports = StripeProductsImporter;

View File

@ -391,7 +391,6 @@ describe('Importer', function () {
})
.catch(function (response) {
response.length.should.equal(6);
// NOTE: a duplicated tag.slug is a warning
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [users.bio] exceeds maximum length of 200 characters.');
@ -402,14 +401,14 @@ describe('Importer', function () {
response[2].errorType.should.equal('ValidationError');
response[2].message.should.eql('Value in [tags.meta_title] exceeds maximum length of 300 characters.');
response[3].message.should.eql('Value in [posts.title] cannot be blank.');
response[3].errorType.should.eql('ValidationError');
response[3].errorType.should.equal('ValidationError');
response[3].message.should.eql('Value in [settings.key] cannot be blank.');
response[4].errorType.should.equal('ValidationError');
response[4].message.should.eql('Value in [posts.title] exceeds maximum length of 255 characters.');
response[4].message.should.eql('Value in [posts.title] cannot be blank.');
response[4].errorType.should.eql('ValidationError');
response[5].errorType.should.equal('ValidationError');
response[5].message.should.eql('Value in [settings.key] cannot be blank.');
response[5].message.should.eql('Value in [posts.title] exceeds maximum length of 255 characters.');
});
});

View File

@ -10,6 +10,7 @@ const config = require('../../../../core/shared/config');
const events = require('../../../../core/server/lib/common/events');
const testUtils = require('../../../utils');
const localUtils = require('./utils');
const models = require('../../../../core/server/models');
let request;
let eventsTriggered;
@ -306,4 +307,74 @@ describe('DB API (canary)', function () {
usersResponse.body.users.should.have.length(3);
});
it('Can import a JSON database with products', async function () {
await request.delete(localUtils.API.getApiQuery('db/'))
.set('Origin', config.get('url'))
.set('Accept', 'application/json')
.expect(204);
const res = await request.post(localUtils.API.getApiQuery('db/'))
.set('Origin', config.get('url'))
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.attach('importfile', path.join(__dirname, '/../../../utils/fixtures/export/products_export.json'))
.expect(200);
// Check if we have a product
const product = await models.Product.findOne({slug: 'ghost-inc'});
should.exist(product);
product.get('name').should.equal('Ghost Inc.');
product.get('description').should.equal('Our daily newsletter');
product.get('welcome_page_url').should.equal('/welcome');
// Check settings
const portalProducts = await models.Settings.findOne({key: 'portal_products'});
should.exist(portalProducts);
JSON.parse(portalProducts.get('value')).should.deepEqual([]);
// Check stripe products
const stripeProduct = await models.StripeProduct.findOne({product_id: product.id});
should.exist(stripeProduct);
stripeProduct.get('stripe_product_id').should.equal('prod_d2c1708c21');
stripeProduct.id.should.not.equal('60be1fc9bd3af33564cfb337');
// Check newsletters
const newsletter = await models.Newsletter.findOne({slug: 'test'});
should.exist(newsletter);
newsletter.get('name').should.equal('Ghost Inc.');
// Make sure sender_email is not set
should(newsletter.get('sender_email')).equal(null);
// Check posts
const post = await models.Post.findOne({slug: 'test-newsletter'}, {withRelated: ['tiers']});
should.exist(post);
post.get('newsletter_id').should.equal(newsletter.id);
post.get('visibility').should.equal('public');
post.get('email_recipient_filter').should.equal('status:-free');
// Check this post is connected to the imported product
post.relations.tiers.models.map(m => m.id).should.match([product.id]);
// Check stripe prices
const monthlyPrice = await models.StripePrice.findOne({id: product.get('monthly_price_id')});
should.exist(monthlyPrice);
const yearlyPrice = await models.StripePrice.findOne({id: product.get('yearly_price_id')});
should.exist(yearlyPrice);
monthlyPrice.get('amount').should.equal(500);
monthlyPrice.get('currency').should.equal('usd');
monthlyPrice.get('interval').should.equal('month');
monthlyPrice.get('stripe_price_id').should.equal('price_a425520db0');
monthlyPrice.get('stripe_product_id').should.equal('prod_d2c1708c21');
yearlyPrice.get('amount').should.equal(4800);
yearlyPrice.get('currency').should.equal('usd');
yearlyPrice.get('interval').should.equal('year');
yearlyPrice.get('stripe_price_id').should.equal('price_d04baebb73');
yearlyPrice.get('stripe_product_id').should.equal('prod_d2c1708c21');
});
});

View File

@ -88,7 +88,7 @@ describe('PostsImporter', function () {
pageTrueTypePost.type.should.equal('post', 'pageTrueTypePost.type');
});
it('Removes the newsletter_id column', function () {
it('Does not remove the newsletter_id column', function () {
const fakePosts = [{
slug: 'post-with-newsletter',
newsletter_id: 'bananas'
@ -100,7 +100,7 @@ describe('PostsImporter', function () {
const postWithoutNewsletter = find(importer.dataToImport, {slug: 'post-with-newsletter'});
should.exist(postWithoutNewsletter);
should.not.exist(postWithoutNewsletter.newsletter_id);
should.exist(postWithoutNewsletter.newsletter_id);
});
it('Maps send_email_when_published', function () {

View File

@ -0,0 +1,141 @@
{
"db": [{
"meta": {
"exported_on": 1616560573131,
"version": "4.1.0"
},
"data": {
"products": [
{
"id": "60be1fb9bd3af33564cf87b2",
"name": "Ghost Inc.",
"slug": "ghost-inc",
"created_at": "2021-09-15T13:31:37.000Z",
"updated_at": "2022-05-20T07:13:52.000Z",
"description": "Our daily newsletter",
"monthly_price_id": "614b122e63d2191233117343",
"yearly_price_id": "614b122f63d2191233117344",
"type": "paid",
"active": 1,
"welcome_page_url": "/welcome",
"visibility": "public"
}
],
"stripe_prices": [
{
"id": "614b122e63d2191233117343",
"stripe_price_id": "price_a425520db0",
"stripe_product_id": "prod_d2c1708c21",
"active": 1,
"nickname": "Monthly",
"currency": "usd",
"amount": 500,
"type": "recurring",
"interval": "month",
"created_at": "2021-12-31T11:23:26.000Z",
"updated_at": "2022-05-18T13:49:01.000Z",
"description": null
},
{
"id": "614b122f63d2191233117344",
"stripe_price_id": "price_d04baebb73",
"stripe_product_id": "prod_d2c1708c21",
"active": 1,
"nickname": "Yearly",
"currency": "usd",
"amount": 4800,
"type": "recurring",
"interval": "year",
"created_at": "2021-12-31T11:23:27.000Z",
"updated_at": "2022-05-18T14:46:25.000Z",
"description": null
}
],
"stripe_products": [
{
"id": "60be1fc9bd3af33564cfb337",
"product_id": "60be1fb9bd3af33564cf87b2",
"stripe_product_id": "prod_d2c1708c21",
"created_at": "2021-09-15T13:31:53.000Z",
"updated_at": "2021-09-15T13:31:53.000Z"
}
],
"settings": [
{
"id": "614a8ae163d21912331172c6",
"group": "portal",
"key": "portal_products",
"value": "[\"60be1fb9bd3af33564cf87b2\"]",
"type": "array",
"flags": null,
"created_at": "2021-12-31T01:46:09.000Z",
"updated_at": "2022-01-14T15:53:50.000Z"
}
],
"posts": [
{
"id": "626104751a34a28c24d4ed78",
"uuid": "e4b7fed0-348e-4415-ac3e-0aa36f5f8956",
"title": "Test newsletter",
"slug": "test-newsletter",
"mobiledoc": "{\"version\":\"0.3.1\",\"atoms\":[],\"cards\":[],\"markups\":[],\"sections\":[[1,\"p\",[]],[1,\"p\",[[0,[],0,\"Hi there! This is a test newsletter.\"]]]],\"ghostVersion\":\"4.0\"}",
"html": "<p></p><p>Hi there! This is a test newsletter.</p>",
"comment_id": "626104751a34a28c24d4ed78",
"plaintext": "\n\nHi there! This is a test newsletter.",
"feature_image": null,
"featured": 0,
"type": "post",
"status": "published",
"locale": null,
"visibility": "public",
"created_at": "2022-06-08T07:15:01.000Z",
"updated_at": "2022-06-08T07:16:15.000Z",
"published_at": "2022-06-08T07:15:28.000Z",
"custom_excerpt": null,
"codeinjection_head": null,
"codeinjection_foot": null,
"custom_template": null,
"canonical_url": null,
"email_recipient_filter": "status:-free",
"newsletter_id": "626022161e917a710c687f9e"
}
],
"posts_products": [
{
"id": "6261047e1a34a28c24d4ed7d",
"post_id": "626104751a34a28c24d4ed78",
"product_id": "60be1fb9bd3af33564cf87b2",
"sort_order": 0
}
],
"newsletters": [
{
"id": "626022161e917a710c687f9e",
"name": "Ghost Inc.",
"description": null,
"slug": "test",
"sender_email": "support@ghost.org",
"sender_reply_to": "newsletter",
"status": "active",
"visibility": "members",
"subscribe_on_signup": 1,
"sort_order": 0,
"header_image": null,
"show_header_icon": 1,
"show_header_title": 1,
"title_font_category": "sans_serif",
"title_alignment": "center",
"show_feature_image": 1,
"body_font_category": "sans_serif",
"footer_content": null,
"show_badge": 1,
"sender_name": null,
"created_at": "2022-06-27T16:01:10.000Z",
"updated_at": null,
"show_header_name": 1,
"uuid": "a90a709b-b1c3-42a7-8d98-a288a682ab50"
}
]
}
}]
}