mirror of
https://github.com/TryGhost/Ghost.git
synced 2024-12-28 21:33:24 +03:00
Refactored members imporeter in preparation for jobs
no issue - This refactor extracts labels related code into a separate module for easier reuse by the "job-aware" batched importer
This commit is contained in:
Nazar Gargol
parent
bcca59ffbe
commit
e8248cdc9e
@ -7,6 +7,7 @@ const config = require('../../../shared/config');
|
||||
const models = require('../../models');
|
||||
const membersService = require('../../services/members');
|
||||
const doImport = require('../../services/members/importer');
|
||||
const memberLabelsImporter = require('../../services/members/importer/labels');
|
||||
const settingsCache = require('../../services/settings/cache');
|
||||
const {i18n} = require('../../lib/common');
|
||||
const logging = require('../../../shared/logging');
|
||||
@ -125,103 +126,11 @@ const sanitizeInput = async (members) => {
|
||||
return {
|
||||
sanitized,
|
||||
invalidCount,
|
||||
validationErrors
|
||||
validationErrors,
|
||||
duplicateStripeCustomersCount
|
||||
};
|
||||
};
|
||||
|
||||
function serializeMemberLabels(labels) {
|
||||
if (_.isString(labels)) {
|
||||
if (labels === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [{
|
||||
name: labels.trim()
|
||||
}];
|
||||
} else if (labels) {
|
||||
return labels.filter((label) => {
|
||||
return !!label;
|
||||
}).map((label) => {
|
||||
if (_.isString(label)) {
|
||||
return {
|
||||
name: label.trim()
|
||||
};
|
||||
}
|
||||
return label;
|
||||
});
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
const findOrCreateLabels = async (labels, options) => {
|
||||
const existingLabels = [];
|
||||
const createdLabels = [];
|
||||
|
||||
for (const label of labels) {
|
||||
const existingLabel = await models.Label.findOne({name: label.name});
|
||||
|
||||
if (existingLabel) {
|
||||
existingLabels.push(existingLabel.toJSON());
|
||||
} else {
|
||||
try {
|
||||
const createdLabel = await models.Label.add(label, options);
|
||||
createdLabels.push(createdLabel.toJSON());
|
||||
} catch (error) {
|
||||
if (error.code && error.message.toLowerCase().indexOf('unique') !== -1) {
|
||||
// ignore if label already exists
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {existingLabels, createdLabels};
|
||||
};
|
||||
|
||||
const handleImportSetLabels = async (labels, options) => {
|
||||
const importSetLabels = serializeMemberLabels(labels);
|
||||
let importLabel;
|
||||
|
||||
const {existingLabels, createdLabels} = await findOrCreateLabels(importSetLabels, options);
|
||||
|
||||
// NOTE: an import label allows for imports to be "undone" via bulk delete
|
||||
if (createdLabels.length) {
|
||||
importLabel = createdLabels[0] && createdLabels[0];
|
||||
} else {
|
||||
const siteTimezone = settingsCache.get('timezone');
|
||||
const name = `Import ${moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm')}`;
|
||||
const result = await findOrCreateLabels([{name}], options);
|
||||
|
||||
const generatedLabel = result.createdLabels.length
|
||||
? result.createdLabels[0]
|
||||
: result.existingLabels[0];
|
||||
importLabel = generatedLabel;
|
||||
importLabel.generated = true;
|
||||
|
||||
createdLabels.push(generatedLabel);
|
||||
}
|
||||
|
||||
return {
|
||||
importSetLabels: [...existingLabels, ...createdLabels],
|
||||
importLabel
|
||||
};
|
||||
};
|
||||
|
||||
const getUniqueMemberLabels = (members) => {
|
||||
const allLabels = [];
|
||||
|
||||
members.forEach((member) => {
|
||||
const labels = (member.labels && member.labels.split(',')) || [];
|
||||
|
||||
if (labels.length) {
|
||||
allLabels.push(...labels);
|
||||
}
|
||||
});
|
||||
|
||||
return _.uniq(allLabels);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
docName: 'members',
|
||||
|
||||
@ -556,16 +465,17 @@ module.exports = {
|
||||
};
|
||||
let duplicateStripeCustomerIdCount = 0;
|
||||
|
||||
let {importSetLabels, importLabel} = await handleImportSetLabels(frame.data.labels, frame.options);
|
||||
|
||||
// NOTE: member-specific labels have to be pre-created as they cause conflicts when processed
|
||||
// in parallel
|
||||
const memberLabels = serializeMemberLabels(getUniqueMemberLabels(frame.data.members));
|
||||
await findOrCreateLabels(memberLabels, frame.options);
|
||||
let {importSetLabels, importLabel} = await memberLabelsImporter.handleAllLabels(
|
||||
frame.data.labels,
|
||||
frame.data.members,
|
||||
settingsCache.get('timezone'),
|
||||
frame.options
|
||||
);
|
||||
|
||||
return Promise.resolve().then(async () => {
|
||||
const {sanitized, invalidCount, validationErrors} = await sanitizeInput(frame.data.members);
|
||||
const {sanitized, invalidCount, validationErrors, duplicateStripeCustomersCount} = await sanitizeInput(frame.data.members);
|
||||
invalid.count += invalidCount;
|
||||
duplicateStripeCustomerIdCount = duplicateStripeCustomersCount;
|
||||
|
||||
if (validationErrors.length) {
|
||||
invalid.errors.push(...validationErrors);
|
||||
@ -574,7 +484,7 @@ module.exports = {
|
||||
return Promise.map(sanitized, ((entry) => {
|
||||
const api = require('./index');
|
||||
entry.labels = (entry.labels && entry.labels.split(',')) || [];
|
||||
const entryLabels = serializeMemberLabels(entry.labels);
|
||||
const entryLabels = memberLabelsImporter.serializeMemberLabels(entry.labels);
|
||||
const mergedLabels = _.unionBy(entryLabels, importSetLabels, 'name');
|
||||
|
||||
cleanupUndefined(entry);
|
||||
@ -695,21 +605,17 @@ module.exports = {
|
||||
|
||||
const createdBy = contextUser(frame.options);
|
||||
|
||||
let {importSetLabels, importLabel} = await handleImportSetLabels(frame.data.labels, frame.options);
|
||||
|
||||
// NOTE: member-specific labels have to be pre-created as they cause conflicts when processed
|
||||
// in parallel
|
||||
const memberLabels = serializeMemberLabels(getUniqueMemberLabels(frame.data.members));
|
||||
const memberLabelsResult = await findOrCreateLabels(memberLabels, frame.options);
|
||||
|
||||
const allLabelModels = [
|
||||
...importSetLabels,
|
||||
...memberLabelsResult.existingLabels,
|
||||
...memberLabelsResult.createdLabels].filter(model => model !== undefined);
|
||||
let {allLabels, importSetLabels, importLabel} = await memberLabelsImporter.handleAllLabels(
|
||||
frame.data.labels,
|
||||
frame.data.members,
|
||||
settingsCache.get('timezone'),
|
||||
frame.options
|
||||
);
|
||||
|
||||
return Promise.resolve().then(async () => {
|
||||
const {sanitized, invalidCount, validationErrors} = await sanitizeInput(frame.data.members);
|
||||
const {sanitized, invalidCount, validationErrors, duplicateStripeCustomersCount} = await sanitizeInput(frame.data.members);
|
||||
invalid.count += invalidCount;
|
||||
duplicateStripeCustomerIdCount = duplicateStripeCustomersCount;
|
||||
|
||||
if (validationErrors.length) {
|
||||
invalid.errors.push(...validationErrors);
|
||||
@ -717,7 +623,7 @@ module.exports = {
|
||||
|
||||
return doImport({
|
||||
members: sanitized,
|
||||
allLabelModels,
|
||||
labels: allLabels,
|
||||
importSetLabels,
|
||||
createdBy
|
||||
});
|
||||
|
||||
@ -21,8 +21,8 @@ const handleUnrecognizedError = (error) => {
|
||||
}
|
||||
};
|
||||
|
||||
const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) => {
|
||||
debug(`Importing members: ${members.length}, labels: ${allLabelModels.length}, import lables: ${importSetLabels.length}, createdBy: ${createdBy}`);
|
||||
const doImport = async ({members, labels, importSetLabels, createdBy}) => {
|
||||
debug(`Importing members: ${members.length}, labels: ${labels.length}, import lables: ${importSetLabels.length}, createdBy: ${createdBy}`);
|
||||
|
||||
let {
|
||||
invalidMembers,
|
||||
@ -30,7 +30,7 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
|
||||
stripeCustomersToFetch,
|
||||
stripeCustomersToCreate,
|
||||
labelAssociationsToInsert
|
||||
} = getMemberData({members, allLabelModels, importSetLabels, createdBy});
|
||||
} = getMemberData({members, labels, importSetLabels, createdBy});
|
||||
|
||||
// NOTE: member insertion has to happen before the rest of insertions to handle validation
|
||||
// errors - remove failed members from label/stripe sets
|
||||
@ -213,8 +213,8 @@ function serializeMemberLabels(labels) {
|
||||
}, []);
|
||||
}
|
||||
|
||||
function getMemberData({members, allLabelModels, importSetLabels, createdBy}) {
|
||||
const labelIdLookup = allLabelModels.reduce(function (labelIdLookupAcc, labelModel) {
|
||||
function getMemberData({members, labels, importSetLabels, createdBy}) {
|
||||
const labelIdLookup = labels.reduce(function (labelIdLookupAcc, labelModel) {
|
||||
return Object.assign(labelIdLookupAcc, {
|
||||
[labelModel.name]: labelModel.id
|
||||
});
|
||||
|
||||
125
core/server/services/members/importer/labels.js
Normal file
125
core/server/services/members/importer/labels.js
Normal file
@ -0,0 +1,125 @@
|
||||
const _ = require('lodash');
|
||||
const moment = require('moment-timezone');
|
||||
const models = require('../../../models');
|
||||
|
||||
const findOrCreateLabels = async (labels, options) => {
|
||||
const existingLabels = [];
|
||||
const createdLabels = [];
|
||||
|
||||
for (const label of labels) {
|
||||
const existingLabel = await models.Label.findOne({name: label.name});
|
||||
|
||||
if (existingLabel) {
|
||||
existingLabels.push(existingLabel.toJSON());
|
||||
} else {
|
||||
try {
|
||||
const createdLabel = await models.Label.add(label, options);
|
||||
createdLabels.push(createdLabel.toJSON());
|
||||
} catch (error) {
|
||||
if (error.code && error.message.toLowerCase().indexOf('unique') !== -1) {
|
||||
// ignore if label already exists
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {existingLabels, createdLabels};
|
||||
};
|
||||
|
||||
const getUniqueMemberLabels = (members) => {
|
||||
const allLabels = [];
|
||||
|
||||
members.forEach((member) => {
|
||||
const labels = (member.labels && member.labels.split(',')) || [];
|
||||
|
||||
if (labels.length) {
|
||||
allLabels.push(...labels);
|
||||
}
|
||||
});
|
||||
|
||||
return _.uniq(allLabels);
|
||||
};
|
||||
|
||||
function serializeMemberLabels(labels) {
|
||||
if (_.isString(labels)) {
|
||||
if (labels === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [{
|
||||
name: labels.trim()
|
||||
}];
|
||||
} else if (labels) {
|
||||
return labels.filter((label) => {
|
||||
return !!label;
|
||||
}).map((label) => {
|
||||
if (_.isString(label)) {
|
||||
return {
|
||||
name: label.trim()
|
||||
};
|
||||
}
|
||||
return label;
|
||||
});
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
const handleImportSetLabels = async (labels, siteTimezone, options) => {
|
||||
const importSetLabels = serializeMemberLabels(labels);
|
||||
let importLabel;
|
||||
|
||||
const {existingLabels, createdLabels} = await findOrCreateLabels(importSetLabels, options);
|
||||
|
||||
// NOTE: an import label allows for imports to be "undone" via bulk delete
|
||||
if (createdLabels.length) {
|
||||
importLabel = createdLabels[0] && createdLabels[0];
|
||||
} else {
|
||||
const name = `Import ${moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm')}`;
|
||||
const result = await findOrCreateLabels([{name}], options);
|
||||
|
||||
const generatedLabel = result.createdLabels.length
|
||||
? result.createdLabels[0]
|
||||
: result.existingLabels[0];
|
||||
importLabel = generatedLabel;
|
||||
importLabel.generated = true;
|
||||
|
||||
createdLabels.push(generatedLabel);
|
||||
}
|
||||
|
||||
return {
|
||||
importSetLabels: [...existingLabels, ...createdLabels],
|
||||
importLabel
|
||||
};
|
||||
};
|
||||
|
||||
const handleMemberLabels = async (members, options) => {
|
||||
const memberLabels = serializeMemberLabels(getUniqueMemberLabels(members));
|
||||
|
||||
return await findOrCreateLabels(memberLabels, options);
|
||||
};
|
||||
|
||||
const handleAllLabels = async (labels, members, siteTimezone, options) => {
|
||||
let {
|
||||
importSetLabels,
|
||||
importLabel
|
||||
} = await handleImportSetLabels(labels, siteTimezone, options);
|
||||
|
||||
const memberLabelsResult = await handleMemberLabels(members);
|
||||
|
||||
const allLabels = [
|
||||
...importSetLabels,
|
||||
...memberLabelsResult.existingLabels,
|
||||
...memberLabelsResult.createdLabels
|
||||
];
|
||||
|
||||
return {
|
||||
allLabels,
|
||||
importSetLabels,
|
||||
importLabel
|
||||
};
|
||||
};
|
||||
|
||||
module.exports.handleAllLabels = handleAllLabels;
|
||||
module.exports.serializeMemberLabels = serializeMemberLabels;
|
||||
Loading…
Reference in New Issue
Block a user