TryGhost/Ghost
1
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2024-11-30 21:40:39 +03:00
Code Issues Projects Releases Wiki Activity

Moved batching logic inside the members importer module

Browse Source 
no issue

- This way importer is more self contained and controller logic doesn't have to know about batch sizes and other unecessary variables
This commit is contained in:
Nazar Gargol 2020-08-13 20:31:11 +12:00
parent c434666ba2
commit 78b4dff656
2 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/server/api/canary/members.js
View File

@ -596,19 +596,14 @@ module.exports = {
}));
}
const CHUNK_SIZE = 100;
const memberBatches = _.chunk(sanitized, CHUNK_SIZE);
return Promise.map(memberBatches, async (membersBatch) => {
return doImport({
membersBatch,
members: sanitized,
allLabelModels,
importSetLabels,
imported,
invalid,
createdBy
});
});
}).then(() => {
// NOTE: grouping by context because messages can contain unique data like "customer_id"
const groupedErrors = _.groupBy(invalid.errors, 'context');

12
core/server/services/members/importer/index.js
View File

@ -124,11 +124,19 @@ const doImport = async ({membersBatch: members, allLabelModels, importSetLabels,
try {
// TODO: below inserts most likely need to be wrapped into transaction
// to avoid creating orphaned member_labels connections
const CHUNK_SIZE = 5000;
const chunkedMembers = _.chunk(mappedMemberBatchData, CHUNK_SIZE);
for (const data of chunkedMembers) {
await db.knex('members')
.insert(mappedMemberBatchData);
.insert(data);
}
const chunkedLebelAssociations = _.chunk(mappedMembersLabelsBatchAssociations, CHUNK_SIZE);
for (const data of chunkedLebelAssociations) {
await db.knex('members_labels')
.insert(mappedMembersLabelsBatchAssociations);
.insert(data);
}
imported.count += mappedMemberBatchData.length;
} catch (error) {