TryGhost/Ghost
1
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2024-11-27 18:52:14 +03:00
Code Issues Projects Releases Wiki Activity

Moved batching logic inside the members importer module

Browse Source 
no issue

- This way importer is more self contained and controller logic doesn't have to know about batch sizes and other unecessary variables
This commit is contained in:
Nazar Gargol 2020-08-13 20:31:11 +12:00
parent c434666ba2
commit 78b4dff656
2 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
core/server/api/canary/members.js
View File

@ -596,18 +596,13 @@ module.exports = {
}));
}
const CHUNK_SIZE = 100;
const memberBatches = _.chunk(sanitized, CHUNK_SIZE);
return Promise.map(memberBatches, async (membersBatch) => {
return doImport({
membersBatch,
allLabelModels,
importSetLabels,
imported,
invalid,
createdBy
});
return doImport({
members: sanitized,
allLabelModels,
importSetLabels,
imported,
invalid,
createdBy
});
}).then(() => {
// NOTE: grouping by context because messages can contain unique data like "customer_id"

16
core/server/services/members/importer/index.js
View File

@ -124,11 +124,19 @@ const doImport = async ({membersBatch: members, allLabelModels, importSetLabels,
try {
// TODO: below inserts most likely need to be wrapped into transaction
// to avoid creating orphaned member_labels connections
await db.knex('members')
.insert(mappedMemberBatchData);
const CHUNK_SIZE = 5000;
const chunkedMembers = _.chunk(mappedMemberBatchData, CHUNK_SIZE);
for (const data of chunkedMembers) {
await db.knex('members')
.insert(data);
}
await db.knex('members_labels')
.insert(mappedMembersLabelsBatchAssociations);
const chunkedLebelAssociations = _.chunk(mappedMembersLabelsBatchAssociations, CHUNK_SIZE);
for (const data of chunkedLebelAssociations) {
await db.knex('members_labels')
.insert(data);
}
imported.count += mappedMemberBatchData.length;
} catch (error) {