Moved batching logic inside the members importer module

no issue

- This way importer is more self contained and controller logic doesn't have to know about batch sizes and other unecessary variables

core/server/api/canary/members.js

@@ -596,18 +596,13 @@ module.exports = {
                     }));
                 }
 
-                const CHUNK_SIZE = 100;
+                return doImport({
-                const memberBatches = _.chunk(sanitized, CHUNK_SIZE);
+                    members: sanitized,
-
+                    allLabelModels,
-                return Promise.map(memberBatches, async (membersBatch) => {
+                    importSetLabels,
-                    return doImport({
+                    imported,
-                        membersBatch,
+                    invalid,
-                        allLabelModels,
+                    createdBy
-                        importSetLabels,
-                        imported,
-                        invalid,
-                        createdBy
-                    });
                 });
             }).then(() => {
                 // NOTE: grouping by context because messages can contain unique data like "customer_id"

core/server/services/members/importer/index.js

@@ -124,11 +124,19 @@ const doImport = async ({membersBatch: members, allLabelModels, importSetLabels,
     try {
         // TODO: below inserts most likely need to be wrapped into transaction
         //       to avoid creating orphaned member_labels connections
-        await db.knex('members')
+        const CHUNK_SIZE = 5000;
-            .insert(mappedMemberBatchData);
+        const chunkedMembers = _.chunk(mappedMemberBatchData, CHUNK_SIZE);
+        for (const data of chunkedMembers) {
+            await db.knex('members')
+                .insert(data);
+        }
 
-        await db.knex('members_labels')
+        const chunkedLebelAssociations = _.chunk(mappedMembersLabelsBatchAssociations, CHUNK_SIZE);
-            .insert(mappedMembersLabelsBatchAssociations);
+
+        for (const data of chunkedLebelAssociations) {
+            await db.knex('members_labels')
+                .insert(data);
+        }
 
         imported.count += mappedMemberBatchData.length;
     } catch (error) {