From 78b4dff656586aa688b724055a1371eeb57866d7 Mon Sep 17 00:00:00 2001
From: Nazar Gargol <nazargargol@gmail.com>
Date: Thu, 13 Aug 2020 20:31:11 +1200
Subject: [PATCH] Moved batching logic inside the members importer module

no issue

- This way importer is more self contained and controller logic doesn't have to know about batch sizes and other unecessary variables
---
 core/server/api/canary/members.js             | 19 +++++++------------
 .../server/services/members/importer/index.js | 16 ++++++++++++----
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/core/server/api/canary/members.js b/core/server/api/canary/members.js
index 3dd5dc539c..977d6cfad4 100644
--- a/core/server/api/canary/members.js
+++ b/core/server/api/canary/members.js
@@ -596,18 +596,13 @@ module.exports = {
                     }));
                 }
 
-                const CHUNK_SIZE = 100;
-                const memberBatches = _.chunk(sanitized, CHUNK_SIZE);
-
-                return Promise.map(memberBatches, async (membersBatch) => {
-                    return doImport({
-                        membersBatch,
-                        allLabelModels,
-                        importSetLabels,
-                        imported,
-                        invalid,
-                        createdBy
-                    });
+                return doImport({
+                    members: sanitized,
+                    allLabelModels,
+                    importSetLabels,
+                    imported,
+                    invalid,
+                    createdBy
                 });
             }).then(() => {
                 // NOTE: grouping by context because messages can contain unique data like "customer_id"
diff --git a/core/server/services/members/importer/index.js b/core/server/services/members/importer/index.js
index bf3c75d2da..b4926de1c0 100644
--- a/core/server/services/members/importer/index.js
+++ b/core/server/services/members/importer/index.js
@@ -124,11 +124,19 @@ const doImport = async ({membersBatch: members, allLabelModels, importSetLabels,
     try {
         // TODO: below inserts most likely need to be wrapped into transaction
         //       to avoid creating orphaned member_labels connections
-        await db.knex('members')
-            .insert(mappedMemberBatchData);
+        const CHUNK_SIZE = 5000;
+        const chunkedMembers = _.chunk(mappedMemberBatchData, CHUNK_SIZE);
+        for (const data of chunkedMembers) {
+            await db.knex('members')
+                .insert(data);
+        }
 
-        await db.knex('members_labels')
-            .insert(mappedMembersLabelsBatchAssociations);
+        const chunkedLebelAssociations = _.chunk(mappedMembersLabelsBatchAssociations, CHUNK_SIZE);
+
+        for (const data of chunkedLebelAssociations) {
+            await db.knex('members_labels')
+                .insert(data);
+        }
 
         imported.count += mappedMemberBatchData.length;
     } catch (error) {