5901 refactor email and calendar auto contact creation to create them by batch (#6038)

Closes #5901

package.json

@@ -59,6 +59,7 @@
     "@types/dompurify": "^3.0.5",
     "@types/facepaint": "^1.2.5",
     "@types/lodash.camelcase": "^4.3.7",
+    "@types/lodash.chunk": "^4.2.9",
     "@types/lodash.merge": "^4.6.7",
     "@types/lodash.pick": "^4.3.7",
     "@types/nodemailer": "^6.4.14",
@@ -114,6 +115,7 @@
     "jsonwebtoken": "^9.0.0",
     "libphonenumber-js": "^1.10.26",
     "lodash.camelcase": "^4.3.0",
+    "lodash.chunk": "^4.2.0",
     "lodash.compact": "^3.0.1",
     "lodash.debounce": "^4.0.8",
     "lodash.groupby": "^4.6.0",

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts

@@ -11,6 +11,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
 import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
 import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
+import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener';
 
 @Module({
   imports: [
@@ -25,7 +26,10 @@ import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-co
     CalendarEventParticipantModule,
     TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
   ],
-  providers: [CreateCompanyAndContactService],
+  providers: [
+    CreateCompanyAndContactService,
+    AutoCompaniesAndContactsCreationMessageChannelListener,
+  ],
   exports: [CreateCompanyAndContactService],
 })
 export class AutoCompaniesAndContactsCreationModule {}

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts

@@ -0,0 +1 @@
+export const CONTACTS_CREATION_BATCH_SIZE = 100;

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts

@@ -13,9 +13,9 @@ import {
 } from 'src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job';
 
 @Injectable()
-export class MessagingMessageChannelListener {
+export class AutoCompaniesAndContactsCreationMessageChannelListener {
   constructor(
-    @InjectMessageQueue(MessageQueue.messagingQueue)
+    @InjectMessageQueue(MessageQueue.contactCreationQueue)
     private readonly messageQueueService: MessageQueueService,
   ) {}
 

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts

@@ -3,6 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 
 import { EntityManager } from 'typeorm';
 import compact from 'lodash.compact';
+import chunk from 'lodash.chunk';
 
 import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
 import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service';
@@ -14,7 +15,6 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos
 import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
 import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
 import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
-import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service';
 import { filterOutContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
 import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
@@ -23,6 +23,8 @@ import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/st
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant';
+import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 
 @Injectable()
 export class CreateCompanyAndContactService {
@@ -42,7 +44,7 @@ export class CreateCompanyAndContactService {
 
   async createCompaniesAndPeople(
     connectedAccountHandle: string,
-    contactsToCreate: Contacts,
+    contactsToCreate: Contact[],
     workspaceId: string,
     transactionManager?: EntityManager,
   ): Promise<PersonWorkspaceEntity[]> {
@@ -132,48 +134,55 @@ export class CreateCompanyAndContactService {
 
   async createCompaniesAndContactsAndUpdateParticipants(
     connectedAccount: ConnectedAccountWorkspaceEntity,
-    contactsToCreate: Contacts,
+    contactsToCreate: Contact[],
     workspaceId: string,
   ) {
-    let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
+    const contactsBatches = chunk(
-    let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
+      contactsToCreate,
-      [];
+      CONTACTS_CREATION_BATCH_SIZE,
-
-    await this.workspaceDataSource?.transaction(
-      async (transactionManager: EntityManager) => {
-        const createdPeople = await this.createCompaniesAndPeople(
-          connectedAccount.handle,
-          contactsToCreate,
-          workspaceId,
-          transactionManager,
-        );
-
-        updatedMessageParticipants =
-          await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
-            createdPeople,
-            workspaceId,
-            transactionManager,
-          );
-
-        updatedCalendarEventParticipants =
-          await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
-            createdPeople,
-            workspaceId,
-            transactionManager,
-          );
-      },
     );
 
-    this.eventEmitter.emit(`messageParticipant.matched`, {
+    for (const contactsBatch of contactsBatches) {
-      workspaceId,
+      let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
-      workspaceMemberId: connectedAccount.accountOwnerId,
+      let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
-      messageParticipants: updatedMessageParticipants,
+        [];
-    });
 
-    this.eventEmitter.emit(`calendarEventParticipant.matched`, {
+      await this.workspaceDataSource?.transaction(
-      workspaceId,
+        async (transactionManager: EntityManager) => {
-      workspaceMemberId: connectedAccount.accountOwnerId,
+          const createdPeople = await this.createCompaniesAndPeople(
-      calendarEventParticipants: updatedCalendarEventParticipants,
+            connectedAccount.handle,
-    });
+            contactsBatch,
+            workspaceId,
+            transactionManager,
+          );
+
+          updatedMessageParticipants =
+            await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
+              createdPeople,
+              workspaceId,
+              transactionManager,
+            );
+
+          updatedCalendarEventParticipants =
+            await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
+              createdPeople,
+              workspaceId,
+              transactionManager,
+            );
+        },
+      );
+
+      this.eventEmitter.emit(`messageParticipant.matched`, {
+        workspaceId,
+        workspaceMemberId: connectedAccount.accountOwnerId,
+        messageParticipants: updatedMessageParticipants,
+      });
+
+      this.eventEmitter.emit(`calendarEventParticipant.matched`, {
+        workspaceId,
+        workspaceMemberId: connectedAccount.accountOwnerId,
+        calendarEventParticipants: updatedCalendarEventParticipants,
+      });
+    }
   }
 }

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts

@@ -2,5 +2,3 @@ export type Contact = {
   handle: string;
   displayName: string;
 };
-
-export type Contacts = Contact[];

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts

@@ -1,9 +1,9 @@
-import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
+import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
 
 describe('getUniqueContactsAndHandles', () => {
   it('should return empty arrays when contacts is empty', () => {
-    const contacts: Contacts = [];
+    const contacts: Contact[] = [];
     const result = getUniqueContactsAndHandles(contacts);
 
     expect(result.uniqueContacts).toEqual([]);
@@ -11,7 +11,7 @@ describe('getUniqueContactsAndHandles', () => {
   });
 
   it('should return unique contacts and handles', () => {
-    const contacts: Contacts = [
+    const contacts: Contact[] = [
       { handle: 'john@twenty.com', displayName: 'John Doe' },
       { handle: 'john@twenty.com', displayName: 'John Doe' },
       { handle: 'jane@twenty.com', displayName: 'Jane Smith' },

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts

@@ -1,12 +1,12 @@
 import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
 import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
-import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
+import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 
 export function filterOutContactsFromCompanyOrWorkspace(
-  contacts: Contacts,
+  contacts: Contact[],
   selfHandle: string,
   workspaceMembers: WorkspaceMemberWorkspaceEntity[],
-): Contacts {
+): Contact[] {
   const selfDomainName = getDomainNameFromHandle(selfHandle);
 
   const workspaceMembersMap = workspaceMembers.reduce(

packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts

@@ -1,10 +1,10 @@
 import uniq from 'lodash.uniq';
 import uniqBy from 'lodash.uniqby';
 
-import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
+import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 
-export function getUniqueContactsAndHandles(contacts: Contacts): {
+export function getUniqueContactsAndHandles(contacts: Contact[]): {
-  uniqueContacts: Contacts;
+  uniqueContacts: Contact[];
   uniqueHandles: string[];
 } {
   if (contacts.length === 0) {

yarn.lock

@@ -17265,6 +17265,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/lodash.chunk@npm:^4.2.9":
+  version: 4.2.9
+  resolution: "@types/lodash.chunk@npm:4.2.9"
+  dependencies:
+    "@types/lodash": "npm:*"
+  checksum: 5759b3d969c5db4b0893b70261ae40d4b9a6466c984c16de6fa1d3945b3199cc09f948a444a3b4e6cfa0dd984044cf937cbc8dab5fe0ac8da67244ed74d9e4e4
+  languageName: node
+  linkType: hard
+
 "@types/lodash.compact@npm:^3.0.9":
   version: 3.0.9
   resolution: "@types/lodash.compact@npm:3.0.9"
@@ -34530,6 +34539,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lodash.chunk@npm:^4.2.0":
+  version: 4.2.0
+  resolution: "lodash.chunk@npm:4.2.0"
+  checksum: f9f99969561ad2f62af1f9a96c5bd0af776f000292b0d8db3126c28eb3b32e210d7c31b49c18d0d7901869bd769057046dc134b60cfa0c2c4ce017823a26bb23
+  languageName: node
+  linkType: hard
+
 "lodash.clonedeep@npm:^4.5.0":
   version: 4.5.0
   resolution: "lodash.clonedeep@npm:4.5.0"
@@ -47351,6 +47367,7 @@ __metadata:
     "@types/jest": "npm:^29.5.11"
     "@types/js-cookie": "npm:^3.0.3"
     "@types/lodash.camelcase": "npm:^4.3.7"
+    "@types/lodash.chunk": "npm:^4.2.9"
     "@types/lodash.compact": "npm:^3.0.9"
     "@types/lodash.debounce": "npm:^4.0.7"
     "@types/lodash.groupby": "npm:^4.6.9"
@@ -47462,6 +47479,7 @@ __metadata:
     jsonwebtoken: "npm:^9.0.0"
     libphonenumber-js: "npm:^1.10.26"
     lodash.camelcase: "npm:^4.3.0"
+    lodash.chunk: "npm:^4.2.0"
     lodash.compact: "npm:^3.0.1"
     lodash.debounce: "npm:^4.0.8"
     lodash.groupby: "npm:^4.6.0"