twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-12-23 03:51:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

5901 refactor email and calendar auto contact creation to create them by batch (#6038)

Browse Source 
Closes #5901
This commit is contained in:
bosiraphael 2024-06-27 16:37:34 +02:00 committed by GitHub
parent d1bb0fb822
commit 4f9527c860
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 86 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -59,6 +59,7 @@
"@types/dompurify": "^3.0.5",
"@types/facepaint": "^1.2.5",
"@types/lodash.camelcase": "^4.3.7",
"@types/lodash.chunk": "^4.2.9",
"@types/lodash.merge": "^4.6.7",
"@types/lodash.pick": "^4.3.7",
"@types/nodemailer": "^6.4.14",
@ -114,6 +115,7 @@
"jsonwebtoken": "^9.0.0",
"libphonenumber-js": "^1.10.26",
"lodash.camelcase": "^4.3.0",
"lodash.chunk": "^4.2.0",
"lodash.compact": "^3.0.1",
"lodash.debounce": "^4.0.8",
"lodash.groupby": "^4.6.0",

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts
View File

@ -11,6 +11,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener';
@Module({
imports: [
@ -25,7 +26,10 @@ import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-co
CalendarEventParticipantModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
],
providers: [CreateCompanyAndContactService],
providers: [
CreateCompanyAndContactService,
AutoCompaniesAndContactsCreationMessageChannelListener,
],
exports: [CreateCompanyAndContactService],
})
export class AutoCompaniesAndContactsCreationModule {}

1
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts Normal file
View File

@ -0,0 +1 @@
export const CONTACTS_CREATION_BATCH_SIZE = 100;

4
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/messaging-message-channel.listener.ts → packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts
View File

@ -13,9 +13,9 @@ import {
} from 'src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job';
@Injectable()
export class MessagingMessageChannelListener {
export class AutoCompaniesAndContactsCreationMessageChannelListener {
constructor(
@InjectMessageQueue(MessageQueue.messagingQueue)
@InjectMessageQueue(MessageQueue.contactCreationQueue)
private readonly messageQueueService: MessageQueueService,
) {}

89
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts
View File

@ -3,6 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
import { EntityManager } from 'typeorm';
import compact from 'lodash.compact';
import chunk from 'lodash.chunk';
import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service';
@ -14,7 +15,6 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service';
import { filterOutContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
@ -23,6 +23,8 @@ import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/st
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
@Injectable()
export class CreateCompanyAndContactService {
@ -42,7 +44,7 @@ export class CreateCompanyAndContactService {
async createCompaniesAndPeople(
connectedAccountHandle: string,
contactsToCreate: Contacts,
contactsToCreate: Contact[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<PersonWorkspaceEntity[]> {
@ -132,48 +134,55 @@ export class CreateCompanyAndContactService {
async createCompaniesAndContactsAndUpdateParticipants(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contacts,
contactsToCreate: Contact[],
workspaceId: string,
) {
let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
[];
await this.workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount.handle,
contactsToCreate,
workspaceId,
transactionManager,
);
updatedMessageParticipants =
await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
updatedCalendarEventParticipants =
await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
},
const contactsBatches = chunk(
contactsToCreate,
CONTACTS_CREATION_BATCH_SIZE,
);
this.eventEmitter.emit(`messageParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
messageParticipants: updatedMessageParticipants,
});
for (const contactsBatch of contactsBatches) {
let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
[];
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
calendarEventParticipants: updatedCalendarEventParticipants,
});
await this.workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount.handle,
contactsBatch,
workspaceId,
transactionManager,
);
updatedMessageParticipants =
await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
updatedCalendarEventParticipants =
await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
},
);
this.eventEmitter.emit(`messageParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
messageParticipants: updatedMessageParticipants,
});
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
calendarEventParticipants: updatedCalendarEventParticipants,
});
}
}
}

2
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts
View File

@ -2,5 +2,3 @@ export type Contact = {
handle: string;
displayName: string;
};
export type Contacts = Contact[];

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts
View File

@ -1,9 +1,9 @@
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
describe('getUniqueContactsAndHandles', () => {
it('should return empty arrays when contacts is empty', () => {
const contacts: Contacts = [];
const contacts: Contact[] = [];
const result = getUniqueContactsAndHandles(contacts);
expect(result.uniqueContacts).toEqual([]);
@ -11,7 +11,7 @@ describe('getUniqueContactsAndHandles', () => {
});
it('should return unique contacts and handles', () => {
const contacts: Contacts = [
const contacts: Contact[] = [
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts
View File

@ -1,12 +1,12 @@
import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
export function filterOutContactsFromCompanyOrWorkspace(
contacts: Contacts,
contacts: Contact[],
selfHandle: string,
workspaceMembers: WorkspaceMemberWorkspaceEntity[],
): Contacts {
): Contact[] {
const selfDomainName = getDomainNameFromHandle(selfHandle);
const workspaceMembersMap = workspaceMembers.reduce(

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts
View File

@ -1,10 +1,10 @@
import uniq from 'lodash.uniq';
import uniqBy from 'lodash.uniqby';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
export function getUniqueContactsAndHandles(contacts: Contacts): {
uniqueContacts: Contacts;
export function getUniqueContactsAndHandles(contacts: Contact[]): {
uniqueContacts: Contact[];
uniqueHandles: string[];
} {
if (contacts.length === 0) {

18
yarn.lock
View File

@ -17265,6 +17265,15 @@ __metadata:
languageName: node
linkType: hard
"@types/lodash.chunk@npm:^4.2.9":
version: 4.2.9
resolution: "@types/lodash.chunk@npm:4.2.9"
dependencies:
"@types/lodash": "npm:*"
checksum: 5759b3d969c5db4b0893b70261ae40d4b9a6466c984c16de6fa1d3945b3199cc09f948a444a3b4e6cfa0dd984044cf937cbc8dab5fe0ac8da67244ed74d9e4e4
languageName: node
linkType: hard
"@types/lodash.compact@npm:^3.0.9":
version: 3.0.9
resolution: "@types/lodash.compact@npm:3.0.9"
@ -34530,6 +34539,13 @@ __metadata:
languageName: node
linkType: hard
"lodash.chunk@npm:^4.2.0":
version: 4.2.0
resolution: "lodash.chunk@npm:4.2.0"
checksum: f9f99969561ad2f62af1f9a96c5bd0af776f000292b0d8db3126c28eb3b32e210d7c31b49c18d0d7901869bd769057046dc134b60cfa0c2c4ce017823a26bb23
languageName: node
linkType: hard
"lodash.clonedeep@npm:^4.5.0":
version: 4.5.0
resolution: "lodash.clonedeep@npm:4.5.0"
@ -47351,6 +47367,7 @@ __metadata:
"@types/jest": "npm:^29.5.11"
"@types/js-cookie": "npm:^3.0.3"
"@types/lodash.camelcase": "npm:^4.3.7"
"@types/lodash.chunk": "npm:^4.2.9"
"@types/lodash.compact": "npm:^3.0.9"
"@types/lodash.debounce": "npm:^4.0.7"
"@types/lodash.groupby": "npm:^4.6.9"
@ -47462,6 +47479,7 @@ __metadata:
jsonwebtoken: "npm:^9.0.0"
libphonenumber-js: "npm:^1.10.26"
lodash.camelcase: "npm:^4.3.0"
lodash.chunk: "npm:^4.2.0"
lodash.compact: "npm:^3.0.1"
lodash.debounce: "npm:^4.0.8"
lodash.groupby: "npm:^4.6.0"