From 5665656b050ff38ed0aba4c26a5c597564441847 Mon Sep 17 00:00:00 2001 From: bosiraphael <71827178+bosiraphael@users.noreply.github.com> Date: Fri, 22 Mar 2024 18:10:55 +0100 Subject: [PATCH] 4489 timebox finish google calendar full sync (#4615) * add lodash differenceWith * add awaits * update sync cursor is working * add logs * use isSyncEnabled information to enqueue jobs * add decorator InjectObjectMetadataRepository * fix gmail-full-sync --- packages/twenty-server/package.json | 2 + .../google-calendar-full-sync.command.ts | 14 ++ .../calendar-channel.repository.ts | 39 +++--- .../calendar-event-attendee.repository.ts | 30 ++++- .../google-calendar-full-sync.service.ts | 127 +++++++++++++----- .../gmail-full-sync.service.ts | 2 + yarn.lock | 18 +++ 7 files changed, 176 insertions(+), 56 deletions(-) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index aeb2b4785b..86599f0edc 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -43,6 +43,7 @@ "class-validator": "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch", "graphql-middleware": "^6.1.35", "jwt-decode": "^4.0.0", + "lodash.differencewith": "^4.5.0", "passport": "^0.7.0", "psl": "^1.9.0", "tsconfig-paths": "^4.2.0" @@ -50,6 +51,7 @@ "devDependencies": { "@nestjs/cli": "10.3.0", "@nx/js": "17.2.8", + "@types/lodash.differencewith": "^4.5.9", "@types/lodash.isempty": "^4.4.7", "@types/lodash.isequal": "^4.5.8", "@types/lodash.isobject": "^3.0.7", diff --git a/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts b/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts index bd3f04de74..d3eb8df0a5 100644 --- a/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts +++ b/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts @@ -11,6 +11,8 @@ import { } from 'src/modules/calendar/jobs/google-calendar-full-sync.job'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; +import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository'; +import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata'; interface GoogleCalendarFullSyncOptions { workspaceId: string; @@ -27,6 +29,8 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner { private readonly messageQueueService: MessageQueueService, @InjectObjectMetadataRepository(ConnectedAccountObjectMetadata) private readonly connectedAccountRepository: ConnectedAccountRepository, + @InjectObjectMetadataRepository(CalendarChannelObjectMetadata) + private readonly calendarChannelRepository: CalendarChannelRepository, ) { super(); } @@ -54,6 +58,16 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner { await this.connectedAccountRepository.getAll(workspaceId); for (const connectedAccount of connectedAccounts) { + const calendarChannel = + await this.calendarChannelRepository.getFirstByConnectedAccountId( + connectedAccount.id, + workspaceId, + ); + + if (!calendarChannel?.isSyncEnabled) { + continue; + } + await this.messageQueueService.add( GoogleCalendarFullSyncJob.name, { diff --git a/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts b/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts index 6273362d5c..95e8299f23 100644 --- a/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts +++ b/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts @@ -28,36 +28,18 @@ export class CalendarChannelRepository { ); } - public async getFirstByConnectedAccountIdOrFail( + public async getFirstByConnectedAccountId( connectedAccountId: string, workspaceId: string, - ): Promise> { + ): Promise | undefined> { const calendarChannels = await this.getByConnectedAccountId( connectedAccountId, workspaceId, ); - if (!calendarChannels || calendarChannels.length === 0) { - throw new Error( - `No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`, - ); - } - return calendarChannels[0]; } - public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail( - connectedAccountId: string, - workspaceId: string, - ): Promise { - const calendarChannel = await this.getFirstByConnectedAccountIdOrFail( - connectedAccountId, - workspaceId, - ); - - return calendarChannel.isContactAutoCreationEnabled; - } - public async getByIds( ids: string[], workspaceId: string, @@ -73,4 +55,21 @@ export class CalendarChannelRepository { transactionManager, ); } + + public async updateSyncCursor( + syncCursor: string, + calendarChannelId: string, + workspaceId: string, + transactionManager?: EntityManager, + ): Promise { + const dataSourceSchema = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + await this.workspaceDataSourceService.executeRawQuery( + `UPDATE ${dataSourceSchema}."calendarChannel" SET "syncCursor" = $1 WHERE "id" = $2`, + [syncCursor, calendarChannelId], + workspaceId, + transactionManager, + ); + } } diff --git a/packages/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts b/packages/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts index 39a556d8ea..20b39a3ac0 100644 --- a/packages/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts +++ b/packages/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { EntityManager } from 'typeorm'; +import differenceWith from 'lodash.differencewith'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record'; @@ -27,7 +28,7 @@ export class CalendarEventAttendeeRepository { this.workspaceDataSourceService.getSchemaName(workspaceId); return await this.workspaceDataSourceService.executeRawQuery( - `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`, + `SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`, [calendarEventAttendeeIds], workspaceId, transactionManager, @@ -47,7 +48,7 @@ export class CalendarEventAttendeeRepository { this.workspaceDataSourceService.getSchemaName(workspaceId); return await this.workspaceDataSourceService.executeRawQuery( - `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`, + `SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "calendarEventId" = ANY($1)`, [calendarEventIds], workspaceId, transactionManager, @@ -67,7 +68,7 @@ export class CalendarEventAttendeeRepository { this.workspaceDataSourceService.getSchemaName(workspaceId); await this.workspaceDataSourceService.executeRawQuery( - `DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`, + `DELETE FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`, [calendarEventAttendeeIds], workspaceId, transactionManager, @@ -119,6 +120,29 @@ export class CalendarEventAttendeeRepository { const dataSourceSchema = this.workspaceDataSourceService.getSchemaName(workspaceId); + const calendarEventIds = Array.from(iCalUIDCalendarEventIdMap.values()); + + const existingCalendarEventAttendees = await this.getByCalendarEventIds( + calendarEventIds, + workspaceId, + transactionManager, + ); + + const calendarEventAttendeesToDelete = differenceWith( + existingCalendarEventAttendees, + calendarEventAttendees, + (existingCalendarEventAttendee, calendarEventAttendee) => + existingCalendarEventAttendee.handle === calendarEventAttendee.handle, + ); + + await this.deleteByIds( + calendarEventAttendeesToDelete.map( + (calendarEventAttendee) => calendarEventAttendee.id, + ), + workspaceId, + transactionManager, + ); + const values = calendarEventAttendees.map((calendarEventAttendee) => ({ ...calendarEventAttendee, calendarEventId: iCalUIDCalendarEventIdMap.get( diff --git a/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts b/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts index fb4a0f9d48..ab418cdafc 100644 --- a/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts +++ b/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts @@ -79,11 +79,15 @@ export class GoogleCalendarFullSyncService { } const calendarChannel = - await this.calendarChannelRepository.getFirstByConnectedAccountIdOrFail( + await this.calendarChannelRepository.getFirstByConnectedAccountId( connectedAccountId, workspaceId, ); + if (!calendarChannel) { + return; + } + const calendarChannelId = calendarChannel.id; const googleCalendarClient = @@ -109,6 +113,7 @@ export class GoogleCalendarFullSyncService { : []; const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle); + let startTime = Date.now(); const googleCalendarEvents = await googleCalendarClient.events.list({ @@ -206,38 +211,94 @@ export class GoogleCalendarFullSyncService { workspaceId, ); - dataSourceMetadata?.transaction(async (transactionManager) => { - this.calendarEventRepository.saveCalendarEvents( - eventsToSave, - workspaceId, - transactionManager, - ); + try { + dataSourceMetadata?.transaction(async (transactionManager) => { + startTime = Date.now(); - this.calendarEventRepository.updateCalendarEvents( - eventsToUpdate, - workspaceId, - transactionManager, - ); + await this.calendarEventRepository.saveCalendarEvents( + eventsToSave, + workspaceId, + transactionManager, + ); - this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations( - calendarChannelEventAssociationsToSave, - workspaceId, - transactionManager, - ); + endTime = Date.now(); - this.calendarEventAttendeesRepository.saveCalendarEventAttendees( - attendeesToSave, - workspaceId, - transactionManager, - ); + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving events in ${ + endTime - startTime + }ms.`, + ); - this.calendarEventAttendeesRepository.updateCalendarEventAttendees( - attendeesToUpdate, - iCalUIDCalendarEventIdMap, - workspaceId, - transactionManager, + startTime = Date.now(); + + await this.calendarEventRepository.updateCalendarEvents( + eventsToUpdate, + workspaceId, + transactionManager, + ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating events in ${ + endTime - startTime + }ms.`, + ); + + startTime = Date.now(); + + await this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations( + calendarChannelEventAssociationsToSave, + workspaceId, + transactionManager, + ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving calendar channel event associations in ${ + endTime - startTime + }ms.`, + ); + + startTime = Date.now(); + + await this.calendarEventAttendeesRepository.saveCalendarEventAttendees( + attendeesToSave, + workspaceId, + transactionManager, + ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving attendees in ${ + endTime - startTime + }ms.`, + ); + + startTime = Date.now(); + + await this.calendarEventAttendeesRepository.updateCalendarEventAttendees( + attendeesToUpdate, + iCalUIDCalendarEventIdMap, + workspaceId, + transactionManager, + ); + + endTime = Date.now(); + + this.logger.log( + `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating attendees in ${ + endTime - startTime + }ms.`, + ); + }); + } catch (error) { + this.logger.error( + `Error during google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}`, ); - }); + } } else { this.logger.log( `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`, @@ -252,11 +313,11 @@ export class GoogleCalendarFullSyncService { startTime = Date.now(); - // await this.calendarChannelService.updateSyncCursor( - // nextSyncToken, - // connectedAccount.id, - // workspaceId, - // ); + await this.calendarChannelRepository.updateSyncCursor( + nextSyncToken, + calendarChannel.id, + workspaceId, + ); endTime = Date.now(); diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts index f7dea7c245..65ac57ee87 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts @@ -211,6 +211,8 @@ export class GmailFullSyncService { this.logger.log( `gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`, ); + + return; } if (errors.length) { diff --git a/yarn.lock b/yarn.lock index 605e0049b9..4664a98c94 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16536,6 +16536,15 @@ __metadata: languageName: node linkType: hard +"@types/lodash.differencewith@npm:^4.5.9": + version: 4.5.9 + resolution: "@types/lodash.differencewith@npm:4.5.9" + dependencies: + "@types/lodash": "npm:*" + checksum: e2433f6daf390c9c09e32532dbd434f133c6ba9e10248d4655ad6c20208d29e620d467fd8a69efe1c66ae15854fca94a4ae36f5ebe0caeb5355272f71b28639f + languageName: node + linkType: hard + "@types/lodash.groupby@npm:^4.6.9": version: 4.6.9 resolution: "@types/lodash.groupby@npm:4.6.9" @@ -33330,6 +33339,13 @@ __metadata: languageName: node linkType: hard +"lodash.differencewith@npm:^4.5.0": + version: 4.5.0 + resolution: "lodash.differencewith@npm:4.5.0" + checksum: 685a51f5e78aa4bb8ff1a22f144b9729e9189ee44d498bd94ab6061885a33efb906436f7e828b50ec91f3b86bd9f35e240da526ec062eac8f7edb42b989c15ed + languageName: node + linkType: hard + "lodash.filter@npm:^4.6.0": version: 4.6.0 resolution: "lodash.filter@npm:4.6.0" @@ -45748,6 +45764,7 @@ __metadata: "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch" "@nx/js": "npm:17.2.8" "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch" + "@types/lodash.differencewith": "npm:^4.5.9" "@types/lodash.isempty": "npm:^4.4.7" "@types/lodash.isequal": "npm:^4.5.8" "@types/lodash.isobject": "npm:^3.0.7" @@ -45760,6 +45777,7 @@ __metadata: class-validator: "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch" graphql-middleware: "npm:^6.1.35" jwt-decode: "npm:^4.0.0" + lodash.differencewith: "npm:^4.5.0" passport: "npm:^0.7.0" psl: "npm:^1.9.0" rimraf: "npm:^5.0.5"