mirror of
https://github.com/twentyhq/twenty.git
synced 2024-08-17 18:00:29 +03:00
4489 timebox finish google calendar full sync (#4615)
* add lodash differenceWith * add awaits * update sync cursor is working * add logs * use isSyncEnabled information to enqueue jobs * add decorator InjectObjectMetadataRepository * fix gmail-full-sync
This commit is contained in:
bosiraphael
GitHub
parent
41aae5bd20
commit
5665656b05
@ -43,6 +43,7 @@
|
||||
"class-validator": "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch",
|
||||
"graphql-middleware": "^6.1.35",
|
||||
"jwt-decode": "^4.0.0",
|
||||
"lodash.differencewith": "^4.5.0",
|
||||
"passport": "^0.7.0",
|
||||
"psl": "^1.9.0",
|
||||
"tsconfig-paths": "^4.2.0"
|
||||
@ -50,6 +51,7 @@
|
||||
"devDependencies": {
|
||||
"@nestjs/cli": "10.3.0",
|
||||
"@nx/js": "17.2.8",
|
||||
"@types/lodash.differencewith": "^4.5.9",
|
||||
"@types/lodash.isempty": "^4.4.7",
|
||||
"@types/lodash.isequal": "^4.5.8",
|
||||
"@types/lodash.isobject": "^3.0.7",
|
||||
|
||||
@ -11,6 +11,8 @@ import {
|
||||
} from 'src/modules/calendar/jobs/google-calendar-full-sync.job';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository';
|
||||
import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata';
|
||||
|
||||
interface GoogleCalendarFullSyncOptions {
|
||||
workspaceId: string;
|
||||
@ -27,6 +29,8 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner {
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(CalendarChannelObjectMetadata)
|
||||
private readonly calendarChannelRepository: CalendarChannelRepository,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@ -54,6 +58,16 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner {
|
||||
await this.connectedAccountRepository.getAll(workspaceId);
|
||||
|
||||
for (const connectedAccount of connectedAccounts) {
|
||||
const calendarChannel =
|
||||
await this.calendarChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccount.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!calendarChannel?.isSyncEnabled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
|
||||
GoogleCalendarFullSyncJob.name,
|
||||
{
|
||||
|
||||
@ -28,36 +28,18 @@ export class CalendarChannelRepository {
|
||||
);
|
||||
}
|
||||
|
||||
public async getFirstByConnectedAccountIdOrFail(
|
||||
public async getFirstByConnectedAccountId(
|
||||
connectedAccountId: string,
|
||||
workspaceId: string,
|
||||
): Promise<ObjectRecord<CalendarChannelObjectMetadata>> {
|
||||
): Promise<ObjectRecord<CalendarChannelObjectMetadata> | undefined> {
|
||||
const calendarChannels = await this.getByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!calendarChannels || calendarChannels.length === 0) {
|
||||
throw new Error(
|
||||
`No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
return calendarChannels[0];
|
||||
}
|
||||
|
||||
public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
|
||||
connectedAccountId: string,
|
||||
workspaceId: string,
|
||||
): Promise<boolean> {
|
||||
const calendarChannel = await this.getFirstByConnectedAccountIdOrFail(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return calendarChannel.isContactAutoCreationEnabled;
|
||||
}
|
||||
|
||||
public async getByIds(
|
||||
ids: string[],
|
||||
workspaceId: string,
|
||||
@ -73,4 +55,21 @@ export class CalendarChannelRepository {
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async updateSyncCursor(
|
||||
syncCursor: string,
|
||||
calendarChannelId: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
): Promise<void> {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
await this.workspaceDataSourceService.executeRawQuery(
|
||||
`UPDATE ${dataSourceSchema}."calendarChannel" SET "syncCursor" = $1 WHERE "id" = $2`,
|
||||
[syncCursor, calendarChannelId],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
import differenceWith from 'lodash.differencewith';
|
||||
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||
@ -27,7 +28,7 @@ export class CalendarEventAttendeeRepository {
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
return await this.workspaceDataSourceService.executeRawQuery(
|
||||
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
|
||||
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`,
|
||||
[calendarEventAttendeeIds],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
@ -47,7 +48,7 @@ export class CalendarEventAttendeeRepository {
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
return await this.workspaceDataSourceService.executeRawQuery(
|
||||
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`,
|
||||
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "calendarEventId" = ANY($1)`,
|
||||
[calendarEventIds],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
@ -67,7 +68,7 @@ export class CalendarEventAttendeeRepository {
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
await this.workspaceDataSourceService.executeRawQuery(
|
||||
`DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
|
||||
`DELETE FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`,
|
||||
[calendarEventAttendeeIds],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
@ -119,6 +120,29 @@ export class CalendarEventAttendeeRepository {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
const calendarEventIds = Array.from(iCalUIDCalendarEventIdMap.values());
|
||||
|
||||
const existingCalendarEventAttendees = await this.getByCalendarEventIds(
|
||||
calendarEventIds,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const calendarEventAttendeesToDelete = differenceWith(
|
||||
existingCalendarEventAttendees,
|
||||
calendarEventAttendees,
|
||||
(existingCalendarEventAttendee, calendarEventAttendee) =>
|
||||
existingCalendarEventAttendee.handle === calendarEventAttendee.handle,
|
||||
);
|
||||
|
||||
await this.deleteByIds(
|
||||
calendarEventAttendeesToDelete.map(
|
||||
(calendarEventAttendee) => calendarEventAttendee.id,
|
||||
),
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const values = calendarEventAttendees.map((calendarEventAttendee) => ({
|
||||
...calendarEventAttendee,
|
||||
calendarEventId: iCalUIDCalendarEventIdMap.get(
|
||||
|
||||
@ -79,11 +79,15 @@ export class GoogleCalendarFullSyncService {
|
||||
}
|
||||
|
||||
const calendarChannel =
|
||||
await this.calendarChannelRepository.getFirstByConnectedAccountIdOrFail(
|
||||
await this.calendarChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!calendarChannel) {
|
||||
return;
|
||||
}
|
||||
|
||||
const calendarChannelId = calendarChannel.id;
|
||||
|
||||
const googleCalendarClient =
|
||||
@ -109,6 +113,7 @@ export class GoogleCalendarFullSyncService {
|
||||
: [];
|
||||
|
||||
const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
|
||||
|
||||
let startTime = Date.now();
|
||||
|
||||
const googleCalendarEvents = await googleCalendarClient.events.list({
|
||||
@ -206,38 +211,94 @@ export class GoogleCalendarFullSyncService {
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
dataSourceMetadata?.transaction(async (transactionManager) => {
|
||||
this.calendarEventRepository.saveCalendarEvents(
|
||||
eventsToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
try {
|
||||
dataSourceMetadata?.transaction(async (transactionManager) => {
|
||||
startTime = Date.now();
|
||||
|
||||
this.calendarEventRepository.updateCalendarEvents(
|
||||
eventsToUpdate,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
await this.calendarEventRepository.saveCalendarEvents(
|
||||
eventsToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations(
|
||||
calendarChannelEventAssociationsToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
endTime = Date.now();
|
||||
|
||||
this.calendarEventAttendeesRepository.saveCalendarEventAttendees(
|
||||
attendeesToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
this.calendarEventAttendeesRepository.updateCalendarEventAttendees(
|
||||
attendeesToUpdate,
|
||||
iCalUIDCalendarEventIdMap,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventRepository.updateCalendarEvents(
|
||||
eventsToUpdate,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations(
|
||||
calendarChannelEventAssociationsToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventAttendeesRepository.saveCalendarEventAttendees(
|
||||
attendeesToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving attendees in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventAttendeesRepository.updateCalendarEventAttendees(
|
||||
attendeesToUpdate,
|
||||
iCalUIDCalendarEventIdMap,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating attendees in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error during google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
@ -252,11 +313,11 @@ export class GoogleCalendarFullSyncService {
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
// await this.calendarChannelService.updateSyncCursor(
|
||||
// nextSyncToken,
|
||||
// connectedAccount.id,
|
||||
// workspaceId,
|
||||
// );
|
||||
await this.calendarChannelRepository.updateSyncCursor(
|
||||
nextSyncToken,
|
||||
calendarChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
|
||||
@ -211,6 +211,8 @@ export class GmailFullSyncService {
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
|
||||
18
yarn.lock
18
yarn.lock
@ -16536,6 +16536,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@types/lodash.differencewith@npm:^4.5.9":
|
||||
version: 4.5.9
|
||||
resolution: "@types/lodash.differencewith@npm:4.5.9"
|
||||
dependencies:
|
||||
"@types/lodash": "npm:*"
|
||||
checksum: e2433f6daf390c9c09e32532dbd434f133c6ba9e10248d4655ad6c20208d29e620d467fd8a69efe1c66ae15854fca94a4ae36f5ebe0caeb5355272f71b28639f
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@types/lodash.groupby@npm:^4.6.9":
|
||||
version: 4.6.9
|
||||
resolution: "@types/lodash.groupby@npm:4.6.9"
|
||||
@ -33330,6 +33339,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"lodash.differencewith@npm:^4.5.0":
|
||||
version: 4.5.0
|
||||
resolution: "lodash.differencewith@npm:4.5.0"
|
||||
checksum: 685a51f5e78aa4bb8ff1a22f144b9729e9189ee44d498bd94ab6061885a33efb906436f7e828b50ec91f3b86bd9f35e240da526ec062eac8f7edb42b989c15ed
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"lodash.filter@npm:^4.6.0":
|
||||
version: 4.6.0
|
||||
resolution: "lodash.filter@npm:4.6.0"
|
||||
@ -45748,6 +45764,7 @@ __metadata:
|
||||
"@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch"
|
||||
"@nx/js": "npm:17.2.8"
|
||||
"@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch"
|
||||
"@types/lodash.differencewith": "npm:^4.5.9"
|
||||
"@types/lodash.isempty": "npm:^4.4.7"
|
||||
"@types/lodash.isequal": "npm:^4.5.8"
|
||||
"@types/lodash.isobject": "npm:^3.0.7"
|
||||
@ -45760,6 +45777,7 @@ __metadata:
|
||||
class-validator: "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch"
|
||||
graphql-middleware: "npm:^6.1.35"
|
||||
jwt-decode: "npm:^4.0.0"
|
||||
lodash.differencewith: "npm:^4.5.0"
|
||||
passport: "npm:^0.7.0"
|
||||
psl: "npm:^1.9.0"
|
||||
rimraf: "npm:^5.0.5"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user