diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts index cbe63c6c4a..fabbf9cf32 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts @@ -13,6 +13,7 @@ import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-eve import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module'; import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service'; +import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service'; import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service'; import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; @@ -56,6 +57,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta providers: [ CalendarChannelSyncStatusService, CalendarEventsImportService, + CalendarEventImportErrorHandlerService, CalendarGetCalendarEventsService, CalendarSaveEventsService, CalendarEventListFetchCronJob, diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-duration.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-duration.ts new file mode 100644 index 0000000000..64028026a9 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-duration.ts @@ -0,0 +1 @@ +export const CALENDAR_THROTTLE_DURATION = 1000 * 60 * 1; // 1 minute diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts.ts new file mode 100644 index 0000000000..f6a3866380 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts.ts @@ -0,0 +1 @@ +export const CALENDAR_THROTTLE_MAX_ATTEMPTS = 4; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts index 63b11b19fd..20f3ec6370 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts @@ -1,22 +1,20 @@ import { Injectable } from '@nestjs/common'; -import { calendar_v3 as calendarV3 } from 'googleapis'; import { GaxiosError } from 'gaxios'; +import { calendar_v3 as calendarV3 } from 'googleapis'; -import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; -import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator'; -import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { GoogleCalendarClientProvider } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/providers/google-calendar.provider'; -import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; -import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; +import { GoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type'; import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util'; +import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util'; +import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util'; +import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; @Injectable() export class GoogleCalendarGetEventsService { constructor( private readonly googleCalendarClientProvider: GoogleCalendarClientProvider, - @InjectWorkspaceRepository(CalendarChannelWorkspaceEntity) - private readonly calendarChannelRepository: WorkspaceRepository, ) {} public async getCalendarEvents( @@ -47,18 +45,7 @@ export class GoogleCalendarGetEventsService { showDeleted: true, }) .catch(async (error: GaxiosError) => { - if (error.response?.status !== 410) { - throw error; - } - - await this.calendarChannelRepository.update( - { - connectedAccountId: connectedAccount.id, - }, - { - syncCursor: '', - }, - ); + this.handleError(error); return { data: { @@ -90,4 +77,34 @@ export class GoogleCalendarGetEventsService { nextSyncCursor: nextSyncToken || '', }; } + + private handleError(error: GaxiosError) { + if ( + error.code && + [ + 'ECONNRESET', + 'ENOTFOUND', + 'ECONNABORTED', + 'ETIMEDOUT', + 'ERR_NETWORK', + ].includes(error.code) + ) { + throw parseGaxiosError(error); + } + if (error.response?.status !== 410) { + const googleCalendarError: GoogleCalendarError = { + code: error.response?.status, + reason: + error.response?.data?.error?.errors?.[0].reason || + error.response?.data?.error || + '', + message: + error.response?.data?.error?.errors?.[0].message || + error.response?.data?.error_description || + '', + }; + + throw parseGoogleCalendarError(googleCalendarError); + } + } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type.ts new file mode 100644 index 0000000000..b007431ad3 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type.ts @@ -0,0 +1,5 @@ +export type GoogleCalendarError = { + code?: number; + reason: string; + message: string; +}; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util.ts new file mode 100644 index 0000000000..4245ff82fe --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util.ts @@ -0,0 +1,28 @@ +import { GaxiosError } from 'gaxios'; + +import { + CalendarEventError, + CalendarEventErrorCode, +} from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type'; + +export const parseGaxiosError = (error: GaxiosError): CalendarEventError => { + const { code } = error; + + switch (code) { + case 'ECONNRESET': + case 'ENOTFOUND': + case 'ECONNABORTED': + case 'ETIMEDOUT': + case 'ERR_NETWORK': + return { + code: CalendarEventErrorCode.TEMPORARY_ERROR, + message: error.message, + }; + + default: + return { + code: CalendarEventErrorCode.UNKNOWN, + message: error.message, + }; + } +}; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util.ts new file mode 100644 index 0000000000..9e5667c629 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util.ts @@ -0,0 +1,86 @@ +import { GoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type'; +import { + CalendarEventError, + CalendarEventErrorCode, +} from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type'; + +export const parseGoogleCalendarError = ( + error: GoogleCalendarError, +): CalendarEventError => { + const { code, reason, message } = error; + + switch (code) { + case 400: + if (reason === 'invalid_grant') { + return { + code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS, + message, + }; + } + if (reason === 'failedPrecondition') { + return { + code: CalendarEventErrorCode.TEMPORARY_ERROR, + message, + }; + } + + return { + code: CalendarEventErrorCode.UNKNOWN, + message, + }; + + case 404: + return { + code: CalendarEventErrorCode.NOT_FOUND, + message, + }; + + case 429: + return { + code: CalendarEventErrorCode.TEMPORARY_ERROR, + message, + }; + + case 403: + if ( + reason === 'rateLimitExceeded' || + reason === 'userRateLimitExceeded' + ) { + return { + code: CalendarEventErrorCode.TEMPORARY_ERROR, + message, + }; + } else { + return { + code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS, + message, + }; + } + + case 401: + return { + code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS, + message, + }; + case 500: + if (reason === 'backendError') { + return { + code: CalendarEventErrorCode.TEMPORARY_ERROR, + message, + }; + } else { + return { + code: CalendarEventErrorCode.UNKNOWN, + message, + }; + } + + default: + break; + } + + return { + code: CalendarEventErrorCode.UNKNOWN, + message, + }; +}; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts index d1e5d0b20c..0d6dd28fcc 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts @@ -6,9 +6,9 @@ import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/typ import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator'; import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { - CalendarChannelWorkspaceEntity, CalendarChannelSyncStage, CalendarChannelSyncStatus, + CalendarChannelWorkspaceEntity, } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; @Injectable() @@ -74,11 +74,18 @@ export class CalendarChannelSyncStatusService { }); } - public async markAsCalendarEventsImportCompleted(calendarChannelId: string) { + public async markAsCompletedAndSchedulePartialMessageListFetch( + calendarChannelId: string, + ) { await this.calendarChannelRepository.update(calendarChannelId, { - syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING, + syncStage: + CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING, syncStatus: CalendarChannelSyncStatus.ACTIVE, + throttleFailureCount: 0, + syncStageStartedAt: null, }); + + await this.schedulePartialCalendarEventListFetch(calendarChannelId); } public async markAsFailedUnknownAndFlushCalendarEventsToImport( diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service.ts new file mode 100644 index 0000000000..93c8b9238f --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service.ts @@ -0,0 +1,144 @@ +import { Injectable } from '@nestjs/common'; + +import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator'; +import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; +import { CALENDAR_THROTTLE_MAX_ATTEMPTS } from 'src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts'; +import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service'; +import { CalendarEventError } from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type'; +import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; + +export enum CalendarEventImportSyncStep { + FULL_CALENDAR_EVENT_LIST_FETCH = 'FULL_CALENDAR_EVENT_LIST_FETCH', + PARTIAL_CALENDAR_EVENT_LIST_FETCH = 'PARTIAL_CALENDAR_EVENT_LIST_FETCH', + CALENDAR_EVENTS_IMPORT = 'CALENDAR_EVENTS_IMPORT', +} + +@Injectable() +export class CalendarEventImportErrorHandlerService { + constructor( + private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService, + @InjectWorkspaceRepository(CalendarChannelWorkspaceEntity) + private readonly calendarChannelRepository: WorkspaceRepository, + ) {} + + public async handleError( + error: CalendarEventError, + syncStep: CalendarEventImportSyncStep, + calendarChannel: Pick< + CalendarChannelWorkspaceEntity, + 'id' | 'throttleFailureCount' + >, + workspaceId: string, + ): Promise { + switch (error.code) { + case 'NOT_FOUND': + await this.handleNotFoundError(syncStep, calendarChannel, workspaceId); + break; + case 'TEMPORARY_ERROR': + await this.handleTemporaryError(syncStep, calendarChannel, workspaceId); + break; + case 'INSUFFICIENT_PERMISSIONS': + await this.handleInsufficientPermissionsError( + calendarChannel, + workspaceId, + ); + break; + case 'UNKNOWN': + await this.handleUnknownError(error, calendarChannel, workspaceId); + break; + } + } + + private async handleTemporaryError( + syncStep: CalendarEventImportSyncStep, + calendarChannel: Pick< + CalendarChannelWorkspaceEntity, + 'id' | 'throttleFailureCount' + >, + workspaceId: string, + ): Promise { + if ( + calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS + ) { + await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport( + calendarChannel.id, + workspaceId, + ); + + return; + } + + await this.calendarChannelRepository.increment( + { + id: calendarChannel.id, + }, + 'throttleFailureCount', + 1, + ); + + switch (syncStep) { + case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH: + await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch( + calendarChannel.id, + ); + break; + + case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH: + await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch( + calendarChannel.id, + ); + break; + + case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT: + await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport( + calendarChannel.id, + ); + break; + + default: + break; + } + } + + private async handleInsufficientPermissionsError( + calendarChannel: Pick, + workspaceId: string, + ): Promise { + await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport( + calendarChannel.id, + workspaceId, + ); + } + + private async handleUnknownError( + error: CalendarEventError, + calendarChannel: Pick, + workspaceId: string, + ): Promise { + await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport( + calendarChannel.id, + workspaceId, + ); + + throw new Error( + `Unknown error occurred while importing calendar events for calendar channel ${calendarChannel.id} in workspace ${workspaceId}: ${error.message}`, + ); + } + + private async handleNotFoundError( + syncStep: CalendarEventImportSyncStep, + calendarChannel: Pick, + workspaceId: string, + ): Promise { + if ( + syncStep === CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH + ) { + return; + } + + await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch( + calendarChannel.id, + workspaceId, + ); + } +} diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts index 78dabc9465..f42a88fc5d 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts @@ -7,11 +7,21 @@ import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inje import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service'; import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service'; -import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; +import { + CalendarEventImportErrorHandlerService, + CalendarEventImportSyncStep, +} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service'; +import { + CalendarGetCalendarEventsService, + GetCalendarEventsResponse, +} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service'; import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util'; import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity'; -import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; +import { + CalendarChannelSyncStage, + CalendarChannelWorkspaceEntity, +} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository'; import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; @@ -29,6 +39,7 @@ export class CalendarEventsImportService { private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService, private readonly getCalendarEventsService: CalendarGetCalendarEventsService, private readonly calendarSaveEventsService: CalendarSaveEventsService, + private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService, ) {} public async processCalendarEventsImport( @@ -36,16 +47,38 @@ export class CalendarEventsImportService { connectedAccount: ConnectedAccountWorkspaceEntity, workspaceId: string, ): Promise { + const syncStep = + calendarChannel.syncStage === + CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING + ? CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH + : CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH; + await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing( calendarChannel.id, ); + let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = []; + let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = ''; - const { calendarEvents, nextSyncCursor } = - await this.getCalendarEventsService.getCalendarEvents( - connectedAccount, - calendarChannel.syncCursor, + try { + const getCalendarEventsResponse = + await this.getCalendarEventsService.getCalendarEvents( + connectedAccount, + calendarChannel.syncCursor, + ); + + calendarEvents = getCalendarEventsResponse.calendarEvents; + nextSyncCursor = getCalendarEventsResponse.nextSyncCursor; + } catch (error) { + await this.calendarEventImportErrorHandlerService.handleError( + error, + syncStep, + calendarChannel, + workspaceId, ); + return; + } + if (!calendarEvents || calendarEvents?.length === 0) { await this.calendarChannelRepository.update( { @@ -104,7 +137,7 @@ export class CalendarEventsImportService { }, ); - await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch( + await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( calendarChannel.id, ); } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type.ts new file mode 100644 index 0000000000..7bf1b56f8d --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type.ts @@ -0,0 +1,11 @@ +export enum CalendarEventErrorCode { + NOT_FOUND = 'NOT_FOUND', + TEMPORARY_ERROR = 'TEMPORARY_ERROR', + INSUFFICIENT_PERMISSIONS = 'INSUFFICIENT_PERMISSIONS', + UNKNOWN = 'UNKNOWN', +} + +export interface CalendarEventError { + message: string; + code: CalendarEventErrorCode; +}