twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-12-25 04:55:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add error handling service for calendar import (#6203)

Browse Source 
Add error handling service for calendar import

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
bosiraphael 2024-07-12 18:56:45 +02:00 committed by GitHub
parent 52aa9abd73
commit c8a889995f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 365 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts
View File

@ -13,6 +13,7 @@ import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-eve
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
@ -56,6 +57,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
providers: [
CalendarChannelSyncStatusService,
CalendarEventsImportService,
CalendarEventImportErrorHandlerService,
CalendarGetCalendarEventsService,
CalendarSaveEventsService,
CalendarEventListFetchCronJob,

1
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-duration.ts Normal file
View File

@ -0,0 +1 @@
export const CALENDAR_THROTTLE_DURATION = 1000 * 60 * 1; // 1 minute

1
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts.ts Normal file
View File

@ -0,0 +1 @@
export const CALENDAR_THROTTLE_MAX_ATTEMPTS = 4;

57
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service.ts
View File

@ -1,22 +1,20 @@
import { Injectable } from '@nestjs/common';
import { calendar_v3 as calendarV3 } from 'googleapis';
import { GaxiosError } from 'gaxios';
import { calendar_v3 as calendarV3 } from 'googleapis';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { GoogleCalendarClientProvider } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/providers/google-calendar.provider';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { GoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type';
import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util';
import { parseGaxiosError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util';
import { parseGoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util';
import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class GoogleCalendarGetEventsService {
constructor(
private readonly googleCalendarClientProvider: GoogleCalendarClientProvider,
@InjectWorkspaceRepository(CalendarChannelWorkspaceEntity)
private readonly calendarChannelRepository: WorkspaceRepository<CalendarChannelWorkspaceEntity>,
) {}
public async getCalendarEvents(
@ -47,18 +45,7 @@ export class GoogleCalendarGetEventsService {
showDeleted: true,
})
.catch(async (error: GaxiosError) => {
if (error.response?.status !== 410) {
throw error;
}
await this.calendarChannelRepository.update(
{
connectedAccountId: connectedAccount.id,
},
{
syncCursor: '',
},
);
this.handleError(error);
return {
data: {
@ -90,4 +77,34 @@ export class GoogleCalendarGetEventsService {
nextSyncCursor: nextSyncToken || '',
};
}
private handleError(error: GaxiosError) {
if (
error.code &&
[
'ECONNRESET',
'ENOTFOUND',
'ECONNABORTED',
'ETIMEDOUT',
'ERR_NETWORK',
].includes(error.code)
) {
throw parseGaxiosError(error);
}
if (error.response?.status !== 410) {
const googleCalendarError: GoogleCalendarError = {
code: error.response?.status,
reason:
error.response?.data?.error?.errors?.[0].reason ||
error.response?.data?.error ||
'',
message:
error.response?.data?.error?.errors?.[0].message ||
error.response?.data?.error_description ||
'',
};
throw parseGoogleCalendarError(googleCalendarError);
}
}
}

5
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type.ts Normal file
View File

@ -0,0 +1,5 @@
export type GoogleCalendarError = {
code?: number;
reason: string;
message: string;
};

28
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-gaxios-error.util.ts Normal file
View File

@ -0,0 +1,28 @@
import { GaxiosError } from 'gaxios';
import {
CalendarEventError,
CalendarEventErrorCode,
} from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type';
export const parseGaxiosError = (error: GaxiosError): CalendarEventError => {
const { code } = error;
switch (code) {
case 'ECONNRESET':
case 'ENOTFOUND':
case 'ECONNABORTED':
case 'ETIMEDOUT':
case 'ERR_NETWORK':
return {
code: CalendarEventErrorCode.TEMPORARY_ERROR,
message: error.message,
};
default:
return {
code: CalendarEventErrorCode.UNKNOWN,
message: error.message,
};
}
};

86
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/parse-google-calendar-error.util.ts Normal file
View File

@ -0,0 +1,86 @@
import { GoogleCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/types/google-calendar-error.type';
import {
CalendarEventError,
CalendarEventErrorCode,
} from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type';
export const parseGoogleCalendarError = (
error: GoogleCalendarError,
): CalendarEventError => {
const { code, reason, message } = error;
switch (code) {
case 400:
if (reason === 'invalid_grant') {
return {
code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS,
message,
};
}
if (reason === 'failedPrecondition') {
return {
code: CalendarEventErrorCode.TEMPORARY_ERROR,
message,
};
}
return {
code: CalendarEventErrorCode.UNKNOWN,
message,
};
case 404:
return {
code: CalendarEventErrorCode.NOT_FOUND,
message,
};
case 429:
return {
code: CalendarEventErrorCode.TEMPORARY_ERROR,
message,
};
case 403:
if (
reason === 'rateLimitExceeded' ||
reason === 'userRateLimitExceeded'
) {
return {
code: CalendarEventErrorCode.TEMPORARY_ERROR,
message,
};
} else {
return {
code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS,
message,
};
}
case 401:
return {
code: CalendarEventErrorCode.INSUFFICIENT_PERMISSIONS,
message,
};
case 500:
if (reason === 'backendError') {
return {
code: CalendarEventErrorCode.TEMPORARY_ERROR,
message,
};
} else {
return {
code: CalendarEventErrorCode.UNKNOWN,
message,
};
}
default:
break;
}
return {
code: CalendarEventErrorCode.UNKNOWN,
message,
};
};

13
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts
View File

@ -6,9 +6,9 @@ import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/typ
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import {
CalendarChannelWorkspaceEntity,
CalendarChannelSyncStage,
CalendarChannelSyncStatus,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
@Injectable()
@ -74,11 +74,18 @@ export class CalendarChannelSyncStatusService {
});
}
public async markAsCalendarEventsImportCompleted(calendarChannelId: string) {
public async markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannelId: string,
) {
await this.calendarChannelRepository.update(calendarChannelId, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
});
await this.schedulePartialCalendarEventListFetch(calendarChannelId);
}
public async markAsFailedUnknownAndFlushCalendarEventsToImport(

144
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service.ts Normal file
View File

@ -0,0 +1,144 @@
import { Injectable } from '@nestjs/common';
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { CALENDAR_THROTTLE_MAX_ATTEMPTS } from 'src/modules/calendar/calendar-event-import-manager/constants/calendar-throttle-max-attempts';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventError } from 'src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export enum CalendarEventImportSyncStep {
FULL_CALENDAR_EVENT_LIST_FETCH = 'FULL_CALENDAR_EVENT_LIST_FETCH',
PARTIAL_CALENDAR_EVENT_LIST_FETCH = 'PARTIAL_CALENDAR_EVENT_LIST_FETCH',
CALENDAR_EVENTS_IMPORT = 'CALENDAR_EVENTS_IMPORT',
}
@Injectable()
export class CalendarEventImportErrorHandlerService {
constructor(
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
@InjectWorkspaceRepository(CalendarChannelWorkspaceEntity)
private readonly calendarChannelRepository: WorkspaceRepository<CalendarChannelWorkspaceEntity>,
) {}
public async handleError(
error: CalendarEventError,
syncStep: CalendarEventImportSyncStep,
calendarChannel: Pick<
CalendarChannelWorkspaceEntity,
'id' | 'throttleFailureCount'
>,
workspaceId: string,
): Promise<void> {
switch (error.code) {
case 'NOT_FOUND':
await this.handleNotFoundError(syncStep, calendarChannel, workspaceId);
break;
case 'TEMPORARY_ERROR':
await this.handleTemporaryError(syncStep, calendarChannel, workspaceId);
break;
case 'INSUFFICIENT_PERMISSIONS':
await this.handleInsufficientPermissionsError(
calendarChannel,
workspaceId,
);
break;
case 'UNKNOWN':
await this.handleUnknownError(error, calendarChannel, workspaceId);
break;
}
}
private async handleTemporaryError(
syncStep: CalendarEventImportSyncStep,
calendarChannel: Pick<
CalendarChannelWorkspaceEntity,
'id' | 'throttleFailureCount'
>,
workspaceId: string,
): Promise<void> {
if (
calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS
) {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
workspaceId,
);
return;
}
await this.calendarChannelRepository.increment(
{
id: calendarChannel.id,
},
'throttleFailureCount',
1,
);
switch (syncStep) {
case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch(
calendarChannel.id,
);
break;
case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
);
break;
case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id,
);
break;
default:
break;
}
}
private async handleInsufficientPermissionsError(
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'id'>,
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannel.id,
workspaceId,
);
}
private async handleUnknownError(
error: CalendarEventError,
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'id'>,
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
workspaceId,
);
throw new Error(
`Unknown error occurred while importing calendar events for calendar channel ${calendarChannel.id} in workspace ${workspaceId}: ${error.message}`,
);
}
private async handleNotFoundError(
syncStep: CalendarEventImportSyncStep,
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'id'>,
workspaceId: string,
): Promise<void> {
if (
syncStep === CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH
) {
return;
}
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannel.id,
workspaceId,
);
}
}

47
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service.ts
View File

@ -7,11 +7,21 @@ import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inje
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import {
CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep,
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-error-handling.service';
import {
CalendarGetCalendarEventsService,
GetCalendarEventsResponse,
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -29,6 +39,7 @@ export class CalendarEventsImportService {
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
private readonly calendarSaveEventsService: CalendarSaveEventsService,
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
) {}
public async processCalendarEventsImport(
@ -36,16 +47,38 @@ export class CalendarEventsImportService {
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
): Promise<void> {
const syncStep =
calendarChannel.syncStage ===
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING
? CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
calendarChannel.id,
);
let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = [];
let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = '';
const { calendarEvents, nextSyncCursor } =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccount,
calendarChannel.syncCursor,
try {
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccount,
calendarChannel.syncCursor,
);
calendarEvents = getCalendarEventsResponse.calendarEvents;
nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleError(
error,
syncStep,
calendarChannel,
workspaceId,
);
return;
}
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelRepository.update(
{
@ -104,7 +137,7 @@ export class CalendarEventsImportService {
},
);
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannel.id,
);
}

11
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/types/calendar-event-error.type.ts Normal file
View File

@ -0,0 +1,11 @@
export enum CalendarEventErrorCode {
NOT_FOUND = 'NOT_FOUND',
TEMPORARY_ERROR = 'TEMPORARY_ERROR',
INSUFFICIENT_PERMISSIONS = 'INSUFFICIENT_PERMISSIONS',
UNKNOWN = 'UNKNOWN',
}
export interface CalendarEventError {
message: string;
code: CalendarEventErrorCode;
}