From e0103bbcdc907b663cb575e35ba29eb5edcbd587 Mon Sep 17 00:00:00 2001 From: bosiraphael <71827178+bosiraphael@users.noreply.github.com> Date: Fri, 31 May 2024 23:20:57 +0200 Subject: [PATCH] 5015 make gmail filters work for partial sync (#5695) Closes #5015 --------- Co-authored-by: Charles Bochet --- .../utils/is-person-email.util.ts | 8 --- .../constants/gmail-excluded-categories.ts | 1 + ...o-delete-from-cache-batch-size.constant.ts | 1 - .../fetch-messages-by-batches.service.ts | 17 +++++- ...mail-full-message-list-fetch-v2.service.ts | 3 + .../gmail-full-message-list-fetch.service.ts | 5 +- .../gmail-messages-import-v2.service.ts | 18 +++++- ...l-fetch-messages-ids-to-exclude.service.ts | 46 +++++++++++++++ .../gmail-get-history.service.ts | 5 +- ...l-partial-message-list-fetch-v2.service.ts | 29 ++++++++- ...gmail-partial-message-list-fetch.module.ts | 2 + .../services/utils/filter-emails.util.ts | 56 ++++++++++++++++++ ...mail-category-excude-search-filter.spec.ts | 27 +++++++++ .../compute-gmail-category-label-id.spec.ts | 9 +++ .../gmail-search-filter.util.spec.ts | 58 ------------------ ...ute-gmail-category-excude-search-filter.ts | 3 + .../utils/compute-gmail-category-label-id.ts | 2 + .../utils/gmail-search-filter.util.ts | 59 ------------------- 18 files changed, 216 insertions(+), 133 deletions(-) delete mode 100644 packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-person-email.util.ts create mode 100644 packages/twenty-server/src/modules/messaging/constants/gmail-excluded-categories.ts delete mode 100644 packages/twenty-server/src/modules/messaging/constants/messages-to-delete-from-cache-batch-size.constant.ts create mode 100644 packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service.ts create mode 100644 packages/twenty-server/src/modules/messaging/services/utils/filter-emails.util.ts create mode 100644 packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-label-id.spec.ts delete mode 100644 packages/twenty-server/src/modules/messaging/utils/__tests__/gmail-search-filter.util.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-excude-search-filter.ts create mode 100644 packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-label-id.ts delete mode 100644 packages/twenty-server/src/modules/messaging/utils/gmail-search-filter.util.ts diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-person-email.util.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-person-email.util.ts deleted file mode 100644 index 387a94e504..0000000000 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-person-email.util.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const isPersonEmail = (email: string | undefined): boolean => { - if (!email) return false; - - const nonPersonalPattern = - /noreply|no-reply|do_not_reply|no\.reply|^(accounts@|info@|admin@|contact@|hello@|support@|sales@|feedback@|service@|help@|mailer-daemon|notifications?|digest|auto|apps|assign|comments|customer-success|enterprise|esign|express|forum|gc@|learn|mailer|marketing|messages|news|notification|payments|receipts|recrutement|security|service|support|team)/; - - return !nonPersonalPattern.test(email); -}; diff --git a/packages/twenty-server/src/modules/messaging/constants/gmail-excluded-categories.ts b/packages/twenty-server/src/modules/messaging/constants/gmail-excluded-categories.ts new file mode 100644 index 0000000000..1f3dcf828a --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/constants/gmail-excluded-categories.ts @@ -0,0 +1 @@ +export const GMAIL_EXCLUDED_CATEGORIES = ['promotions', 'social', 'forums']; diff --git a/packages/twenty-server/src/modules/messaging/constants/messages-to-delete-from-cache-batch-size.constant.ts b/packages/twenty-server/src/modules/messaging/constants/messages-to-delete-from-cache-batch-size.constant.ts deleted file mode 100644 index 1f8a59b88f..0000000000 --- a/packages/twenty-server/src/modules/messaging/constants/messages-to-delete-from-cache-batch-size.constant.ts +++ /dev/null @@ -1 +0,0 @@ -export const MESSAGES_TO_DELETE_FROM_CACHE_BATCH_SIZE = 1000; diff --git a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts index 24a3d87858..6bfda7d21a 100644 --- a/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service.ts @@ -186,6 +186,8 @@ export class FetchMessagesByBatchesService { const bodyData = this.getBodyData(message); const text = bodyData ? Buffer.from(bodyData, 'base64').toString() : ''; + const attachments = this.getAttachmentData(message); + return { id, headerMessageId: messageId, @@ -199,7 +201,7 @@ export class FetchMessagesByBatchesService { cc: rawCc ? addressparser(rawCc) : undefined, bcc: rawBcc ? addressparser(rawBcc) : undefined, text, - attachments: [], + attachments, }; } @@ -214,6 +216,19 @@ export class FetchMessagesByBatchesService { ?.body?.data; } + private getAttachmentData(message: gmail_v1.Schema$Message) { + return ( + message.payload?.parts + ?.filter((part) => part.filename && part.body?.attachmentId) + .map((part) => ({ + filename: part.filename || '', + id: part.body?.attachmentId || '', + mimeType: part.mimeType || '', + size: part.body?.size || 0, + })) || [] + ); + } + private getPropertyFromHeaders( message: gmail_v1.Schema$Message, property: string, diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch-v2.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch-v2.service.ts index 87e85b0a00..d776ffd835 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch-v2.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch-v2.service.ts @@ -21,6 +21,8 @@ import { GmailErrorHandlingService, } from 'src/modules/messaging/services/gmail-error-handling/gmail-error-handling.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/services/message-channel-sync-status/message-channel-sync-status.service'; +import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/utils/compute-gmail-category-excude-search-filter'; +import { GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/constants/gmail-excluded-categories'; @Injectable() export class GmailFullMessageListFetchV2Service { @@ -97,6 +99,7 @@ export class GmailFullMessageListFetchV2Service { userId: 'me', maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT, pageToken, + q: computeGmailCategoryExcludeSearchFilter(GMAIL_EXCLUDED_CATEGORIES), }); } catch (error) { return { diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch.service.ts index 51dd0157b7..e2ab6e0f5b 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-full-message-list-fetch/gmail-full-message-list-fetch.service.ts @@ -26,7 +26,8 @@ import { MessageChannelSyncStatus, } from 'src/modules/messaging/standard-objects/message-channel.workspace-entity'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; -import { gmailSearchFilterEmailAdresses } from 'src/modules/messaging/utils/gmail-search-filter.util'; +import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/utils/compute-gmail-category-excude-search-filter'; +import { GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/constants/gmail-excluded-categories'; @Injectable() export class GmailFullMessageListFetchService { @@ -167,7 +168,7 @@ export class GmailFullMessageListFetchService { userId: 'me', maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT, pageToken, - q: gmailSearchFilterEmailAdresses(includedEmails, blocklistedEmails), + q: computeGmailCategoryExcludeSearchFilter(GMAIL_EXCLUDED_CATEGORIES), }); if (response.data?.messages) { diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-messages-import/gmail-messages-import-v2.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-messages-import/gmail-messages-import-v2.service.ts index 92d535e123..80cb11f0c2 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-messages-import/gmail-messages-import-v2.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-messages-import/gmail-messages-import-v2.service.ts @@ -18,6 +18,10 @@ import { MessageChannelSyncStatusService } from 'src/modules/messaging/services/ import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service'; import { GmailMessagesImportService } from 'src/modules/messaging/services/gmail-messages-import/gmail-messages-import.service'; import { MessagingTelemetryService } from 'src/modules/messaging/services/telemetry/messaging-telemetry.service'; +import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity'; +import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; +import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository'; +import { filterEmails } from 'src/modules/messaging/services/utils/filter-emails.util'; @Injectable() export class GmailMessagesImportV2Service { @@ -32,6 +36,8 @@ export class GmailMessagesImportV2Service { private readonly gmailErrorHandlingService: GmailErrorHandlingService, private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService, private readonly messagingTelemetryService: MessagingTelemetryService, + @InjectObjectMetadataRepository(BlocklistWorkspaceEntity) + private readonly blocklistRepository: BlocklistRepository, ) {} async processMessageBatchImport( @@ -88,7 +94,7 @@ export class GmailMessagesImportV2Service { const messageQueries = createQueriesFromMessageIds(messageIdsToFetch); try { - const messagesToSave = + const allMessages = await this.fetchMessagesByBatchesService.fetchAllMessages( messageQueries, connectedAccount.accessToken, @@ -96,6 +102,16 @@ export class GmailMessagesImportV2Service { connectedAccount.id, ); + const blocklist = await this.blocklistRepository.getByWorkspaceMemberId( + connectedAccount.accountOwnerId, + workspaceId, + ); + + const messagesToSave = filterEmails( + allMessages, + blocklist.map((blocklistItem) => blocklistItem.handle), + ); + if (!messagesToSave.length) { await this.messageChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch( messageChannel.id, diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service.ts new file mode 100644 index 0000000000..b0df2f31ca --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service.ts @@ -0,0 +1,46 @@ +import { Injectable } from '@nestjs/common'; + +import { gmail_v1 } from 'googleapis'; + +import { GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/constants/gmail-excluded-categories'; +import { GmailGetHistoryService } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service'; +import { computeGmailCategoryLabelId } from 'src/modules/messaging/utils/compute-gmail-category-label-id'; +import { assertNotNull } from 'src/utils/assert'; + +@Injectable() +export class GmailFetchMessageIdsToExcludeService { + constructor( + private readonly gmailGetHistoryService: GmailGetHistoryService, + ) {} + + public async fetchEmailIdsToExcludeOrThrow( + gmailClient: gmail_v1.Gmail, + lastSyncHistoryId: string, + ): Promise { + const emailIds: string[] = []; + + for (const category of GMAIL_EXCLUDED_CATEGORIES) { + const { history, error } = await this.gmailGetHistoryService.getHistory( + gmailClient, + lastSyncHistoryId, + ['messageAdded'], + computeGmailCategoryLabelId(category), + ); + + if (error) { + throw error; + } + + const emailIdsFromCategory = history + .map((history) => history.messagesAdded) + .flat() + .map((message) => message?.message?.id) + .filter((id) => id) + .filter(assertNotNull); + + emailIds.push(...emailIdsFromCategory); + } + + return emailIds; + } +} diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service.ts index 84373a2409..ebb68e2d0f 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service.ts @@ -13,6 +13,8 @@ export class GmailGetHistoryService { public async getHistory( gmailClient: gmail_v1.Gmail, lastSyncHistoryId: string, + historyTypes?: ('messageAdded' | 'messageDeleted')[], + labelId?: string, ): Promise<{ history: gmail_v1.Schema$History[]; historyId?: string | null; @@ -31,7 +33,8 @@ export class GmailGetHistoryService { maxResults: GMAIL_USERS_HISTORY_MAX_RESULT, pageToken, startHistoryId: lastSyncHistoryId, - historyTypes: ['messageAdded', 'messageDeleted'], + historyTypes: historyTypes || ['messageAdded', 'messageDeleted'], + labelId, }); } catch (error) { return { diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch-v2.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch-v2.service.ts index 31d5db0f4a..91270d78f0 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch-v2.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch-v2.service.ts @@ -16,6 +16,7 @@ import { GmailGetHistoryService } from 'src/modules/messaging/services/gmail-par import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record'; import { GmailErrorHandlingService } from 'src/modules/messaging/services/gmail-error-handling/gmail-error-handling.service'; import { MessageChannelSyncStatusService } from 'src/modules/messaging/services/message-channel-sync-status/message-channel-sync-status.service'; +import { GmailFetchMessageIdsToExcludeService } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service'; @Injectable() export class GmailPartialMessageListFetchV2Service { @@ -36,6 +37,7 @@ export class GmailPartialMessageListFetchV2Service { private readonly gmailErrorHandlingService: GmailErrorHandlingService, private readonly gmailGetHistoryService: GmailGetHistoryService, private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService, + private readonly gmailFetchMessageIdsToExcludeService: GmailFetchMessageIdsToExcludeService, ) {} public async processMessageListFetch( @@ -94,13 +96,36 @@ export class GmailPartialMessageListFetchV2Service { const { messagesAdded, messagesDeleted } = await this.gmailGetHistoryService.getMessageIdsFromHistory(history); + let messageIdsToFilter: string[] = []; + + try { + messageIdsToFilter = + await this.gmailFetchMessageIdsToExcludeService.fetchEmailIdsToExcludeOrThrow( + gmailClient, + lastSyncHistoryId, + ); + } catch (error) { + await this.gmailErrorHandlingService.handleGmailError( + error, + 'partial-message-list-fetch', + messageChannel, + workspaceId, + ); + + return; + } + + const messagesAddedFiltered = messagesAdded.filter( + (messageId) => !messageIdsToFilter.includes(messageId), + ); + await this.cacheStorage.setAdd( `messages-to-import:${workspaceId}:gmail:${messageChannel.id}`, - messagesAdded, + messagesAddedFiltered, ); this.logger.log( - `Added ${messagesAdded.length} messages to import for workspace ${workspaceId} and account ${connectedAccount.id}`, + `Added ${messagesAddedFiltered.length} messages to import for workspace ${workspaceId} and account ${connectedAccount.id}`, ); await this.messageChannelMessageAssociationRepository.deleteByMessageExternalIdsAndMessageChannelId( diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch.module.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch.module.ts index 99168cadcd..cd0a0c31f9 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch.module.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch.module.ts @@ -8,6 +8,7 @@ import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module'; import { GmailErrorHandlingModule } from 'src/modules/messaging/services/gmail-error-handling/gmail-error-handling.module'; +import { GmailFetchMessageIdsToExcludeService } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-fetch-messages-ids-to-exclude.service'; import { GmailGetHistoryService } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-get-history.service'; import { GmailPartialMessageListFetchV2Service } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch-v2.service'; import { GmailPartialMessageListFetchService } from 'src/modules/messaging/services/gmail-partial-message-list-fetch/gmail-partial-message-list-fetch.service'; @@ -35,6 +36,7 @@ import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/standard-ob GmailPartialMessageListFetchService, GmailPartialMessageListFetchV2Service, GmailGetHistoryService, + GmailFetchMessageIdsToExcludeService, ], exports: [ GmailPartialMessageListFetchService, diff --git a/packages/twenty-server/src/modules/messaging/services/utils/filter-emails.util.ts b/packages/twenty-server/src/modules/messaging/services/utils/filter-emails.util.ts new file mode 100644 index 0000000000..2a95c0f5fa --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/services/utils/filter-emails.util.ts @@ -0,0 +1,56 @@ +import { GmailMessage } from 'src/modules/messaging/types/gmail-message'; +import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util'; + +// Todo: refactor this into several utils +export const filterEmails = (messages: GmailMessage[], blocklist: string[]) => { + return filterOutBlocklistedMessages( + filterOutIcsAttachments(filterOutNonPersonalEmails(messages)), + blocklist, + ); +}; + +const filterOutBlocklistedMessages = ( + messages: GmailMessage[], + blocklist: string[], +) => { + return messages.filter((message) => { + if (!message.participants) { + return true; + } + + return message.participants.every( + (participant) => !isEmailBlocklisted(participant.handle, blocklist), + ); + }); +}; + +const filterOutIcsAttachments = (messages: GmailMessage[]) => { + return messages.filter((message) => { + if (!message.attachments) { + return true; + } + + return message.attachments.every( + (attachment) => !attachment.filename.endsWith('.ics'), + ); + }); +}; + +const isPersonEmail = (email: string): boolean => { + const nonPersonalPattern = + /noreply|no-reply|do_not_reply|no\.reply|^(info@|contact@|hello@|support@|feedback@|service@|help@|invites@|invite@|welcome@|alerts@|team@)/; + + return !nonPersonalPattern.test(email); +}; + +const filterOutNonPersonalEmails = (messages: GmailMessage[]) => { + return messages.filter((message) => { + if (!message.participants) { + return true; + } + + return message.participants.every((participant) => + isPersonEmail(participant.handle), + ); + }); +}; diff --git a/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts b/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts new file mode 100644 index 0000000000..d77f177f07 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts @@ -0,0 +1,27 @@ +import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/utils/compute-gmail-category-excude-search-filter'; + +describe('computeGmailCategoryExcludeSearchFilter', () => { + it('should return correct exclude search filter with empty category array', () => { + const result = computeGmailCategoryExcludeSearchFilter([]); + + expect(result).toBe(''); + }); + + it('should return correct exclude search filter with one category', () => { + const result = computeGmailCategoryExcludeSearchFilter(['CATEGORY1']); + + expect(result).toBe('-category:CATEGORY1'); + }); + + it('should return correct exclude search filter with multiple categories', () => { + const result = computeGmailCategoryExcludeSearchFilter([ + 'CATEGORY1', + 'CATEGORY2', + 'CATEGORY3', + ]); + + expect(result).toBe( + '-category:CATEGORY1 -category:CATEGORY2 -category:CATEGORY3', + ); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-label-id.spec.ts b/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-label-id.spec.ts new file mode 100644 index 0000000000..2373669efe --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/utils/__tests__/compute-gmail-category-label-id.spec.ts @@ -0,0 +1,9 @@ +import { computeGmailCategoryLabelId } from 'src/modules/messaging/utils/compute-gmail-category-label-id'; + +describe('computeGmailCategoryLabelId', () => { + it('should return correct category label id', () => { + const result = computeGmailCategoryLabelId('CATEGORY1'); + + expect(result).toBe('CATEGORY_CATEGORY1'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/utils/__tests__/gmail-search-filter.util.spec.ts b/packages/twenty-server/src/modules/messaging/utils/__tests__/gmail-search-filter.util.spec.ts deleted file mode 100644 index b8f1152c41..0000000000 --- a/packages/twenty-server/src/modules/messaging/utils/__tests__/gmail-search-filter.util.spec.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - excludedCategoriesAndFileTypesString, - gmailSearchFilterEmailAdresses, - gmailSearchFilterExcludeEmailAdresses, - gmailSearchFilterIncludeOnlyEmailAdresses, - gmailSearchFilterNonPersonalEmails, -} from 'src/modules/messaging/utils/gmail-search-filter.util'; - -describe('gmailSearchFilterExcludeEmailAdresses', () => { - it('should return correct search filter for excluding emails', () => { - const emails = ['hello@twenty.com', 'hey@twenty.com']; - const result = gmailSearchFilterExcludeEmailAdresses(emails); - - expect(result).toBe( - `(in:inbox from:-(${gmailSearchFilterNonPersonalEmails}|hello@twenty.com|hey@twenty.com)|(in:sent to:-(${gmailSearchFilterNonPersonalEmails}|hello@twenty.com|hey@twenty.com)) ${excludedCategoriesAndFileTypesString}`, - ); - }); - it('should return correct search filter for excluding emails when no emails are provided', () => { - const result = gmailSearchFilterExcludeEmailAdresses(); - - expect(result).toBe( - `from:-(${gmailSearchFilterNonPersonalEmails}) ${excludedCategoriesAndFileTypesString}`, - ); - }); -}); - -describe('gmailSearchFilterIncludeOnlyEmailAdresses', () => { - it('should return correct search filter for including emails', () => { - const emails = ['hello@twenty.com', 'hey@twenty.com']; - const result = gmailSearchFilterIncludeOnlyEmailAdresses(emails); - - expect(result).toBe( - `(in:inbox from:(hello@twenty.com|hey@twenty.com)|(in:sent to:(hello@twenty.com|hey@twenty.com)) ${excludedCategoriesAndFileTypesString}`, - ); - }); - it('should return undefined when no emails are provided', () => { - const result = gmailSearchFilterIncludeOnlyEmailAdresses(); - - expect(result).toBe(undefined); - }); -}); - -describe('gmailSearchFilterEmailAdresses', () => { - it('should return correct search filter for including emails and excluding emails', () => { - const includedEmails = ['hello@twenty.com', 'hey@twenty.com']; - - const excludedEmails = ['noreply@twenty.com', 'no-reply@twenty.com']; - - const result = gmailSearchFilterEmailAdresses( - includedEmails, - excludedEmails, - ); - - expect(result).toBe( - `(in:inbox from:((hello@twenty.com|hey@twenty.com) -(${gmailSearchFilterNonPersonalEmails}|noreply@twenty.com|no-reply@twenty.com))|(in:sent to:((hello@twenty.com|hey@twenty.com) -(${gmailSearchFilterNonPersonalEmails}|noreply@twenty.com|no-reply@twenty.com)) ${excludedCategoriesAndFileTypesString}`, - ); - }); -}); diff --git a/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-excude-search-filter.ts b/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-excude-search-filter.ts new file mode 100644 index 0000000000..e8deb1140d --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-excude-search-filter.ts @@ -0,0 +1,3 @@ +export const computeGmailCategoryExcludeSearchFilter = ( + excludedCategories: string[], +) => excludedCategories.map((category) => `-category:${category}`).join(' '); diff --git a/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-label-id.ts b/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-label-id.ts new file mode 100644 index 0000000000..7c3de6e2a6 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/utils/compute-gmail-category-label-id.ts @@ -0,0 +1,2 @@ +export const computeGmailCategoryLabelId = (category: string) => + `CATEGORY_${category.toUpperCase()}`; diff --git a/packages/twenty-server/src/modules/messaging/utils/gmail-search-filter.util.ts b/packages/twenty-server/src/modules/messaging/utils/gmail-search-filter.util.ts deleted file mode 100644 index cb823c179d..0000000000 --- a/packages/twenty-server/src/modules/messaging/utils/gmail-search-filter.util.ts +++ /dev/null @@ -1,59 +0,0 @@ -export const gmailSearchFilterNonPersonalEmails = - '*noreply@|*no-reply@|*do_not_reply@|*no.reply@|*info@|*contact@|*hello@|*support@|*feedback@|*service@|*help@'; - -export const excludedCategories = ['promotions', 'social', 'forums']; - -export const excludedFileTypes = ['.ics']; - -export const excludedCategoriesAndFileTypesString = `-category:${excludedCategories.join( - ' -category:', -)} -filename:${excludedFileTypes.join(' -filename:')}`; - -export const gmailSearchFilterExcludeEmailAdresses = ( - emails?: string[], -): string => { - if (!emails || emails.length === 0) { - return `from:-(${gmailSearchFilterNonPersonalEmails}) ${excludedCategoriesAndFileTypesString}`; - } - - return `(in:inbox from:-(${gmailSearchFilterNonPersonalEmails}|${emails.join( - '|', - )})|(in:sent to:-(${gmailSearchFilterNonPersonalEmails}|${emails.join( - '|', - )})) ${excludedCategoriesAndFileTypesString}`; -}; - -export const gmailSearchFilterIncludeOnlyEmailAdresses = ( - emails?: string[], -): string | undefined => { - if (!emails || emails.length === 0) { - return undefined; - } - - return `(in:inbox from:(${emails.join('|')})|(in:sent to:(${emails.join( - '|', - )})) ${excludedCategoriesAndFileTypesString}`; -}; - -export const gmailSearchFilterEmailAdresses = ( - includedEmails?: string[] | undefined, - excludedEmails?: string[] | undefined, -): string | undefined => { - if (!includedEmails || includedEmails.length === 0) { - return gmailSearchFilterExcludeEmailAdresses(excludedEmails); - } - - if (!excludedEmails || excludedEmails.length === 0) { - return gmailSearchFilterIncludeOnlyEmailAdresses(includedEmails); - } - - return `(in:inbox from:((${includedEmails.join( - '|', - )}) -(${gmailSearchFilterNonPersonalEmails}|${excludedEmails.join( - '|', - )}))|(in:sent to:((${includedEmails.join( - '|', - )}) -(${gmailSearchFilterNonPersonalEmails}|${excludedEmails.join( - '|', - )})) ${excludedCategoriesAndFileTypesString}`; -};