4747 create deleted listener on blocklist (#5067)

Closes #4747
This commit is contained in:
bosiraphael 2024-04-24 16:10:56 +02:00 committed by GitHub
parent adbc8ab96f
commit 0f47426d19
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 364 additions and 21 deletions

View File

@ -47,6 +47,7 @@ import { NotFoundError } from 'src/engine/utils/graphql-errors.util';
import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
import { WorkspaceQueryRunnerOptions } from './interfaces/query-runner-option.interface';
import {
@ -432,6 +433,12 @@ export class WorkspaceQueryRunnerService {
workspaceId,
objectMetadataItem,
);
const deletedBlocklistItem = await this.handleDeleteBlocklistItem(
args.id,
workspaceId,
objectMetadataItem,
);
// TODO END
const result = await this.execute(query, workspaceId);
@ -459,6 +466,7 @@ export class WorkspaceQueryRunnerService {
properties: {
before: {
...(deletedWorkspaceMember ?? {}),
...(deletedBlocklistItem ?? {}),
...this.removeNestedProperties(parsedResults?.[0]),
},
},
@ -615,4 +623,36 @@ export class WorkspaceQueryRunnerService {
return workspaceMemberResult.edges?.[0]?.node;
}
async handleDeleteBlocklistItem(
id: string,
workspaceId: string,
objectMetadataItem: ObjectMetadataInterface,
) {
if (objectMetadataItem.standardId !== STANDARD_OBJECT_IDS.blocklist) {
return;
}
const blocklistItemResult = await this.executeAndParse<IRecord>(
`
query {
blocklistCollection(filter: {id: {eq: "${id}"}}) {
edges {
node {
handle
workspaceMember {
id
}
}
}
}
}
`,
objectMetadataItem,
'',
workspaceId,
);
return blocklistItemResult.edges?.[0]?.node;
}
}

View File

@ -44,6 +44,7 @@ import { WorkspaceGoogleCalendarSyncModule } from 'src/modules/calendar/services
import { AutoCompaniesAndContactsCreationModule } from 'src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module';
import { GmailFetchMessagesFromCacheCronJob } from 'src/modules/messaging/crons/jobs/gmail-fetch-messages-from-cache.cron.job';
import { GmailPartialSyncCronJob } from 'src/modules/messaging/crons/jobs/gmail-partial-sync.cron.job';
import { BlocklistReimportMessagesJob } from 'src/modules/messaging/jobs/blocklist-reimport-messages.job';
import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/jobs/blocklist-item-delete-messages.job';
import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job';
@ -59,6 +60,7 @@ import { TimelineActivityModule } from 'src/modules/timeline/timeline-activity.m
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { BlocklistItemDeleteCalendarEventsJob } from 'src/modules/calendar/jobs/blocklist-item-delete-calendar-events.job';
import { BlocklistReimportCalendarEventsJob } from 'src/modules/calendar/jobs/blocklist-reimport-calendar-events.job';
@Module({
imports: [
@ -191,6 +193,14 @@ import { BlocklistItemDeleteCalendarEventsJob } from 'src/modules/calendar/jobs/
provide: BlocklistItemDeleteCalendarEventsJob.name,
useClass: BlocklistItemDeleteCalendarEventsJob,
},
{
provide: BlocklistReimportMessagesJob.name,
useClass: BlocklistReimportMessagesJob,
},
{
provide: BlocklistReimportCalendarEventsJob.name,
useClass: BlocklistReimportCalendarEventsJob,
},
],
})
export class JobsModule {

View File

@ -3,6 +3,7 @@ import { FieldMetadataInterface } from './field-metadata.interface';
export interface ObjectMetadataInterface {
id: string;
standardId?: string | null;
nameSingular: string;
namePlural: string;
labelSingular: string;

View File

@ -0,0 +1,59 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { GoogleCalendarSyncService } from 'src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
export type BlocklistReimportCalendarEventsJobData = {
workspaceId: string;
workspaceMemberId: string;
handle: string;
};
@Injectable()
export class BlocklistReimportCalendarEventsJob
implements MessageQueueJob<BlocklistReimportCalendarEventsJobData>
{
private readonly logger = new Logger(BlocklistReimportCalendarEventsJob.name);
constructor(
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly googleCalendarSyncService: GoogleCalendarSyncService,
) {}
async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
const { workspaceId, workspaceMemberId, handle } = data;
this.logger.log(
`Reimporting calendar events from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const connectedAccount =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccount || connectedAccount.length === 0) {
this.logger.error(
`No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`,
);
return;
}
await this.googleCalendarSyncService.startGoogleCalendarSync(
workspaceId,
connectedAccount[0].id,
handle,
);
this.logger.log(
`Reimporting calendar events from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`,
);
}
}

View File

@ -2,12 +2,17 @@ import { Inject, Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import {
BlocklistItemDeleteCalendarEventsJobData,
BlocklistItemDeleteCalendarEventsJob,
} from 'src/modules/calendar/jobs/blocklist-item-delete-calendar-events.job';
import {
BlocklistReimportCalendarEventsJobData,
BlocklistReimportCalendarEventsJob,
} from 'src/modules/calendar/jobs/blocklist-reimport-calendar-events.job';
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
@Injectable()
@ -29,4 +34,18 @@ export class CalendarBlocklistListener {
},
);
}
@OnEvent('blocklist.deleted')
async handleDeletedEvent(
payload: ObjectRecordDeleteEvent<BlocklistObjectMetadata>,
) {
await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
BlocklistReimportCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
workspaceMemberId: payload.properties.before.workspaceMember.id,
handle: payload.properties.before.handle,
},
);
}
}

View File

@ -68,6 +68,7 @@ export class GoogleCalendarSyncService {
public async startGoogleCalendarSync(
workspaceId: string,
connectedAccountId: string,
emailOrDomainToReimport?: string,
): Promise<void> {
const connectedAccount = await this.connectedAccountRepository.getById(
connectedAccountId,
@ -137,8 +138,9 @@ export class GoogleCalendarSyncService {
const googleCalendarEvents = await googleCalendarClient.events.list({
calendarId: 'primary',
maxResults: 500,
syncToken,
syncToken: emailOrDomainToReimport ? undefined : syncToken,
pageToken: nextPageToken,
q: emailOrDomainToReimport,
showDeleted: true,
});
@ -174,10 +176,19 @@ export class GoogleCalendarSyncService {
return;
}
const filteredEvents = filterOutBlocklistedEvents(
events,
blocklistedEmails,
);
let filteredEvents = filterOutBlocklistedEvents(events, blocklistedEmails);
if (emailOrDomainToReimport) {
// We still need to filter the events to only keep the ones that have the email or domain we want to reimport
// because the q parameter in the list method also filters the events that have the email or domain in their summary, description ...
// The q parameter allows us to narrow down the events
filteredEvents = filteredEvents.filter(
(event) =>
event.attendees?.some(
(attendee) => attendee.email?.endsWith(emailOrDomainToReimport),
),
);
}
const eventExternalIds = filteredEvents.map((event) => event.id as string);

View File

@ -43,6 +43,25 @@ export class ConnectedAccountRepository {
);
}
public async getAllByWorkspaceMemberId(
workspaceMemberId: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<ConnectedAccountObjectMetadata>[] | undefined> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const connectedAccounts =
await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."connectedAccount" WHERE "accountOwnerId" = $1`,
[workspaceMemberId],
workspaceId,
transactionManager,
);
return connectedAccounts;
}
public async getAllByHandleAndWorkspaceMemberId(
handle: string,
workspaceMemberId: string,

View File

@ -0,0 +1,59 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { GmailFullSyncService } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service';
export type BlocklistReimportMessagesJobData = {
workspaceId: string;
workspaceMemberId: string;
handle: string;
};
@Injectable()
export class BlocklistReimportMessagesJob
implements MessageQueueJob<BlocklistReimportMessagesJobData>
{
private readonly logger = new Logger(BlocklistReimportMessagesJob.name);
constructor(
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly gmailFullSyncService: GmailFullSyncService,
) {}
async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
const { workspaceId, workspaceMemberId, handle } = data;
this.logger.log(
`Reimporting messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const connectedAccount =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccount || connectedAccount.length === 0) {
this.logger.error(
`No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`,
);
return;
}
await this.gmailFullSyncService.fetchConnectedAccountThreads(
workspaceId,
connectedAccount[0].id,
[handle],
);
this.logger.log(
`Reimporting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`,
);
}
}

View File

@ -2,9 +2,14 @@ import { Inject, Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
import {
BlocklistReimportMessagesJob,
BlocklistReimportMessagesJobData,
} from 'src/modules/messaging/jobs/blocklist-reimport-messages.job';
import {
BlocklistItemDeleteMessagesJobData,
BlocklistItemDeleteMessagesJob,
@ -18,10 +23,10 @@ export class MessagingBlocklistListener {
) {}
@OnEvent('blocklist.created')
handleCreatedEvent(
async handleCreatedEvent(
payload: ObjectRecordCreateEvent<BlocklistObjectMetadata>,
) {
this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
BlocklistItemDeleteMessagesJob.name,
{
workspaceId: payload.workspaceId,
@ -29,4 +34,18 @@ export class MessagingBlocklistListener {
},
);
}
@OnEvent('blocklist.deleted')
async handleDeletedEvent(
payload: ObjectRecordDeleteEvent<BlocklistObjectMetadata>,
) {
await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
BlocklistReimportMessagesJob.name,
{
workspaceId: payload.workspaceId,
workspaceMemberId: payload.properties.before.workspaceMember.id,
handle: payload.properties.before.handle,
},
);
}
}

View File

@ -25,8 +25,8 @@ import {
MessageChannelObjectMetadata,
MessageChannelSyncStatus,
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmail-search-filter.util';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { gmailSearchFilterEmailAdresses } from 'src/modules/messaging/utils/gmail-search-filter.util';
@Injectable()
export class GmailFullSyncService {
@ -54,6 +54,7 @@ export class GmailFullSyncService {
public async fetchConnectedAccountThreads(
workspaceId: string,
connectedAccountId: string,
includedEmails?: string[],
) {
const connectedAccount = await this.connectedAccountRepository.getById(
connectedAccountId,
@ -109,19 +110,20 @@ export class GmailFullSyncService {
workspaceId,
);
const gmailClient: gmail_v1.Gmail =
await this.gmailClientProvider.getGmailClient(refreshToken);
const blocklistedEmails = await this.fetchBlocklistEmails(
connectedAccount.accountOwnerId,
workspaceId,
);
await workspaceDataSource
?.transaction(async (transactionManager) => {
const gmailClient: gmail_v1.Gmail =
await this.gmailClientProvider.getGmailClient(refreshToken);
const blocklistedEmails = await this.fetchBlocklistEmails(
connectedAccount.accountOwnerId,
workspaceId,
);
await this.fetchAllMessageIdsFromGmailAndStoreInCache(
gmailClient,
gmailMessageChannel.id,
includedEmails || [],
blocklistedEmails,
workspaceId,
transactionManager,
@ -150,6 +152,7 @@ export class GmailFullSyncService {
public async fetchAllMessageIdsFromGmailAndStoreInCache(
gmailClient: gmail_v1.Gmail,
messageChannelId: string,
includedEmails: string[],
blocklistedEmails: string[],
workspaceId: string,
transactionManager?: EntityManager,
@ -164,7 +167,7 @@ export class GmailFullSyncService {
userId: 'me',
maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
pageToken,
q: gmailSearchFilterExcludeEmails(blocklistedEmails),
q: gmailSearchFilterEmailAdresses(includedEmails, blocklistedEmails),
});
if (response.data?.messages) {

View File

@ -0,0 +1,58 @@
import {
excludedCategoriesAndFileTypesString,
gmailSearchFilterEmailAdresses,
gmailSearchFilterExcludeEmailAdresses,
gmailSearchFilterIncludeOnlyEmailAdresses,
gmailSearchFilterNonPersonalEmails,
} from 'src/modules/messaging/utils/gmail-search-filter.util';
describe('gmailSearchFilterExcludeEmailAdresses', () => {
it('should return correct search filter for excluding emails', () => {
const emails = ['hello@twenty.com', 'hey@twenty.com'];
const result = gmailSearchFilterExcludeEmailAdresses(emails);
expect(result).toBe(
`(in:inbox from:-(${gmailSearchFilterNonPersonalEmails}|hello@twenty.com|hey@twenty.com)|(in:sent to:-(${gmailSearchFilterNonPersonalEmails}|hello@twenty.com|hey@twenty.com)) ${excludedCategoriesAndFileTypesString}`,
);
});
it('should return correct search filter for excluding emails when no emails are provided', () => {
const result = gmailSearchFilterExcludeEmailAdresses();
expect(result).toBe(
`from:-(${gmailSearchFilterNonPersonalEmails}) ${excludedCategoriesAndFileTypesString}`,
);
});
});
describe('gmailSearchFilterIncludeOnlyEmailAdresses', () => {
it('should return correct search filter for including emails', () => {
const emails = ['hello@twenty.com', 'hey@twenty.com'];
const result = gmailSearchFilterIncludeOnlyEmailAdresses(emails);
expect(result).toBe(
`(in:inbox from:(hello@twenty.com|hey@twenty.com)|(in:sent to:(hello@twenty.com|hey@twenty.com)) ${excludedCategoriesAndFileTypesString}`,
);
});
it('should return undefined when no emails are provided', () => {
const result = gmailSearchFilterIncludeOnlyEmailAdresses();
expect(result).toBe(undefined);
});
});
describe('gmailSearchFilterEmailAdresses', () => {
it('should return correct search filter for including emails and excluding emails', () => {
const includedEmails = ['hello@twenty.com', 'hey@twenty.com'];
const excludedEmails = ['noreply@twenty.com', 'no-reply@twenty.com'];
const result = gmailSearchFilterEmailAdresses(
includedEmails,
excludedEmails,
);
expect(result).toBe(
`(in:inbox from:((hello@twenty.com|hey@twenty.com) -(${gmailSearchFilterNonPersonalEmails}|noreply@twenty.com|no-reply@twenty.com))|(in:sent to:((hello@twenty.com|hey@twenty.com) -(${gmailSearchFilterNonPersonalEmails}|noreply@twenty.com|no-reply@twenty.com)) ${excludedCategoriesAndFileTypesString}`,
);
});
});

View File

@ -1,14 +1,59 @@
export const gmailSearchFilterNonPersonalEmails =
'*noreply@|*no-reply@|*do_not_reply@|*no.reply@|*info@|*contact@|*hello@|*support@|*feedback@|*service@|*help@';
export const gmailSearchFilterExcludeEmails = (emails: string[]): string => {
if (emails.length === 0) {
return `from:-(${gmailSearchFilterNonPersonalEmails}) -category:promotions -category:social -category:forums -filename:.ics`;
export const excludedCategories = ['promotions', 'social', 'forums'];
export const excludedFileTypes = ['.ics'];
export const excludedCategoriesAndFileTypesString = `-category:${excludedCategories.join(
' -category:',
)} -filename:${excludedFileTypes.join(' -filename:')}`;
export const gmailSearchFilterExcludeEmailAdresses = (
emails?: string[],
): string => {
if (!emails || emails.length === 0) {
return `from:-(${gmailSearchFilterNonPersonalEmails}) ${excludedCategoriesAndFileTypesString}`;
}
return `(in:inbox from:-(${gmailSearchFilterNonPersonalEmails}|${emails.join(
'|',
)})|(in:sent to:-(${gmailSearchFilterNonPersonalEmails}|${emails.join(
'|',
)})) -category:promotions -category:social -category:forums -filename:.ics`;
)})) ${excludedCategoriesAndFileTypesString}`;
};
export const gmailSearchFilterIncludeOnlyEmailAdresses = (
emails?: string[],
): string | undefined => {
if (!emails || emails.length === 0) {
return undefined;
}
return `(in:inbox from:(${emails.join('|')})|(in:sent to:(${emails.join(
'|',
)})) ${excludedCategoriesAndFileTypesString}`;
};
export const gmailSearchFilterEmailAdresses = (
includedEmails?: string[] | undefined,
excludedEmails?: string[] | undefined,
): string | undefined => {
if (!includedEmails || includedEmails.length === 0) {
return gmailSearchFilterExcludeEmailAdresses(excludedEmails);
}
if (!excludedEmails || excludedEmails.length === 0) {
return gmailSearchFilterIncludeOnlyEmailAdresses(includedEmails);
}
return `(in:inbox from:((${includedEmails.join(
'|',
)}) -(${gmailSearchFilterNonPersonalEmails}|${excludedEmails.join(
'|',
)}))|(in:sent to:((${includedEmails.join(
'|',
)}) -(${gmailSearchFilterNonPersonalEmails}|${excludedEmails.join(
'|',
)})) ${excludedCategoriesAndFileTypesString}`;
};