mirror of
https://github.com/twentyhq/twenty.git
synced 2024-10-04 12:57:10 +03:00
[messaging] Add message deletion during partial sync (#4972)
## Context - Rename remaining V2 services. - Delete messages in DB when gmail history tells us they've been deleted. I removed the logic where we store those in a cache since it's a bit overkill because we don't need to query gmail and can use those ids directly. The strategy is to delete the message channel message association of the current channel, not the message or the thread since they can still be linked to other channels. However, we will need to call the threadCleaner service on the workspace to remove orphan threads/non-associated messages. Note: deletion for full-sync is a bit tricky because we need the full list of message ids to compare with the DB and make sure we don't over-delete. Currently, to keep memory, we don't have a variable that holds all ids as we flush it after each page. Easier solution would be to wipe everything before each full sync but it's probably not great for the user experience if they are currently manipulating messages since full-sync can happen without a user intervention (if a partial sync fails due to historyId being invalidated by google for some reason)
This commit is contained in:
Weiko
GitHub
parent
19a3be7b1b
commit
2efc794b43
@ -58,6 +58,11 @@ export class TimelineMessagingService {
|
||||
${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
|
||||
WHERE
|
||||
"messageParticipant"."personId" = ANY($1)
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma
|
||||
WHERE mcma."messageId" = message.id
|
||||
)
|
||||
GROUP BY
|
||||
message."messageThreadId",
|
||||
message.id
|
||||
@ -128,6 +133,11 @@ export class TimelineMessagingService {
|
||||
${dataSourceSchema}."message" message
|
||||
WHERE
|
||||
message."messageThreadId" = ANY($1)
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma
|
||||
WHERE mcma."messageId" = message.id
|
||||
)
|
||||
GROUP BY
|
||||
message."messageThreadId"
|
||||
`,
|
||||
@ -248,6 +258,11 @@ export class TimelineMessagingService {
|
||||
${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id
|
||||
WHERE
|
||||
"messageParticipant"."personId" = ANY($1)
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma
|
||||
WHERE mcma."messageId" = message.id
|
||||
)
|
||||
`,
|
||||
[personIds],
|
||||
workspaceId,
|
||||
|
||||
@ -48,8 +48,8 @@ import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job
|
||||
import { GmailPartialSyncJob } from 'src/modules/messaging/jobs/gmail-partial-sync.job';
|
||||
import { MessagingCreateCompanyAndContactAfterSyncJob } from 'src/modules/messaging/jobs/messaging-create-company-and-contact-after-sync.job';
|
||||
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
|
||||
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
|
||||
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
|
||||
import { GmailFullSyncModule } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module';
|
||||
import { GmailPartialSyncModule } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module';
|
||||
import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module';
|
||||
import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
@ -81,9 +81,9 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
|
||||
MessageChannelObjectMetadata,
|
||||
EventObjectMetadata,
|
||||
]),
|
||||
GmailFullSynV2Module,
|
||||
GmailFullSyncModule,
|
||||
GmailFetchMessageContentFromCacheModule,
|
||||
GmailPartialSyncV2Module,
|
||||
GmailPartialSyncModule,
|
||||
CalendarEventParticipantModule,
|
||||
],
|
||||
providers: [
|
||||
|
||||
@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||
import { GmailFullSyncService } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service';
|
||||
|
||||
export type GmailFullSyncJobData = {
|
||||
workspaceId: string;
|
||||
@ -16,7 +16,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
|
||||
|
||||
constructor(
|
||||
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
|
||||
private readonly gmailFullSyncV2Service: GmailFullSyncV2Service,
|
||||
private readonly gmailFullSyncV2Service: GmailFullSyncService,
|
||||
) {}
|
||||
|
||||
async handle(data: GmailFullSyncJobData): Promise<void> {
|
||||
|
||||
@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service';
|
||||
|
||||
export type GmailPartialSyncJobData = {
|
||||
workspaceId: string;
|
||||
|
||||
@ -7,7 +7,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||
import { GmailFullSyncService } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
@ -25,7 +25,7 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailFullSyncV2Service],
|
||||
exports: [GmailFullSyncV2Service],
|
||||
providers: [GmailFullSyncService],
|
||||
exports: [GmailFullSyncService],
|
||||
})
|
||||
export class GmailFullSynV2Module {}
|
||||
export class GmailFullSyncModule {}
|
||||
@ -29,8 +29,8 @@ import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmai
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFullSyncV2Service {
|
||||
private readonly logger = new Logger(GmailFullSyncV2Service.name);
|
||||
export class GmailFullSyncService {
|
||||
private readonly logger = new Logger(GmailFullSyncService.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
@ -7,7 +7,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||
import { GmailPartialSyncV2Service as GmailPartialSyncService } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service';
|
||||
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||
@ -27,7 +27,7 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailPartialSyncV2Service],
|
||||
exports: [GmailPartialSyncV2Service],
|
||||
providers: [GmailPartialSyncService],
|
||||
exports: [GmailPartialSyncService],
|
||||
})
|
||||
export class GmailPartialSyncV2Module {}
|
||||
export class GmailPartialSyncModule {}
|
||||
@ -24,6 +24,8 @@ import {
|
||||
GmailFullSyncJob,
|
||||
GmailFullSyncJobData,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync.job';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository';
|
||||
|
||||
@Injectable()
|
||||
export class GmailPartialSyncV2Service {
|
||||
@ -40,6 +42,10 @@ export class GmailPartialSyncV2Service {
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
@InjectObjectMetadataRepository(
|
||||
MessageChannelMessageAssociationObjectMetadata,
|
||||
)
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
@ -201,9 +207,11 @@ export class GmailPartialSyncV2Service {
|
||||
messagesAdded,
|
||||
);
|
||||
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-delete:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||
await this.messageChannelMessageAssociationRepository.deleteByMessageExternalIdsAndMessageChannelId(
|
||||
messagesDeleted,
|
||||
gmailMessageChannel.id,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateLastSyncCursorIfHigher(
|
||||
Loading…
Reference in New Issue
Block a user