twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-12-27 22:32:49 +03:00
Code Issues Packages Projects Releases Wiki Activity

4285 timebox create google calendar full sync (#4442)

Browse Source 
* calendar module

* wip

* creating a folder for common files between calendar and messages

* wip

* wip

* wip

* wip

* update calendar search filter

* wip

* working on full sync service

* reorganizing folders

* adding repositories

* fix typo

* working on full-sync service

* Add calendarQueue to MessageQueue enum and update dependencies

* start transaction

* wip

* add save and update functions for event

* wip

* save events

* improving step by step

* add calendar scope

* fix nest modules imports

* renaming

* create calendar channel

* create job for google calendar full-sync

* call GoogleCalendarFullSyncJob after connected account creation

* ask for scope conditionnally

* fixes

* create channels conditionnally

* fix

* fixes

* fix FK bug

* filter out canceled events

* create save and update functions for calendarEventAttendee repository

* saving messageParticipants is working

* save calendarEventAttendees is working

* add calendarEvent cleaner

* calendar event cleaner is working

* working on updating attendees

* wip

* reintroducing google-gmail endpoint to ensure smooth deploy

* modify callbackURL

* modify front url

* changes to be able to merge

* put back feature flag

* fixes after PR comments

* add feature flag check

* remove unused modules

* separate delete connected account associated job data in two jobs

* fix error

* rename calendar_v3 as calendarV3

* Update packages/twenty-server/src/workspace/calendar-and-messaging/utils/valueStringForBatchRawQuery.util.ts

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>

* improve readability

* renaming to remove plural

* renaming to remove plural

* don't throw if no connected account is found

* use calendar queue

* modify usage of HttpService in fetch-by-batch

* modify valuesStringForBatchRawQuery to improve api and return flattened values

* fix auth module feature flag import

* fix getFlattenedValuesAndValuesStringForBatchRawQuery

---------

Co-authored-by: Jérémy M <jeremy.magrin@gmail.com>
This commit is contained in:
bosiraphael 2024-03-14 11:23:31 +01:00 committed by GitHub
parent e0dac82e07
commit 3caf860848
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
76 changed files with 1856 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-docs/docs/start/self-hosting/self-hosting.mdx
View File

@ -68,7 +68,9 @@ import TabItem from '@theme/TabItem';
<OptionTable options={[
['MESSAGING_PROVIDER_GMAIL_ENABLED', 'false', 'Enable Gmail API connection'],
['CALENDAR_PROVIDER_GMAIL_ENABLED', 'false', 'Enable Google Calendar API connection'],
['MESSAGING_PROVIDER_GMAIL_CALLBACK_URL', '', 'Gmail auth callback'],
['AUTH_GOOGLE_APIS_CALLBACK_URL', '', 'Google API auth callback'],
['AUTH_GOOGLE_ENABLED', 'false', 'Enable Goole SSO login'],
['AUTH_GOOGLE_CLIENT_ID', '', 'Google client ID'],
['AUTH_GOOGLE_CLIENT_SECRET', '', 'Google client secret'],

2
packages/twenty-server/.env.example
View File

@ -20,6 +20,7 @@ SIGN_IN_PREFILLED=true
# FRONT_AUTH_CALLBACK_URL=http://localhost:3001/verify
# AUTH_GOOGLE_ENABLED=false
# MESSAGING_PROVIDER_GMAIL_ENABLED=false
# CALENDAR_PROVIDER_GOOGLE_ENABLED=false
# IS_BILLING_ENABLED=false
# BILLING_PLAN_REQUIRED_LINK=https://twenty.com/stripe-redirection
# IS_SIGN_UP_DISABLED=false
@ -27,6 +28,7 @@ SIGN_IN_PREFILLED=true
# AUTH_GOOGLE_CLIENT_SECRET=replace_me_with_google_client_secret
# AUTH_GOOGLE_CALLBACK_URL=http://localhost:3000/auth/google/redirect
# MESSAGING_PROVIDER_GMAIL_CALLBACK_URL=http://localhost:3000/auth/google-gmail/get-access-token
# AUTH_GOOGLE_APIS_CALLBACK_URL=http://localhost:3000/auth/google-apis/get-access-token
# STORAGE_TYPE=local
# STORAGE_LOCAL_PATH=.local-storage
# SUPPORT_DRIVER=front

2
packages/twenty-server/src/command.module.ts
View File

@ -4,6 +4,7 @@ import { DatabaseCommandModule } from 'src/database/commands/database-command.mo
import { FetchWorkspaceMessagesCommandsModule } from 'src/workspace/messaging/commands/fetch-workspace-messages-commands.module';
import { WorkspaceHealthCommandModule } from 'src/workspace/workspace-health/commands/workspace-health-command.module';
import { WorkspaceCleanerModule } from 'src/workspace/workspace-cleaner/workspace-cleaner.module';
import { WorkspaceCalendarSyncCommandsModule } from 'src/workspace/calendar/commands/workspace-calendar-sync-commands.module';
import { AppModule } from './app.module';
@ -16,6 +17,7 @@ import { WorkspaceMigrationRunnerCommandsModule } from './workspace/workspace-mi
WorkspaceSyncMetadataCommandsModule,
DatabaseCommandModule,
FetchWorkspaceMessagesCommandsModule,
WorkspaceCalendarSyncCommandsModule,
WorkspaceCleanerModule,
WorkspaceHealthCommandModule,
WorkspaceMigrationRunnerCommandsModule,

14
packages/twenty-server/src/core/auth/auth.module.ts
View File

@ -14,12 +14,14 @@ import { UserModule } from 'src/core/user/user.module';
import { WorkspaceManagerModule } from 'src/workspace/workspace-manager/workspace-manager.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { GoogleAuthController } from 'src/core/auth/controllers/google-auth.controller';
import { GoogleGmailAuthController } from 'src/core/auth/controllers/google-gmail-auth.controller';
import { GoogleAPIsAuthController } from 'src/core/auth/controllers/google-apis-auth.controller';
import { VerifyAuthController } from 'src/core/auth/controllers/verify-auth.controller';
import { TokenService } from 'src/core/auth/services/token.service';
import { GoogleGmailService } from 'src/core/auth/services/google-gmail.service';
import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
import { UserWorkspaceModule } from 'src/core/user-workspace/user-workspace.module';
import { SignUpService } from 'src/core/auth/services/sign-up.service';
import { GoogleGmailAuthController } from 'src/core/auth/controllers/google-gmail-auth.controller';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { AuthResolver } from './auth.resolver';
@ -45,12 +47,16 @@ const jwtModule = JwtModule.registerAsync({
UserModule,
WorkspaceManagerModule,
TypeORMModule,
TypeOrmModule.forFeature([Workspace, User, RefreshToken], 'core'),
TypeOrmModule.forFeature(
[Workspace, User, RefreshToken, FeatureFlagEntity],
'core',
),
HttpModule,
UserWorkspaceModule,
],
controllers: [
GoogleAuthController,
GoogleAPIsAuthController,
GoogleGmailAuthController,
VerifyAuthController,
],
@ -60,7 +66,7 @@ const jwtModule = JwtModule.registerAsync({
TokenService,
JwtAuthStrategy,
AuthResolver,
GoogleGmailService,
GoogleAPIsService,
],
exports: [jwtModule, TokenService],
})

63
packages/twenty-server/src/core/auth/controllers/google-apis-auth.controller.ts Normal file
View File

@ -0,0 +1,63 @@
import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common';
import { Response } from 'express';
import { GoogleAPIsProviderEnabledGuard } from 'src/core/auth/guards/google-apis-provider-enabled.guard';
import { GoogleAPIsOauthGuard } from 'src/core/auth/guards/google-apis-oauth.guard';
import { GoogleAPIsRequest } from 'src/core/auth/strategies/google-apis.auth.strategy';
import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
import { TokenService } from 'src/core/auth/services/token.service';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Controller('auth/google-apis')
export class GoogleAPIsAuthController {
constructor(
private readonly googleAPIsService: GoogleAPIsService,
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
) {}
@Get()
@UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
async googleAuth() {
// As this method is protected by Google Auth guard, it will trigger Google SSO flow
return;
}
@Get('get-access-token')
@UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
async googleAuthGetAccessToken(
@Req() req: GoogleAPIsRequest,
@Res() res: Response,
) {
const { user } = req;
const { email, accessToken, refreshToken, transientToken } = user;
const { workspaceMemberId, workspaceId } =
await this.tokenService.verifyTransientToken(transientToken);
const demoWorkspaceIds = this.environmentService.getDemoWorkspaceIds();
if (demoWorkspaceIds.includes(workspaceId)) {
throw new Error('Cannot connect Google account to demo workspace');
}
if (!workspaceId) {
throw new Error('Workspace not found');
}
await this.googleAPIsService.saveConnectedAccount({
handle: email,
workspaceMemberId: workspaceMemberId,
workspaceId: workspaceId,
provider: 'google',
accessToken,
refreshToken,
});
return res.redirect(
`${this.environmentService.getFrontBaseUrl()}/settings/accounts`,
);
}
}

16
packages/twenty-server/src/core/auth/controllers/google-gmail-auth.controller.ts
View File

@ -2,32 +2,32 @@ import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common';
import { Response } from 'express';
import { GoogleGmailProviderEnabledGuard } from 'src/core/auth/guards/google-gmail-provider-enabled.guard';
import { GoogleGmailOauthGuard } from 'src/core/auth/guards/google-gmail-oauth.guard';
import { GoogleGmailRequest } from 'src/core/auth/strategies/google-gmail.auth.strategy';
import { GoogleGmailService } from 'src/core/auth/services/google-gmail.service';
import { GoogleAPIsOauthGuard } from 'src/core/auth/guards/google-apis-oauth.guard';
import { GoogleAPIsProviderEnabledGuard } from 'src/core/auth/guards/google-apis-provider-enabled.guard';
import { GoogleAPIsService } from 'src/core/auth/services/google-apis.service';
import { TokenService } from 'src/core/auth/services/token.service';
import { GoogleAPIsRequest } from 'src/core/auth/strategies/google-apis.auth.strategy';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Controller('auth/google-gmail')
export class GoogleGmailAuthController {
constructor(
private readonly googleGmailService: GoogleGmailService,
private readonly googleGmailService: GoogleAPIsService,
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
) {}
@Get()
@UseGuards(GoogleGmailProviderEnabledGuard, GoogleGmailOauthGuard)
@UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
async googleAuth() {
// As this method is protected by Google Auth guard, it will trigger Google SSO flow
return;
}
@Get('get-access-token')
@UseGuards(GoogleGmailProviderEnabledGuard, GoogleGmailOauthGuard)
@UseGuards(GoogleAPIsProviderEnabledGuard, GoogleAPIsOauthGuard)
async googleAuthGetAccessToken(
@Req() req: GoogleGmailRequest,
@Req() req: GoogleAPIsRequest,
@Res() res: Response,
) {
const { user } = req;

2
packages/twenty-server/src/core/auth/guards/google-gmail-oauth.guard.ts → packages/twenty-server/src/core/auth/guards/google-apis-oauth.guard.ts
View File

@ -2,7 +2,7 @@ import { ExecutionContext, Injectable } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
@Injectable()
export class GoogleGmailOauthGuard extends AuthGuard('google-gmail') {
export class GoogleAPIsOauthGuard extends AuthGuard('google-apis') {
constructor() {
super({
prompt: 'select_account',

24
packages/twenty-server/src/core/auth/guards/google-apis-provider-enabled.guard.ts Normal file
View File

@ -0,0 +1,24 @@
import { Injectable, CanActivate, NotFoundException } from '@nestjs/common';
import { Observable } from 'rxjs';
import { GoogleAPIsStrategy } from 'src/core/auth/strategies/google-apis.auth.strategy';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Injectable()
export class GoogleAPIsProviderEnabledGuard implements CanActivate {
constructor(private readonly environmentService: EnvironmentService) {}
canActivate(): boolean | Promise<boolean> | Observable<boolean> {
if (
!this.environmentService.isMessagingProviderGmailEnabled() &&
!this.environmentService.isCalendarProviderGoogleEnabled()
) {
throw new NotFoundException('Google apis auth is not enabled');
}
new GoogleAPIsStrategy(this.environmentService);
return true;
}
}

21
packages/twenty-server/src/core/auth/guards/google-gmail-provider-enabled.guard.ts
View File

@ -1,21 +0,0 @@
import { Injectable, CanActivate, NotFoundException } from '@nestjs/common';
import { Observable } from 'rxjs';
import { GoogleGmailStrategy } from 'src/core/auth/strategies/google-gmail.auth.strategy';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Injectable()
export class GoogleGmailProviderEnabledGuard implements CanActivate {
constructor(private readonly environmentService: EnvironmentService) {}
canActivate(): boolean | Promise<boolean> | Observable<boolean> {
if (!this.environmentService.isMessagingProviderGmailEnabled()) {
throw new NotFoundException('Gmail auth is not enabled');
}
new GoogleGmailStrategy(this.environmentService);
return true;
}
}

82
packages/twenty-server/src/core/auth/services/google-gmail.service.ts → packages/twenty-server/src/core/auth/services/google-apis.service.ts
View File

@ -1,6 +1,8 @@
import { ConflictException, Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { v4 } from 'uuid';
import { Repository } from 'typeorm';
import { DataSourceService } from 'src/metadata/data-source/data-source.service';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
@ -11,14 +13,28 @@ import {
GmailFullSyncJob,
GmailFullSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import {
GoogleCalendarFullSyncJob,
GoogleCalendarFullSyncJobData,
} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/core/feature-flag/feature-flag.entity';
@Injectable()
export class GoogleGmailService {
export class GoogleAPIsService {
constructor(
private readonly dataSourceService: DataSourceService,
private readonly typeORMService: TypeORMService,
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@Inject(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
private readonly environmentService: EnvironmentService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {}
providerName = 'google';
@ -53,6 +69,12 @@ export class GoogleGmailService {
const connectedAccountId = v4();
const IsCalendarEnabled = await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsCalendarEnabled,
value: true,
});
await workspaceDataSource?.transaction(async (manager) => {
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`,
@ -66,22 +88,52 @@ export class GoogleGmailService {
],
);
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`,
['share_everything', handle, connectedAccountId, 'email'],
);
if (this.environmentService.isMessagingProviderGmailEnabled()) {
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."messageChannel" ("visibility", "handle", "connectedAccountId", "type") VALUES ($1, $2, $3, $4)`,
['share_everything', handle, connectedAccountId, 'email'],
);
}
if (
this.environmentService.isCalendarProviderGoogleEnabled() &&
IsCalendarEnabled
) {
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."calendarChannel" ("visibility", "handle", "connectedAccountId") VALUES ($1, $2, $3)`,
['SHARE_EVERYTHING', handle, connectedAccountId],
);
}
});
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId,
},
{
retryLimit: 2,
},
);
if (this.environmentService.isMessagingProviderGmailEnabled()) {
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId,
},
{
retryLimit: 2,
},
);
}
if (
this.environmentService.isCalendarProviderGoogleEnabled() &&
IsCalendarEnabled
) {
await this.calendarQueueService.add<GoogleCalendarFullSyncJobData>(
GoogleCalendarFullSyncJob.name,
{
workspaceId,
connectedAccountId,
},
{
retryLimit: 2,
},
);
}
return;
}

30
packages/twenty-server/src/core/auth/strategies/google-gmail.auth.strategy.ts → packages/twenty-server/src/core/auth/strategies/google-apis.auth.strategy.ts
View File

@ -6,7 +6,7 @@ import { Request } from 'express';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
export type GoogleGmailRequest = Request & {
export type GoogleAPIsRequest = Request & {
user: {
firstName?: string | null;
lastName?: string | null;
@ -20,20 +20,28 @@ export type GoogleGmailRequest = Request & {
};
@Injectable()
export class GoogleGmailStrategy extends PassportStrategy(
export class GoogleAPIsStrategy extends PassportStrategy(
Strategy,
'google-gmail',
'google-apis',
) {
constructor(environmentService: EnvironmentService) {
const scope = ['email', 'profile'];
if (environmentService.isMessagingProviderGmailEnabled()) {
scope.push('https://www.googleapis.com/auth/gmail.readonly');
}
if (environmentService.isCalendarProviderGoogleEnabled()) {
scope.push('https://www.googleapis.com/auth/calendar');
}
super({
clientID: environmentService.getAuthGoogleClientId(),
clientSecret: environmentService.getAuthGoogleClientSecret(),
callbackURL: environmentService.getMessagingProviderGmailCallbackUrl(),
scope: [
'email',
'profile',
'https://www.googleapis.com/auth/gmail.readonly',
],
callbackURL: environmentService.isCalendarProviderGoogleEnabled()
? environmentService.getAuthGoogleAPIsCallbackUrl()
: environmentService.getMessagingProviderGmailCallbackUrl(),
scope,
passReqToCallback: true,
});
}
@ -52,7 +60,7 @@ export class GoogleGmailStrategy extends PassportStrategy(
}
async validate(
request: GoogleGmailRequest,
request: GoogleAPIsRequest,
accessToken: string,
refreshToken: string,
profile: any,
@ -65,7 +73,7 @@ export class GoogleGmailStrategy extends PassportStrategy(
? JSON.parse(request.query.state)
: undefined;
const user: GoogleGmailRequest['user'] = {
const user: GoogleAPIsRequest['user'] = {
email: emails[0].value,
firstName: name.givenName,
lastName: name.familyName,

11
packages/twenty-server/src/integrations/environment/environment.service.ts
View File

@ -155,12 +155,23 @@ export class EnvironmentService {
);
}
isCalendarProviderGoogleEnabled(): boolean {
return (
this.configService.get<boolean>('CALENDAR_PROVIDER_GOOGLE_ENABLED') ??
false
);
}
getMessagingProviderGmailCallbackUrl(): string | undefined {
return this.configService.get<string>(
'MESSAGING_PROVIDER_GMAIL_CALLBACK_URL',
);
}
getAuthGoogleAPIsCallbackUrl(): string | undefined {
return this.configService.get<string>('AUTH_GOOGLE_APIS_CALLBACK_URL');
}
isAuthGoogleEnabled(): boolean {
return this.configService.get<boolean>('AUTH_GOOGLE_ENABLED') ?? false;
}

24
packages/twenty-server/src/integrations/message-queue/jobs.module.ts
View File

@ -17,15 +17,15 @@ import { EmailSenderJob } from 'src/integrations/email/email-sender.job';
import { UserModule } from 'src/core/user/user.module';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { FetchAllWorkspacesMessagesJob } from 'src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { MatchMessageParticipantJob } from 'src/workspace/messaging/jobs/match-message-participant.job';
import { CreateCompaniesAndContactsAfterSyncJob } from 'src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job';
import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module';
import { MessageParticipantModule } from 'src/workspace/messaging/repositories/message-participant/message-participant.module';
import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module';
import { DataSeedDemoWorkspaceJob } from 'src/database/commands/data-seed-demo-workspace/jobs/data-seed-demo-workspace.job';
import { DeleteConnectedAccountAssociatedDataJob } from 'src/workspace/messaging/jobs/delete-connected-acount-associated-data.job';
import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { ThreadCleanerModule } from 'src/workspace/messaging/services/thread-cleaner/thread-cleaner.module';
import { UpdateSubscriptionJob } from 'src/core/billing/jobs/update-subscription.job';
import { BillingModule } from 'src/core/billing/billing.module';
@ -33,9 +33,13 @@ import { UserWorkspaceModule } from 'src/core/user-workspace/user-workspace.modu
import { StripeModule } from 'src/core/billing/stripe/stripe.module';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { CalendarModule } from 'src/workspace/calendar/calendar.module';
import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
import { GoogleCalendarFullSyncJob } from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
import { CalendarEventCleanerModule } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { RecordPositionBackfillJob } from 'src/workspace/workspace-query-runner/jobs/record-position-backfill.job';
import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runner/services/record-position-backfill-module';
import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job';
@Module({
imports: [
@ -49,9 +53,11 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn
MessagingModule,
MessageParticipantModule,
MessageChannelModule,
CalendarModule,
ObjectMetadataModule,
StripeModule,
ThreadCleanerModule,
CalendarEventCleanerModule,
TypeORMModule,
TypeOrmModule.forFeature([Workspace, FeatureFlagEntity], 'core'),
TypeOrmModule.forFeature([DataSourceEntity], 'metadata'),
@ -69,6 +75,10 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn
provide: GmailPartialSyncJob.name,
useClass: GmailPartialSyncJob,
},
{
provide: GoogleCalendarFullSyncJob.name,
useClass: GoogleCalendarFullSyncJob,
},
{
provide: CallWebhookJobsJob.name,
useClass: CallWebhookJobsJob,
@ -99,8 +109,12 @@ import { RecordPositionBackfillModule } from 'src/workspace/workspace-query-runn
useClass: DataSeedDemoWorkspaceJob,
},
{
provide: DeleteConnectedAccountAssociatedDataJob.name,
useClass: DeleteConnectedAccountAssociatedDataJob,
provide: DeleteConnectedAccountAssociatedMessagingDataJob.name,
useClass: DeleteConnectedAccountAssociatedMessagingDataJob,
},
{
provide: DeleteConnectedAccountAssociatedCalendarDataJob.name,
useClass: DeleteConnectedAccountAssociatedCalendarDataJob,
},
{ provide: UpdateSubscriptionJob.name, useClass: UpdateSubscriptionJob },
{

1
packages/twenty-server/src/integrations/message-queue/message-queue.constants.ts
View File

@ -6,6 +6,7 @@ export enum MessageQueue {
webhookQueue = 'webhook-queue',
cronQueue = 'cron-queue',
emailQueue = 'email-queue',
calendarQueue = 'calendar-queue',
billingQueue = 'billing-queue',
recordPositionBackfillQueue = 'record-position-backfill-queue',
}

21
packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module.ts Normal file
View File

@ -0,0 +1,21 @@
import { Module } from '@nestjs/common';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import { CreateCompanyModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module';
import { CreateContactModule } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
WorkspaceDataSourceModule,
CreateContactModule,
CreateCompanyModule,
WorkspaceMemberModule,
PersonModule,
],
providers: [CreateCompanyAndContactService],
exports: [CreateCompanyAndContactService],
})
export class CreateCompaniesAndContactsModule {}

10
packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service.ts → packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service.ts
View File

@ -5,16 +5,16 @@ import compact from 'lodash/compact';
import { Participant } from 'src/workspace/messaging/types/gmail-message';
import { getDomainNameFromHandle } from 'src/workspace/messaging/utils/get-domain-name-from-handle.util';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
import { getUniqueParticipantsAndHandles } from 'src/workspace/messaging/utils/get-unique-participants-and-handles.util';
import { filterOutParticipantsFromCompanyOrWorkspace } from 'src/workspace/messaging/utils/filter-out-participants-from-company-or-workspace.util';
import { isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class CreateCompaniesAndContactsService {
export class CreateCompanyAndContactService {
constructor(
private readonly personService: PersonService,
private readonly createContactService: CreateContactService,

2
packages/twenty-server/src/workspace/messaging/services/create-company/create-company.module.ts → packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.module.ts
View File

@ -1,7 +1,7 @@
import { Module } from '@nestjs/common';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
@Module({

0
packages/twenty-server/src/workspace/messaging/services/create-company/create-company.service.ts → packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service.ts
View File

4
packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.module.ts → packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.module.ts
View File

@ -1,8 +1,8 @@
import { Module } from '@nestjs/common';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
@Module({
imports: [WorkspaceDataSourceModule, PersonModule],

2
packages/twenty-server/src/workspace/messaging/services/create-contact/create-contact.service.ts → packages/twenty-server/src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service.ts
View File

@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/workspace/messaging/utils/get-first-name-and-last-name-from-handle-and-display-name.util';
type ContactToCreate = {

2
packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.module.ts → packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module.ts
View File

@ -1,6 +1,6 @@
import { Module } from '@nestjs/common';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({

0
packages/twenty-server/src/workspace/messaging/repositories/blocklist/blocklist.service.ts → packages/twenty-server/src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service.ts
View File

2
packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.module.ts → packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module.ts
View File

@ -1,6 +1,6 @@
import { Module } from '@nestjs/common';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({

0
packages/twenty-server/src/workspace/messaging/repositories/connected-account/connected-account.service.ts → packages/twenty-server/src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service.ts
View File

4
packages/twenty-server/src/workspace/messaging/services/gmail-refresh-access-token.service.ts → packages/twenty-server/src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service.ts
View File

@ -3,10 +3,10 @@ import { Injectable } from '@nestjs/common';
import axios from 'axios';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
@Injectable()
export class GmailRefreshAccessTokenService {
export class GoogleAPIsRefreshAccessTokenService {
constructor(
private readonly environmentService: EnvironmentService,
private readonly connectedAccountService: ConnectedAccountService,

5
packages/twenty-server/src/workspace/calendar-and-messaging/types/batch-queries.ts Normal file
View File

@ -0,0 +1,5 @@
type Query = {
uri: string;
};
export type BatchQueries = Query[];

56
packages/twenty-server/src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util.ts Normal file
View File

@ -0,0 +1,56 @@
export const valuesStringForBatchRawQuery = (
values: {
[key: string]: any;
}[],
typesArray: string[] = [],
) => {
const castedValues = values.reduce((acc, _, rowIndex) => {
const numberOfColumns = typesArray.length;
const rowValues = Array.from(
{ length: numberOfColumns },
(_, columnIndex) => {
const placeholder = `$${rowIndex * numberOfColumns + columnIndex + 1}`;
const typeCast = typesArray[columnIndex]
? `::${typesArray[columnIndex]}`
: '';
return `${placeholder}${typeCast}`;
},
).join(', ');
acc.push(`(${rowValues})`);
return acc;
}, [] as string[]);
return castedValues.join(', ');
};
export const getFlattenedValuesAndValuesStringForBatchRawQuery = (
values: {
[key: string]: any;
}[],
keyTypeMap: {
[key: string]: string;
},
): {
flattenedValues: any[];
valuesString: string;
} => {
const keysToInsert = Object.keys(keyTypeMap);
const flattenedValues = values.flatMap((value) =>
keysToInsert.map((key) => value[key]),
);
const valuesString = valuesStringForBatchRawQuery(
values,
Object.values(keyTypeMap),
);
return {
flattenedValues,
valuesString,
};
};

41
packages/twenty-server/src/workspace/calendar/calendar.module.ts Normal file
View File

@ -0,0 +1,41 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { CalendarChannelEventAssociationModule } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module';
import { CalendarChannelModule } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.module';
import { CalendarEventAttendeeModule } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module';
import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module';
import { CalendarEventCleanerModule } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
EnvironmentModule,
WorkspaceDataSourceModule,
ConnectedAccountModule,
CalendarChannelModule,
CalendarChannelEventAssociationModule,
CalendarEventModule,
CalendarEventAttendeeModule,
CreateCompaniesAndContactsModule,
WorkspaceMemberModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
CompanyModule,
PersonModule,
BlocklistModule,
CalendarEventCleanerModule,
],
providers: [GoogleCalendarFullSyncService, GoogleCalendarClientProvider],
exports: [GoogleCalendarFullSyncService],
})
export class CalendarModule {}

66
packages/twenty-server/src/workspace/calendar/commands/google-calendar-full-sync.command.ts Normal file
View File

@ -0,0 +1,66 @@
import { Inject } from '@nestjs/common';
import { Command, CommandRunner, Option } from 'nest-commander';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import {
GoogleCalendarFullSyncJobData,
GoogleCalendarFullSyncJob,
} from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
interface GoogleCalendarFullSyncOptions {
workspaceId: string;
}
@Command({
name: 'workspace:google-calendar-full-sync',
description:
'Start google calendar full-sync for all workspaceMembers in a workspace.',
})
export class GoogleCalendarFullSyncCommand extends CommandRunner {
constructor(
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
private readonly connectedAccountService: ConnectedAccountService,
) {
super();
}
async run(
_passedParam: string[],
options: GoogleCalendarFullSyncOptions,
): Promise<void> {
await this.fetchWorkspaceCalendars(options.workspaceId);
return;
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
private async fetchWorkspaceCalendars(workspaceId: string): Promise<void> {
const connectedAccounts =
await this.connectedAccountService.getAll(workspaceId);
for (const connectedAccount of connectedAccounts) {
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
GoogleCalendarFullSyncJob.name,
{
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
retryLimit: 2,
},
);
}
}
}

19
packages/twenty-server/src/workspace/calendar/commands/workspace-calendar-sync-commands.module.ts Normal file
View File

@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { GoogleCalendarFullSyncCommand } from 'src/workspace/calendar/commands/google-calendar-full-sync.command';
@Module({
imports: [
DataSourceModule,
TypeORMModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
ConnectedAccountModule,
],
providers: [GoogleCalendarFullSyncCommand],
})
export class WorkspaceCalendarSyncCommandsModule {}

53
packages/twenty-server/src/workspace/calendar/jobs/google-calendar-full-sync.job.ts Normal file
View File

@ -0,0 +1,53 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GoogleCalendarFullSyncService } from 'src/workspace/calendar/services/google-calendar-full-sync.service';
export type GoogleCalendarFullSyncJobData = {
workspaceId: string;
connectedAccountId: string;
nextPageToken?: string;
};
@Injectable()
export class GoogleCalendarFullSyncJob
implements MessageQueueJob<GoogleCalendarFullSyncJobData>
{
private readonly logger = new Logger(GoogleCalendarFullSyncJob.name);
constructor(
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly googleCalendarFullSyncService: GoogleCalendarFullSyncService,
) {}
async handle(data: GoogleCalendarFullSyncJobData): Promise<void> {
this.logger.log(
`google calendar full-sync for workspace ${
data.workspaceId
} and account ${data.connectedAccountId} ${
data.nextPageToken ? `and ${data.nextPageToken} pageToken` : ''
}`,
);
try {
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);
} catch (e) {
this.logger.error(
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
e,
);
return;
}
await this.googleCalendarFullSyncService.startGoogleCalendarFullSync(
data.workspaceId,
data.connectedAccountId,
data.nextPageToken,
);
}
}

11
packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-assocation.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarChannelEventAssociationService],
exports: [CalendarChannelEventAssociationService],
})
export class CalendarChannelEventAssociationModule {}

172
packages/twenty-server/src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service.ts Normal file
View File

@ -0,0 +1,172 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
@Injectable()
export class CalendarChannelEventAssociationService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByEventExternalIdsAndCalendarChannelId(
eventExternalIds: string[],
calendarChannelId: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (eventExternalIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`,
[eventExternalIds, calendarChannelId],
workspaceId,
transactionManager,
);
}
public async deleteByEventExternalIdsAndCalendarChannelId(
eventExternalIds: string[],
calendarChannelId: string,
workspaceId: string,
transactionManager?: EntityManager,
) {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "eventExternalId" = ANY($1) AND "calendarChannelId" = $2`,
[eventExternalIds, calendarChannelId],
workspaceId,
transactionManager,
);
}
public async getByCalendarChannelIds(
calendarChannelIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (calendarChannelIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "calendarChannelId" = ANY($1)`,
[calendarChannelIds],
workspaceId,
transactionManager,
);
}
public async deleteByCalendarChannelIds(
calendarChannelIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (calendarChannelIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "calendarChannelId" = ANY($1)`,
[calendarChannelIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
ids: string[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (ids.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarChannelEventAssociation" WHERE "id" = ANY($1)`,
[ids],
workspaceId,
transactionManager,
);
}
public async getByCalendarEventIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelEventAssociationObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannelEventAssociation"
WHERE "calendarEventId" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async saveCalendarChannelEventAssociations(
calendarChannelEventAssociations: Omit<
ObjectRecord<CalendarChannelEventAssociationObjectMetadata>,
'id' | 'createdAt' | 'updatedAt' | 'calendarChannel' | 'calendarEvent'
>[],
workspaceId: string,
transactionManager?: EntityManager,
) {
if (calendarChannelEventAssociations.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const {
flattenedValues: calendarChannelEventAssociationValues,
valuesString,
} = getFlattenedValuesAndValuesStringForBatchRawQuery(
calendarChannelEventAssociations,
{
calendarChannelId: 'uuid',
calendarEventId: 'uuid',
eventExternalId: 'text',
},
);
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarChannelEventAssociation" ("calendarChannelId", "calendarEventId", "eventExternalId")
VALUES ${valuesString}`,
calendarChannelEventAssociationValues,
workspaceId,
transactionManager,
);
}
}

11
packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarChannelService],
exports: [CalendarChannelService],
})
export class CalendarChannelModule {}

76
packages/twenty-server/src/workspace/calendar/repositories/calendar-channel/calendar-channel.service.ts Normal file
View File

@ -0,0 +1,76 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { CalendarChannelObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel.object-metadata';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
@Injectable()
export class CalendarChannelService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByConnectedAccountId(
connectedAccountId: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "connectedAccountId" = $1 LIMIT 1`,
[connectedAccountId],
workspaceId,
transactionManager,
);
}
public async getFirstByConnectedAccountIdOrFail(
connectedAccountId: string,
workspaceId: string,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>> {
const calendarChannels = await this.getByConnectedAccountId(
connectedAccountId,
workspaceId,
);
if (!calendarChannels || calendarChannels.length === 0) {
throw new Error(
`No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
);
}
return calendarChannels[0];
}
public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
connectedAccountId: string,
workspaceId: string,
): Promise<boolean> {
const calendarChannel = await this.getFirstByConnectedAccountIdOrFail(
connectedAccountId,
workspaceId,
);
return calendarChannel.isContactAutoCreationEnabled;
}
public async getByIds(
ids: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarChannelObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarChannel" WHERE "id" = ANY($1)`,
[ids],
workspaceId,
transactionManager,
);
}
}

11
packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarEventAttendeeService],
exports: [CalendarEventAttendeeService],
})
export class CalendarEventAttendeeModule {}

151
packages/twenty-server/src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service.ts Normal file
View File

@ -0,0 +1,151 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
import { CalendarEventAttendee } from 'src/workspace/calendar/types/calendar-event';
@Injectable()
export class CalendarEventAttendeeService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByIds(
calendarEventAttendeeIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
if (calendarEventAttendeeIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
[calendarEventAttendeeIds],
workspaceId,
transactionManager,
);
}
public async getByCalendarEventIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
calendarEventAttendeeIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendeeIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
[calendarEventAttendeeIds],
workspaceId,
transactionManager,
);
}
public async saveCalendarEventAttendees(
calendarEventAttendees: CalendarEventAttendee[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendees.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(
calendarEventAttendees,
{
calendarEventId: 'uuid',
handle: 'text',
displayName: 'text',
isOrganizer: 'boolean',
responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`,
},
);
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarEventAttendee" ("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus") VALUES ${valuesString}`,
flattenedValues,
workspaceId,
transactionManager,
);
}
public async updateCalendarEventAttendees(
calendarEventAttendees: CalendarEventAttendee[],
iCalUIDCalendarEventIdMap: Map<string, string>,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventAttendees.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const values = calendarEventAttendees.map((calendarEventAttendee) => ({
...calendarEventAttendee,
calendarEventId: iCalUIDCalendarEventIdMap.get(
calendarEventAttendee.iCalUID,
),
}));
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(values, {
calendarEventId: 'uuid',
handle: 'text',
displayName: 'text',
isOrganizer: 'boolean',
responseStatus: `${dataSourceSchema}."calendarEventAttendee_responsestatus_enum"`,
});
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."calendarEventAttendee" AS "calendarEventAttendee"
SET "displayName" = "newValues"."displayName",
"isOrganizer" = "newValues"."isOrganizer",
"responseStatus" = "newValues"."responseStatus"
FROM (VALUES ${valuesString}) AS "newValues"("calendarEventId", "handle", "displayName", "isOrganizer", "responseStatus")
WHERE "calendarEventAttendee"."handle" = "newValues"."handle"
AND "calendarEventAttendee"."calendarEventId" = "newValues"."calendarEventId"`,
flattenedValues,
workspaceId,
transactionManager,
);
}
}

11
packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [CalendarEventService],
exports: [CalendarEventService],
})
export class CalendarEventModule {}

202
packages/twenty-server/src/workspace/calendar/repositories/calendar-event/calendar-event.service.ts Normal file
View File

@ -0,0 +1,202 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata';
import { getFlattenedValuesAndValuesStringForBatchRawQuery } from 'src/workspace/calendar-and-messaging/utils/getFlattenedValuesAndValuesStringForBatchRawQuery.util';
import { CalendarEvent } from 'src/workspace/calendar/types/calendar-event';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
@Injectable()
export class CalendarEventService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async getByIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventObjectMetadata>[]> {
if (calendarEventIds.length === 0) {
return [];
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await this.workspaceDataSourceService.executeRawQuery(
`SELECT * FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async deleteByIds(
calendarEventIds: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEventIds.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`DELETE FROM ${dataSourceSchema}."calendarEvent" WHERE "id" = ANY($1)`,
[calendarEventIds],
workspaceId,
transactionManager,
);
}
public async getNonAssociatedCalendarEventIdsPaginated(
limit: number,
offset: number,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<ObjectRecord<CalendarEventAttendeeObjectMetadata>[]> {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const nonAssociatedCalendarEvents =
await this.workspaceDataSourceService.executeRawQuery(
`SELECT m.id FROM ${dataSourceSchema}."calendarEvent" m
LEFT JOIN ${dataSourceSchema}."calendarChannelEventAssociation" ccea
ON m.id = ccea."calendarEventId"
WHERE ccea.id IS NULL
LIMIT $1 OFFSET $2`,
[limit, offset],
workspaceId,
transactionManager,
);
return nonAssociatedCalendarEvents.map(({ id }) => id);
}
public async getICalUIDCalendarEventIdMap(
iCalUIDs: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<Map<string, string>> {
if (iCalUIDs.length === 0) {
return new Map();
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const calendarEvents: {
id: string;
iCalUID: string;
}[] = await this.workspaceDataSourceService.executeRawQuery(
`SELECT id, "iCalUID" FROM ${dataSourceSchema}."calendarEvent" WHERE "iCalUID" = ANY($1)`,
[iCalUIDs],
workspaceId,
transactionManager,
);
const iCalUIDsCalendarEvnetIdsMap = new Map<string, string>();
calendarEvents.forEach((calendarEvent) => {
iCalUIDsCalendarEvnetIdsMap.set(calendarEvent.iCalUID, calendarEvent.id);
});
return iCalUIDsCalendarEvnetIdsMap;
}
public async saveCalendarEvents(
calendarEvents: CalendarEvent[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEvents.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, {
id: 'uuid',
title: 'text',
isCanceled: 'boolean',
isFullDay: 'boolean',
startsAt: 'timestamptz',
endsAt: 'timestamptz',
externalCreatedAt: 'timestamptz',
externalUpdatedAt: 'timestamptz',
description: 'text',
location: 'text',
iCalUID: 'text',
conferenceSolution: 'text',
conferenceUri: 'text',
recurringEventExternalId: 'text',
});
await this.workspaceDataSourceService.executeRawQuery(
`INSERT INTO ${dataSourceSchema}."calendarEvent" ("id", "title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId") VALUES ${valuesString}`,
flattenedValues,
workspaceId,
transactionManager,
);
}
public async updateCalendarEvents(
calendarEvents: CalendarEvent[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<void> {
if (calendarEvents.length === 0) {
return;
}
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
const { flattenedValues, valuesString } =
getFlattenedValuesAndValuesStringForBatchRawQuery(calendarEvents, {
title: 'text',
isCanceled: 'boolean',
isFullDay: 'boolean',
startsAt: 'timestamptz',
endsAt: 'timestamptz',
externalCreatedAt: 'timestamptz',
externalUpdatedAt: 'timestamptz',
description: 'text',
location: 'text',
iCalUID: 'text',
conferenceSolution: 'text',
conferenceUri: 'text',
recurringEventExternalId: 'text',
});
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."calendarEvent" AS "calendarEvent"
SET "title" = "newData"."title",
"isCanceled" = "newData"."isCanceled",
"isFullDay" = "newData"."isFullDay",
"startsAt" = "newData"."startsAt",
"endsAt" = "newData"."endsAt",
"externalCreatedAt" = "newData"."externalCreatedAt",
"externalUpdatedAt" = "newData"."externalUpdatedAt",
"description" = "newData"."description",
"location" = "newData"."location",
"conferenceSolution" = "newData"."conferenceSolution",
"conferenceUri" = "newData"."conferenceUri",
"recurringEventExternalId" = "newData"."recurringEventExternalId"
FROM (VALUES ${valuesString})
AS "newData"("title", "isCanceled", "isFullDay", "startsAt", "endsAt", "externalCreatedAt", "externalUpdatedAt", "description", "location", "iCalUID", "conferenceSolution", "conferenceUri", "recurringEventExternalId")
WHERE "calendarEvent"."iCalUID" = "newData"."iCalUID"`,
flattenedValues,
workspaceId,
transactionManager,
);
}
}

13
packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module.ts Normal file
View File

@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { CalendarEventModule } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.module';
import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service';
@Module({
imports: [DataSourceModule, TypeORMModule, CalendarEventModule],
providers: [CalendarEventCleanerService],
exports: [CalendarEventCleanerService],
})
export class CalendarEventCleanerModule {}

20
packages/twenty-server/src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service.ts Normal file
View File

@ -0,0 +1,20 @@
import { Injectable } from '@nestjs/common';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { deleteUsingPagination } from 'src/workspace/messaging/services/thread-cleaner/utils/delete-using-pagination.util';
@Injectable()
export class CalendarEventCleanerService {
constructor(private readonly calendarEventService: CalendarEventService) {}
public async cleanWorkspaceCalendarEvents(workspaceId: string) {
await deleteUsingPagination(
workspaceId,
500,
this.calendarEventService.getNonAssociatedCalendarEventIdsPaginated.bind(
this.calendarEventService,
),
this.calendarEventService.deleteByIds.bind(this.calendarEventService),
);
}
}

274
packages/twenty-server/src/workspace/calendar/services/google-calendar-full-sync.service.ts Normal file
View File

@ -0,0 +1,274 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/core/feature-flag/feature-flag.entity';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
import { googleCalendarSearchFilterExcludeEmails } from 'src/workspace/calendar/utils/google-calendar-search-filter.util';
import { CalendarChannelEventAssociationService } from 'src/workspace/calendar/repositories/calendar-channel-event-association/calendar-channel-event-association.service';
import { CalendarChannelService } from 'src/workspace/calendar/repositories/calendar-channel/calendar-channel.service';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { CalendarEventService } from 'src/workspace/calendar/repositories/calendar-event/calendar-event.service';
import { formatGoogleCalendarEvent } from 'src/workspace/calendar/utils/format-google-calendar-event.util';
import { GoogleCalendarFullSyncJobData } from 'src/workspace/calendar/jobs/google-calendar-full-sync.job';
import { CalendarEventAttendeeService } from 'src/workspace/calendar/repositories/calendar-event-attendee/calendar-event-attendee.service';
@Injectable()
export class GoogleCalendarFullSyncService {
private readonly logger = new Logger(GoogleCalendarFullSyncService.name);
constructor(
private readonly googleCalendarClientProvider: GoogleCalendarClientProvider,
@Inject(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
private readonly connectedAccountService: ConnectedAccountService,
private readonly calendarEventService: CalendarEventService,
private readonly calendarChannelService: CalendarChannelService,
private readonly calendarChannelEventAssociationService: CalendarChannelEventAssociationService,
private readonly calendarEventAttendeesService: CalendarEventAttendeeService,
private readonly blocklistService: BlocklistService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async startGoogleCalendarFullSync(
workspaceId: string,
connectedAccountId: string,
pageToken?: string,
): Promise<void> {
const connectedAccount = await this.connectedAccountService.getById(
connectedAccountId,
workspaceId,
);
if (!connectedAccount) {
return;
}
const refreshToken = connectedAccount.refreshToken;
const workspaceMemberId = connectedAccount.accountOwnerId;
if (!refreshToken) {
throw new Error(
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
);
}
const calendarChannel =
await this.calendarChannelService.getFirstByConnectedAccountIdOrFail(
connectedAccountId,
workspaceId,
);
const calendarChannelId = calendarChannel.id;
const googleCalendarClient =
await this.googleCalendarClientProvider.getGoogleCalendarClient(
refreshToken,
);
const isBlocklistEnabledFeatureFlag =
await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsBlocklistEnabled,
value: true,
});
const isBlocklistEnabled =
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
const blocklist = isBlocklistEnabled
? await this.blocklistService.getByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
)
: [];
const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
let startTime = Date.now();
const googleCalendarEvents = await googleCalendarClient.events.list({
calendarId: 'primary',
maxResults: 500,
pageToken: pageToken,
q: googleCalendarSearchFilterExcludeEmails(blocklistedEmails),
});
let endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${
endTime - startTime
}ms.`,
);
const {
items: events,
nextPageToken,
nextSyncToken,
} = googleCalendarEvents.data;
if (!events || events?.length === 0) {
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
);
return;
}
const eventExternalIds = events.map((event) => event.id as string);
startTime = Date.now();
const existingCalendarChannelEventAssociations =
await this.calendarChannelEventAssociationService.getByEventExternalIdsAndCalendarChannelId(
eventExternalIds,
calendarChannelId,
workspaceId,
);
endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${
endTime - startTime
}ms.`,
);
// TODO: In V2, we will also import deleted events by doing batch GET queries on the canceled events
// The canceled events start and end are not accessible in the list query
const formattedEvents = events
.filter((event) => event.status !== 'cancelled')
.map((event) => formatGoogleCalendarEvent(event));
// TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE
const existingEventExternalIds =
existingCalendarChannelEventAssociations.map(
(association) => association.eventExternalId,
);
const eventsToSave = formattedEvents.filter(
(event) => !existingEventExternalIds.includes(event.externalId),
);
const eventsToUpdate = formattedEvents.filter((event) =>
existingEventExternalIds.includes(event.externalId),
);
const calendarChannelEventAssociationsToSave = eventsToSave.map(
(event) => ({
calendarEventId: event.id,
eventExternalId: event.externalId,
calendarChannelId,
}),
);
const attendeesToSave = eventsToSave.flatMap((event) => event.attendees);
const attendeesToUpdate = eventsToUpdate.flatMap(
(event) => event.attendees,
);
const iCalUIDCalendarEventIdMap =
await this.calendarEventService.getICalUIDCalendarEventIdMap(
eventsToUpdate.map((event) => event.iCalUID),
workspaceId,
);
if (events.length > 0) {
const dataSourceMetadata =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
dataSourceMetadata?.transaction(async (transactionManager) => {
this.calendarEventService.saveCalendarEvents(
eventsToSave,
workspaceId,
transactionManager,
);
this.calendarEventService.updateCalendarEvents(
eventsToUpdate,
workspaceId,
transactionManager,
);
this.calendarChannelEventAssociationService.saveCalendarChannelEventAssociations(
calendarChannelEventAssociationsToSave,
workspaceId,
transactionManager,
);
this.calendarEventAttendeesService.saveCalendarEventAttendees(
attendeesToSave,
workspaceId,
transactionManager,
);
this.calendarEventAttendeesService.updateCalendarEventAttendees(
attendeesToUpdate,
iCalUIDCalendarEventIdMap,
workspaceId,
transactionManager,
);
});
} else {
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
);
}
if (!nextSyncToken) {
throw new Error(
`No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
);
}
startTime = Date.now();
// await this.calendarChannelService.updateSyncCursor(
// nextSyncToken,
// connectedAccount.id,
// workspaceId,
// );
endTime = Date.now();
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${
endTime - startTime
}ms.`,
);
this.logger.log(
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} ${
nextPageToken ? `and ${nextPageToken} pageToken` : ''
} done.`,
);
if (nextPageToken) {
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
GoogleCalendarFullSyncService.name,
{
workspaceId,
connectedAccountId,
nextPageToken,
},
{
retryLimit: 2,
},
);
}
}
}

11
packages/twenty-server/src/workspace/calendar/services/providers/calendar-providers.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { EnvironmentModule } from 'src/integrations/environment/environment.module';
import { GoogleCalendarClientProvider } from 'src/workspace/calendar/services/providers/google-calendar/google-calendar.provider';
@Module({
imports: [EnvironmentModule],
providers: [GoogleCalendarClientProvider],
exports: [GoogleCalendarClientProvider],
})
export class CalendarProvidersModule {}

42
packages/twenty-server/src/workspace/calendar/services/providers/google-calendar/google-calendar.provider.ts Normal file
View File

@ -0,0 +1,42 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { calendar_v3 as calendarV3, google } from 'googleapis';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Injectable()
export class GoogleCalendarClientProvider {
constructor(private readonly environmentService: EnvironmentService) {}
public async getGoogleCalendarClient(
refreshToken: string,
): Promise<calendarV3.Calendar> {
const oAuth2Client = await this.getOAuth2Client(refreshToken);
const googleCalendarClient = google.calendar({
version: 'v3',
auth: oAuth2Client,
});
return googleCalendarClient;
}
private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
const googleCalendarClientId =
this.environmentService.getAuthGoogleClientId();
const googleCalendarClientSecret =
this.environmentService.getAuthGoogleClientSecret();
const oAuth2Client = new google.auth.OAuth2(
googleCalendarClientId,
googleCalendarClientSecret,
);
oAuth2Client.setCredentials({
refresh_token: refreshToken,
});
return oAuth2Client;
}
}

31
packages/twenty-server/src/workspace/calendar/types/calendar-event.ts Normal file
View File

@ -0,0 +1,31 @@
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { CalendarEventObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata';
import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
export type CalendarEvent = Omit<
ObjectRecord<CalendarEventObjectMetadata>,
| 'createdAt'
| 'updatedAt'
| 'calendarChannelEventAssociations'
| 'calendarEventAttendees'
| 'eventAttendees'
>;
export type CalendarEventAttendee = Omit<
ObjectRecord<CalendarEventAttendeeObjectMetadata>,
| 'id'
| 'createdAt'
| 'updatedAt'
| 'personId'
| 'workspaceMemberId'
| 'person'
| 'workspaceMember'
| 'calendarEvent'
> & {
iCalUID: string;
};
export type CalendarEventWithAttendees = CalendarEvent & {
externalId: string;
attendees: CalendarEventAttendee[];
};

52
packages/twenty-server/src/workspace/calendar/utils/format-google-calendar-event.util.ts Normal file
View File

@ -0,0 +1,52 @@
import { calendar_v3 } from 'googleapis';
import { v4 } from 'uuid';
import { CalendarEventWithAttendees } from 'src/workspace/calendar/types/calendar-event';
import { CalendarEventAttendeeResponseStatus } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
export const formatGoogleCalendarEvent = (
event: calendar_v3.Schema$Event,
): CalendarEventWithAttendees => {
const id = v4();
const formatResponseStatus = (status: string | null | undefined) => {
switch (status) {
case 'accepted':
return CalendarEventAttendeeResponseStatus.ACCEPTED;
case 'declined':
return CalendarEventAttendeeResponseStatus.DECLINED;
case 'tentative':
return CalendarEventAttendeeResponseStatus.TENTATIVE;
default:
return CalendarEventAttendeeResponseStatus.NEEDS_ACTION;
}
};
return {
id,
title: event.summary ?? '',
isCanceled: event.status === 'cancelled',
isFullDay: event.start?.dateTime == null,
startsAt: event.start?.dateTime ?? event.start?.date ?? null,
endsAt: event.end?.dateTime ?? event.end?.date ?? null,
externalId: event.id ?? '',
externalCreatedAt: event.created ?? null,
externalUpdatedAt: event.updated ?? null,
description: event.description ?? '',
location: event.location ?? '',
iCalUID: event.iCalUID ?? '',
conferenceSolution:
event.conferenceData?.conferenceSolution?.key?.type ?? '',
conferenceUri: event.conferenceData?.entryPoints?.[0]?.uri ?? '',
recurringEventExternalId: event.recurringEventId ?? '',
attendees:
event.attendees?.map((attendee) => ({
calendarEventId: id,
iCalUID: event.iCalUID ?? '',
handle: attendee.email ?? '',
displayName: attendee.displayName ?? '',
isOrganizer: attendee.organizer === true,
responseStatus: formatResponseStatus(attendee.responseStatus),
})) ?? [],
};
};

9
packages/twenty-server/src/workspace/calendar/utils/google-calendar-search-filter.util.ts Normal file
View File

@ -0,0 +1,9 @@
export const googleCalendarSearchFilterExcludeEmails = (
emails: string[],
): string => {
if (emails.length === 0) {
return '';
}
return `email=-${emails.join(', -')}`;
};

2
packages/twenty-server/src/workspace/messaging/commands/crons/fetch-all-workspaces-messages.job.ts
View File

@ -7,7 +7,7 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { Workspace } from 'src/core/workspace/workspace.entity';
import {
GmailPartialSyncJobData,

2
packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts
View File

@ -6,7 +6,7 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
import { GmailFullSyncCommand } from 'src/workspace/messaging/commands/gmail-full-sync.command';
import { GmailPartialSyncCommand } from 'src/workspace/messaging/commands/gmail-partial-sync.command';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/start-fetch-all-workspaces-messages.cron.command';
import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/workspace/messaging/commands/stop-fetch-all-workspaces-messages.cron.command';

2
packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts
View File

@ -8,7 +8,7 @@ import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
interface GmailFullSyncOptions {
workspaceId: string;

2
packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts
View File

@ -8,7 +8,7 @@ import {
GmailPartialSyncJob,
GmailPartialSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-partial-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
interface GmailPartialSyncOptions {
workspaceId: string;

4
packages/twenty-server/src/workspace/messaging/jobs/create-companies-and-contacts-after-sync.job.ts
View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
@ -19,7 +19,7 @@ export class CreateCompaniesAndContactsAfterSyncJob
CreateCompaniesAndContactsAfterSyncJob.name,
);
constructor(
private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService,
private readonly createCompaniesAndContactsService: CreateCompanyAndContactService,
private readonly messageChannelService: MessageChannelService,
private readonly messageParticipantService: MessageParticipantService,
) {}

40
packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job.ts Normal file
View File

@ -0,0 +1,40 @@
import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { CalendarEventCleanerService } from 'src/workspace/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service';
export type DeleteConnectedAccountAssociatedCalendarDataJobData = {
workspaceId: string;
connectedAccountId: string;
};
@Injectable()
export class DeleteConnectedAccountAssociatedCalendarDataJob
implements
MessageQueueJob<DeleteConnectedAccountAssociatedCalendarDataJobData>
{
private readonly logger = new Logger(
DeleteConnectedAccountAssociatedCalendarDataJob.name,
);
constructor(
private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {}
async handle(
data: DeleteConnectedAccountAssociatedCalendarDataJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
data.workspaceId,
);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated calendar data in workspace ${data.workspaceId}`,
);
}
}

15
packages/twenty-server/src/workspace/messaging/jobs/delete-connected-acount-associated-data.job.ts → packages/twenty-server/src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job.ts
View File

@ -4,32 +4,33 @@ import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/messa
import { ThreadCleanerService } from 'src/workspace/messaging/services/thread-cleaner/thread-cleaner.service';
export type DeleteConnectedAccountAssociatedDataJobData = {
export type DeleteConnectedAccountAssociatedMessagingDataJobData = {
workspaceId: string;
connectedAccountId: string;
};
@Injectable()
export class DeleteConnectedAccountAssociatedDataJob
implements MessageQueueJob<DeleteConnectedAccountAssociatedDataJobData>
export class DeleteConnectedAccountAssociatedMessagingDataJob
implements
MessageQueueJob<DeleteConnectedAccountAssociatedMessagingDataJobData>
{
private readonly logger = new Logger(
DeleteConnectedAccountAssociatedDataJob.name,
DeleteConnectedAccountAssociatedMessagingDataJob.name,
);
constructor(private readonly threadCleanerService: ThreadCleanerService) {}
async handle(
data: DeleteConnectedAccountAssociatedDataJobData,
data: DeleteConnectedAccountAssociatedMessagingDataJobData,
): Promise<void> {
this.logger.log(
`Deleting connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`,
`Deleting connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
await this.threadCleanerService.cleanWorkspaceThreads(data.workspaceId);
this.logger.log(
`Deleted connected account ${data.connectedAccountId} associated data in workspace ${data.workspaceId}`,
`Deleted connected account ${data.connectedAccountId} associated messaging data in workspace ${data.workspaceId}`,
);
}
}

6
packages/twenty-server/src/workspace/messaging/jobs/gmail-full-sync.job.ts
View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
export type GmailFullSyncJobData = {
@ -16,7 +16,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
private readonly logger = new Logger(GmailFullSyncJob.name);
constructor(
private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService,
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly gmailFullSyncService: GmailFullSyncService,
) {}
@ -28,7 +28,7 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
);
try {
await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken(
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);

6
packages/twenty-server/src/workspace/messaging/jobs/gmail-partial-sync.job.ts
View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service';
export type GmailPartialSyncJobData = {
@ -17,7 +17,7 @@ export class GmailPartialSyncJob
private readonly logger = new Logger(GmailPartialSyncJob.name);
constructor(
private readonly gmailRefreshAccessTokenService: GmailRefreshAccessTokenService,
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIsRefreshAccessTokenService,
private readonly gmailPartialSyncService: GmailPartialSyncService,
) {}
@ -27,7 +27,7 @@ export class GmailPartialSyncJob
);
try {
await this.gmailRefreshAccessTokenService.refreshAndSaveAccessToken(
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
);

24
packages/twenty-server/src/workspace/messaging/listeners/messaging-connected-account.listener.ts
View File

@ -5,9 +5,13 @@ import { ObjectRecordDeleteEvent } from 'src/integrations/event-emitter/types/ob
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import {
DeleteConnectedAccountAssociatedDataJobData,
DeleteConnectedAccountAssociatedDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-acount-associated-data.job';
DeleteConnectedAccountAssociatedCalendarDataJobData,
DeleteConnectedAccountAssociatedCalendarDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-account-associated-calendar-data.job';
import {
DeleteConnectedAccountAssociatedMessagingDataJobData,
DeleteConnectedAccountAssociatedMessagingDataJob,
} from 'src/workspace/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { ConnectedAccountObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/connected-account.object-metadata';
@Injectable()
@ -15,14 +19,24 @@ export class MessagingConnectedAccountListener {
constructor(
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@Inject(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
) {}
@OnEvent('connectedAccount.deleted')
handleDeletedEvent(
payload: ObjectRecordDeleteEvent<ConnectedAccountObjectMetadata>,
) {
this.messageQueueService.add<DeleteConnectedAccountAssociatedDataJobData>(
DeleteConnectedAccountAssociatedDataJob.name,
this.messageQueueService.add<DeleteConnectedAccountAssociatedMessagingDataJobData>(
DeleteConnectedAccountAssociatedMessagingDataJob.name,
{
workspaceId: payload.workspaceId,
connectedAccountId: payload.deletedRecord.id,
},
);
this.calendarQueueService.add<DeleteConnectedAccountAssociatedCalendarDataJobData>(
DeleteConnectedAccountAssociatedCalendarDataJob.name,
{
workspaceId: payload.workspaceId,
connectedAccountId: payload.deletedRecord.id,

27
packages/twenty-server/src/workspace/messaging/messaging.module.ts
View File

@ -1,7 +1,8 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { HttpModule } from '@nestjs/axios';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module';
import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module';
import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module';
@ -9,25 +10,26 @@ import { EnvironmentModule } from 'src/integrations/environment/environment.modu
import { MessagingPersonListener } from 'src/workspace/messaging/listeners/messaging-person.listener';
import { MessageModule } from 'src/workspace/messaging/repositories/message/message.module';
import { GmailClientProvider } from 'src/workspace/messaging/services/providers/gmail/gmail-client.provider';
import { CreateContactService } from 'src/workspace/messaging/services/create-contact/create-contact.service';
import { CreateCompanyService } from 'src/workspace/messaging/services/create-company/create-company.service';
import { CreateContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-contact/create-contact.service';
import { CreateCompanyService } from 'src/workspace/auto-companies-and-contacts-creation/create-company/create-company.service';
import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
import { GmailPartialSyncService } from 'src/workspace/messaging/services/gmail-partial-sync.service';
import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services/gmail-refresh-access-token.service';
import { GoogleAPIsRefreshAccessTokenService } from 'src/workspace/calendar-and-messaging/services/google-apis-refresh-access-token.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { MessageParticipantModule } from 'src/workspace/messaging/repositories/message-participant/message-participant.module';
import { MessagingWorkspaceMemberListener } from 'src/workspace/messaging/listeners/messaging-workspace-member.listener';
import { MessagingMessageChannelListener } from 'src/workspace/messaging/listeners/messaging-message-channel.listener';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { CompanyModule } from 'src/workspace/messaging/repositories/company/company.module';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import { MessagingConnectedAccountListener } from 'src/workspace/messaging/listeners/messaging-connected-account.listener';
import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/blocklist.module';
import { BlocklistModule } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.module';
import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service';
@Module({
imports: [
EnvironmentModule,
@ -44,12 +46,15 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/
CompanyModule,
PersonModule,
BlocklistModule,
HttpModule.register({
baseURL: 'https://www.googleapis.com/batch/gmail/v1',
}),
],
providers: [
GmailFullSyncService,
GmailPartialSyncService,
FetchMessagesByBatchesService,
GmailRefreshAccessTokenService,
GoogleAPIsRefreshAccessTokenService,
GmailClientProvider,
CreateContactService,
CreateCompanyService,
@ -59,11 +64,13 @@ import { BlocklistModule } from 'src/workspace/messaging/repositories/blocklist/
MessageService,
SaveMessagesAndCreateContactsService,
MessagingConnectedAccountListener,
FetchByBatchesService,
],
exports: [
GmailPartialSyncService,
GmailFullSyncService,
GmailRefreshAccessTokenService,
GoogleAPIsRefreshAccessTokenService,
FetchByBatchesService,
],
})
export class MessagingModule {}

4
packages/twenty-server/src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook.ts
View File

@ -12,8 +12,8 @@ import { FindManyResolverArgs } from 'src/workspace/workspace-resolver-builder/i
import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
@Injectable()
export class MessageFindManyPreQueryHook implements WorkspacePreQueryHook {

4
packages/twenty-server/src/workspace/messaging/query-hooks/messaging-query-hook.module.ts
View File

@ -2,10 +2,10 @@ import { Module } from '@nestjs/common';
import { MessageFindManyPreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-many.pre-query.hook';
import { MessageFindOnePreQueryHook } from 'src/workspace/messaging/query-hooks/message/message-find-one.pre-query-hook';
import { ConnectedAccountModule } from 'src/workspace/messaging/repositories/connected-account/connected-account.module';
import { ConnectedAccountModule } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.module';
import { MessageChannelMessageAssociationModule } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-assocation.module';
import { MessageChannelModule } from 'src/workspace/messaging/repositories/message-channel/message-channel.module';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { WorkspaceMemberModule } from 'src/workspace/repositories/workspace-member/workspace-member.module';
@Module({
imports: [

2
packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.module.ts
View File

@ -2,7 +2,7 @@ import { Module } from '@nestjs/common';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { PersonModule } from 'src/workspace/repositories/person/person.module';
@Module({
imports: [WorkspaceDataSourceModule, PersonModule],

2
packages/twenty-server/src/workspace/messaging/repositories/message-participant/message-participant.service.ts
View File

@ -9,7 +9,7 @@ import {
ParticipantWithId,
ParticipantWithMessageId,
} from 'src/workspace/messaging/types/gmail-message';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
@Injectable()
export class MessageParticipantService {

2
packages/twenty-server/src/workspace/messaging/repositories/message/message.module.ts
View File

@ -6,7 +6,7 @@ import { MessageParticipantModule } from 'src/workspace/messaging/repositories/m
import { MessageThreadModule } from 'src/workspace/messaging/repositories/message-thread/message-thread.module';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module';
import { CreateCompaniesAndContactsModule } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
@Module({
imports: [

21
packages/twenty-server/src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.module.ts
View File

@ -1,21 +0,0 @@
import { Module } from '@nestjs/common';
import { PersonModule } from 'src/workspace/messaging/repositories/person/person.module';
import { WorkspaceMemberModule } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.module';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyModule } from 'src/workspace/messaging/services/create-company/create-company.module';
import { CreateContactModule } from 'src/workspace/messaging/services/create-contact/create-contact.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
WorkspaceDataSourceModule,
CreateContactModule,
CreateCompanyModule,
WorkspaceMemberModule,
PersonModule,
],
providers: [CreateCompaniesAndContactsService],
exports: [CreateCompaniesAndContactsService],
})
export class CreateCompaniesAndContactsModule {}

128
packages/twenty-server/src/workspace/messaging/services/fetch-by-batch.service.ts Normal file
View File

@ -0,0 +1,128 @@
import { HttpService } from '@nestjs/axios';
import { Injectable } from '@nestjs/common';
import { AxiosResponse } from 'axios';
import { BatchQueries } from 'src/workspace/calendar-and-messaging/types/batch-queries';
import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response';
@Injectable()
export class FetchByBatchesService {
constructor(private readonly httpService: HttpService) {}
async fetchAllByBatches(
queries: BatchQueries,
accessToken: string,
boundary: string,
): Promise<AxiosResponse<any, any>[]> {
const batchLimit = 50;
let batchOffset = 0;
let batchResponses: AxiosResponse<any, any>[] = [];
while (batchOffset < queries.length) {
const batchResponse = await this.fetchBatch(
queries,
accessToken,
batchOffset,
batchLimit,
boundary,
);
batchResponses = batchResponses.concat(batchResponse);
batchOffset += batchLimit;
}
return batchResponses;
}
async fetchBatch(
queries: BatchQueries,
accessToken: string,
batchOffset: number,
batchLimit: number,
boundary: string,
): Promise<AxiosResponse<any, any>> {
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
const response = await this.httpService.axiosRef.post(
'/',
this.createBatchBody(limitedQueries, boundary),
{
headers: {
'Content-Type': 'multipart/mixed; boundary=' + boundary,
Authorization: 'Bearer ' + accessToken,
},
},
);
return response;
}
createBatchBody(queries: BatchQueries, boundary: string): string {
let batchBody: string[] = [];
queries.forEach(function (call) {
const method = 'GET';
const uri = call.uri;
batchBody = batchBody.concat([
'--',
boundary,
'\r\n',
'Content-Type: application/http',
'\r\n\r\n',
method,
' ',
uri,
'\r\n\r\n',
]);
});
return batchBody.concat(['--', boundary, '--']).join('');
}
parseBatch(
responseCollection: AxiosResponse<any, any>,
): GmailMessageParsedResponse[] {
const responseItems: GmailMessageParsedResponse[] = [];
const boundary = this.getBatchSeparator(responseCollection);
const responseLines: string[] = responseCollection.data.split(
'--' + boundary,
);
responseLines.forEach(function (response) {
const startJson = response.indexOf('{');
const endJson = response.lastIndexOf('}');
if (startJson < 0 || endJson < 0) return;
const responseJson = response.substring(startJson, endJson + 1);
const item = JSON.parse(responseJson);
responseItems.push(item);
});
return responseItems;
}
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
const headers = responseCollection.headers;
const contentType: string = headers['content-type'];
if (!contentType) return '';
const components = contentType.split('; ');
const boundary = components.find((item) => item.startsWith('boundary='));
return boundary?.replace('boundary=', '').trim() || '';
}
}

130
packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts
View File

@ -1,6 +1,6 @@
import { Injectable, Logger } from '@nestjs/common';
import axios, { AxiosInstance, AxiosResponse } from 'axios';
import { AxiosResponse } from 'axios';
import { simpleParser, AddressObject } from 'mailparser';
import planer from 'planer';
@ -10,17 +10,13 @@ import {
} from 'src/workspace/messaging/types/gmail-message';
import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query';
import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response';
import { FetchByBatchesService } from 'src/workspace/messaging/services/fetch-by-batch.service';
@Injectable()
export class FetchMessagesByBatchesService {
private readonly httpService: AxiosInstance;
private readonly logger = new Logger(FetchMessagesByBatchesService.name);
constructor() {
this.httpService = axios.create({
baseURL: 'https://www.googleapis.com/batch/gmail/v1',
});
}
constructor(private readonly fetchByBatchesService: FetchByBatchesService) {}
async fetchAllMessages(
queries: MessageQuery[],
@ -30,7 +26,7 @@ export class FetchMessagesByBatchesService {
connectedAccountId?: string,
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
let startTime = Date.now();
const batchResponses = await this.fetchAllByBatches(
const batchResponses = await this.fetchByBatchesService.fetchAllByBatches(
queries,
accessToken,
'batch_gmail_messages',
@ -59,126 +55,10 @@ export class FetchMessagesByBatchesService {
return formattedResponse;
}
async fetchAllByBatches(
queries: MessageQuery[],
accessToken: string,
boundary: string,
): Promise<AxiosResponse<any, any>[]> {
const batchLimit = 50;
let batchOffset = 0;
let batchResponses: AxiosResponse<any, any>[] = [];
while (batchOffset < queries.length) {
const batchResponse = await this.fetchBatch(
queries,
accessToken,
batchOffset,
batchLimit,
boundary,
);
batchResponses = batchResponses.concat(batchResponse);
batchOffset += batchLimit;
}
return batchResponses;
}
async fetchBatch(
queries: MessageQuery[],
accessToken: string,
batchOffset: number,
batchLimit: number,
boundary: string,
): Promise<AxiosResponse<any, any>> {
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
const response = await this.httpService.post(
'/',
this.createBatchBody(limitedQueries, boundary),
{
headers: {
'Content-Type': 'multipart/mixed; boundary=' + boundary,
Authorization: 'Bearer ' + accessToken,
},
},
);
return response;
}
createBatchBody(queries: MessageQuery[], boundary: string): string {
let batchBody: string[] = [];
queries.forEach(function (call) {
const method = 'GET';
const uri = call.uri;
batchBody = batchBody.concat([
'--',
boundary,
'\r\n',
'Content-Type: application/http',
'\r\n\r\n',
method,
' ',
uri,
'\r\n\r\n',
]);
});
return batchBody.concat(['--', boundary, '--']).join('');
}
parseBatch(
responseCollection: AxiosResponse<any, any>,
): GmailMessageParsedResponse[] {
const responseItems: GmailMessageParsedResponse[] = [];
const boundary = this.getBatchSeparator(responseCollection);
const responseLines: string[] = responseCollection.data.split(
'--' + boundary,
);
responseLines.forEach(function (response) {
const startJson = response.indexOf('{');
const endJson = response.lastIndexOf('}');
if (startJson < 0 || endJson < 0) return;
const responseJson = response.substring(startJson, endJson + 1);
const item = JSON.parse(responseJson);
responseItems.push(item);
});
return responseItems;
}
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
const headers = responseCollection.headers;
const contentType: string = headers['content-type'];
if (!contentType) return '';
const components = contentType.split('; ');
const boundary = components.find((item) => item.startsWith('boundary='));
return boundary?.replace('boundary=', '').trim() || '';
}
async formatBatchResponseAsGmailMessage(
responseCollection: AxiosResponse<any, any>,
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
const parsedResponses = this.parseBatch(
const parsedResponses = this.fetchByBatchesService.parseBatch(
responseCollection,
) as GmailMessageParsedResponse[];

6
packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts
View File

@ -11,12 +11,12 @@ import {
GmailFullSyncJobData,
GmailFullSyncJob,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter.util';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import {
FeatureFlagEntity,

4
packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts
View File

@ -12,13 +12,13 @@ import {
GmailFullSyncJob,
GmailFullSyncJobData,
} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
import { ConnectedAccountService } from 'src/workspace/calendar-and-messaging/repositories/connected-account/connected-account.service';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
import { GmailMessage } from 'src/workspace/messaging/types/gmail-message';
import { isPersonEmail } from 'src/workspace/messaging/utils/is-person-email.util';
import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
import { BlocklistService } from 'src/workspace/calendar-and-messaging/repositories/blocklist/blocklist.service';
import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
import {
FeatureFlagEntity,

4
packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts
View File

@ -5,7 +5,7 @@ import { EntityManager } from 'typeorm';
import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
import { CreateCompaniesAndContactsService } from 'src/workspace/messaging/services/create-companies-and-contacts/create-companies-and-contacts.service';
import { CreateCompanyAndContactService } from 'src/workspace/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
import {
GmailMessage,
ParticipantWithMessageId,
@ -23,7 +23,7 @@ export class SaveMessagesAndCreateContactsService {
constructor(
private readonly messageService: MessageService,
private readonly messageChannelService: MessageChannelService,
private readonly createCompaniesAndContactsService: CreateCompaniesAndContactsService,
private readonly createCompaniesAndContactsService: CreateCompanyAndContactService,
private readonly messageParticipantService: MessageParticipantService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}

0
packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.ts → packages/twenty-server/src/workspace/messaging/utils/gmail-search-filter.util.ts
View File

3
packages/twenty-server/src/workspace/messaging/repositories/person/person.module.ts → packages/twenty-server/src/workspace/repositories/person/person.module.ts
View File

@ -1,9 +1,8 @@
import { Module } from '@nestjs/common';
import { PersonService } from 'src/workspace/messaging/repositories/person/person.service';
import { PersonService } from 'src/workspace/repositories/person/person.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
// TODO: Move outside of the messaging module
@Module({
imports: [WorkspaceDataSourceModule],
providers: [PersonService],

0
packages/twenty-server/src/workspace/messaging/repositories/person/person.service.ts → packages/twenty-server/src/workspace/repositories/person/person.service.ts
View File

3
packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.module.ts → packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.module.ts
View File

@ -1,9 +1,8 @@
import { Module } from '@nestjs/common';
import { WorkspaceMemberService } from 'src/workspace/messaging/repositories/workspace-member/workspace-member.service';
import { WorkspaceMemberService } from 'src/workspace/repositories/workspace-member/workspace-member.service';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
// TODO: Move outside of the messaging module
@Module({
imports: [WorkspaceDataSourceModule],
providers: [WorkspaceMemberService],

0
packages/twenty-server/src/workspace/messaging/repositories/workspace-member/workspace-member.service.ts → packages/twenty-server/src/workspace/repositories/workspace-member/workspace-member.service.ts
View File

13
packages/twenty-server/src/workspace/workspace-sync-metadata/standard-objects/calendar-event.object-metadata.ts
View File

@ -12,6 +12,7 @@ import { ObjectMetadata } from 'src/workspace/workspace-sync-metadata/decorators
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
import { CalendarChannelEventAssociationObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-channel-event-association.object-metadata';
import { CalendarEventAttendeeObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/calendar-event-attendee.object-metadata';
import { IsNullable } from 'src/workspace/workspace-sync-metadata/decorators/is-nullable.decorator';
import { standardObjectIds } from 'src/workspace/workspace-sync-metadata/constants/standard-object-ids';
import { calendarEventStandardFieldIds } from 'src/workspace/workspace-sync-metadata/constants/standard-field-ids';
@ -62,7 +63,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Start DateTime',
icon: 'IconCalendarClock',
})
startsAt: string;
@IsNullable()
startsAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.endsAt,
@ -71,7 +73,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'End DateTime',
icon: 'IconCalendarClock',
})
endsAt: string;
@IsNullable()
endsAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.externalCreatedAt,
@ -80,7 +83,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Creation DateTime',
icon: 'IconCalendarPlus',
})
externalCreatedAt: string;
@IsNullable()
externalCreatedAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.externalUpdatedAt,
@ -89,7 +93,8 @@ export class CalendarEventObjectMetadata extends BaseObjectMetadata {
description: 'Update DateTime',
icon: 'IconCalendarCog',
})
externalUpdatedAt: string;
@IsNullable()
externalUpdatedAt: string | null;
@FieldMetadata({
standardId: calendarEventStandardFieldIds.description,

2
packages/twenty-server/src/workspace/workspace.module.ts
View File

@ -6,6 +6,7 @@ import { WorkspaceSchemaStorageModule } from 'src/workspace/workspace-schema-sto
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { ScalarsExplorerService } from 'src/workspace/services/scalars-explorer.service';
import { MessagingModule } from 'src/workspace/messaging/messaging.module';
import { CalendarModule } from 'src/workspace/calendar/calendar.module';
import { WorkspaceFactory } from './workspace.factory';
@ -21,6 +22,7 @@ import { WorkspaceResolverBuilderModule } from './workspace-resolver-builder/wor
WorkspaceResolverBuilderModule,
WorkspaceSchemaStorageModule,
MessagingModule,
CalendarModule,
],
providers: [WorkspaceFactory, ScalarsExplorerService],
exports: [WorkspaceFactory],