From 3c4168759a85f80c53664e8ff7caef1c81eacafe Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Thu, 12 Sep 2024 15:57:30 +0200 Subject: [PATCH] Refactor metadata caching (#7011) This PR introduces the following changes: - add the metadataVersion to all our metadata cache keys to ease troubleshooting: image - introduce a cache recompute lock to avoid overloading the database to recompute the cache many time --- packages/twenty-server/src/app.module.ts | 4 +- .../data-seed-demo-workspace.module.ts | 12 +- .../data-seed-demo-workspace.service.ts | 20 +++- .../0-24-set-message-direction.command.ts | 2 +- .../__tests__/workspace.factory.spec.ts | 4 +- .../api/graphql/core-graphql-api.module.ts | 11 +- .../errors/graphql-query-runner.exception.ts | 1 + .../api/graphql/workspace-schema.factory.ts | 56 +++++---- .../workspace/workspace.module.ts | 4 +- .../workspace/workspace.resolver.ts | 2 - .../field-metadata/field-metadata.service.ts | 2 +- .../object-metadata.service.ts | 2 +- .../relation-metadata.service.ts | 2 +- .../foreign-table/foreign-table.service.ts | 2 +- .../remote-table/remote-table.service.ts | 2 +- .../workspace-metadata-cache.exception.ts | 12 ++ .../workspace-metadata-cache.service.ts | 112 ++++++++++++++++++ .../workspace-metadata-cache.module.ts | 18 +++ .../workspace-metadata-version.exception.ts | 12 ++ .../workspace-metadata-version.service.ts | 52 ++++++++ .../workspace-metadata-version.module.ts | 4 +- .../workspace-metadata-version.service.ts | 81 ------------- ...l-hydrate-request-from-token.middleware.ts | 6 +- .../exceptions/twenty-orm.exception.ts | 15 +++ .../entity-schema-relation.factory.ts | 2 + .../factories/entity-schema.factory.ts | 2 + .../scoped-workspace-context.factory.ts | 4 +- .../factories/workspace-datasource.factory.ts | 80 ++++++------- .../twenty-orm/twenty-orm-global.manager.ts | 6 +- .../engine/twenty-orm/twenty-orm.module.ts | 4 +- .../workspace-cache-storage.service.ts | 85 ++++++++++--- .../workspace-sync-metadata.service.ts | 2 +- 32 files changed, 420 insertions(+), 203 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts create mode 100644 packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.ts index 35af3a3e26..8260087acc 100644 --- a/packages/twenty-server/src/app.module.ts +++ b/packages/twenty-server/src/app.module.ts @@ -20,9 +20,9 @@ import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphq import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; import { MessageQueueDriverType } from 'src/engine/core-modules/message-queue/interfaces'; import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module'; -import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { ModulesModule } from 'src/modules/modules.module'; import { CoreEngineModule } from './engine/core-modules/core-engine.module'; @@ -51,7 +51,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module'; // Modules module, contains all business logic modules ModulesModule, // Needed for the user workspace middleware - WorkspaceMetadataVersionModule, + WorkspaceCacheStorageModule, // Api modules CoreGraphQLApiModule, MetadataGraphQLApiModule, diff --git a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module.ts b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module.ts index 63a808235e..d3ce4a65fa 100644 --- a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module.ts +++ b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module.ts @@ -1,11 +1,17 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; -import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; -import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { DataSeedDemoWorkspaceService } from 'src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service'; +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; @Module({ - imports: [WorkspaceManagerModule, EnvironmentModule], + imports: [ + WorkspaceManagerModule, + EnvironmentModule, + TypeOrmModule.forFeature([Workspace], 'core'), + ], providers: [DataSeedDemoWorkspaceService], exports: [DataSeedDemoWorkspaceService], }) diff --git a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts index 316caecac6..7f55575a0b 100644 --- a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts +++ b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts @@ -1,18 +1,24 @@ import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; -import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; -import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; import { deleteCoreSchema, seedCoreSchema, } from 'src/database/typeorm-seeds/core/demo'; import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; +import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; @Injectable() export class DataSeedDemoWorkspaceService { constructor( private readonly environmentService: EnvironmentService, private readonly workspaceManagerService: WorkspaceManagerService, + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, ) {} async seedDemo(): Promise { @@ -27,8 +33,14 @@ export class DataSeedDemoWorkspaceService { ); } for (const workspaceId of demoWorkspaceIds) { - await deleteCoreSchema(rawDataSource, workspaceId); - await this.workspaceManagerService.delete(workspaceId); + const existingWorkspaces = await this.workspaceRepository.findBy({ + id: workspaceId, + }); + + if (existingWorkspaces.length > 0) { + await this.workspaceManagerService.delete(workspaceId); + await deleteCoreSchema(rawDataSource, workspaceId); + } await seedCoreSchema(rawDataSource, workspaceId); await this.workspaceManagerService.initDemo(workspaceId); diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts index b8689ecfaa..74170cdc8c 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts @@ -9,7 +9,7 @@ import { Workspace, WorkspaceActivationStatus, } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum'; import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity'; diff --git a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts index 57d051363e..05d0cbaa9e 100644 --- a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts +++ b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts @@ -6,7 +6,7 @@ import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace- import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; describe('WorkspaceSchemaFactory', () => { @@ -41,7 +41,7 @@ describe('WorkspaceSchemaFactory', () => { useValue: {}, }, { - provide: WorkspaceMetadataVersionService, + provide: WorkspaceMetadataCacheService, useValue: {}, }, ], diff --git a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts index c9b0aa8523..1b0603186f 100644 --- a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts @@ -4,25 +4,18 @@ import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars- import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module'; import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; -import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceSchemaFactory } from './workspace-schema.factory'; @Module({ imports: [ - // TODO: Seems like it's breaking /metadata query and mutation arguments - // we should investigate this issue - // GraphQLModule.forRootAsync({ - // driver: YogaDriver, - // imports: [CoreEngineModule, GraphQLConfigModule], - // useClass: GraphQLConfigService, - // }), MetadataEngineModule, WorkspaceSchemaBuilderModule, WorkspaceResolverBuilderModule, WorkspaceCacheStorageModule, - WorkspaceMetadataVersionModule, + WorkspaceMetadataCacheModule, ], providers: [WorkspaceSchemaFactory, ScalarsExplorerService], exports: [WorkspaceSchemaFactory], diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts index efbbf1183a..55c70c5af2 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts @@ -19,4 +19,5 @@ export enum GraphqlQueryRunnerExceptionCode { RECORD_NOT_FOUND = 'RECORD_NOT_FOUND', INVALID_ARGS_FIRST = 'INVALID_ARGS_FIRST', INVALID_ARGS_LAST = 'INVALID_ARGS_LAST', + METADATA_CACHE_VERSION_NOT_FOUND = 'METADATA_CACHE_VERSION_NOT_FOUND', } diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index 71d58e1419..c5a550b066 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; import { GraphQLSchema, printSchema } from 'graphql'; import { gql } from 'graphql-tag'; +import { + GraphqlQueryRunnerException, + GraphqlQueryRunnerExceptionCode, +} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories'; import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Injectable() export class WorkspaceSchemaFactory { constructor( private readonly dataSourceService: DataSourceService, - private readonly objectMetadataService: ObjectMetadataService, private readonly scalarsExplorerService: ScalarsExplorerService, private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory, private readonly workspaceResolverFactory: WorkspaceResolverFactory, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, + private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, ) {} async createGraphQLSchema(authContext: AuthContext): Promise { @@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory { authContext.workspace.id, ); - // Can'f find any data sources for this workspace if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) { return new GraphQLSchema({}); } - // Validate cache version - await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated( - authContext.workspace.id, - ); - - // Get object metadata from cache - let objectMetadataCollection = - await this.workspaceCacheStorageService.getObjectMetadataCollection( + const currentCacheVersion = + await this.workspaceCacheStorageService.getMetadataVersion( authContext.workspace.id, ); - // If object metadata is not cached, get it from the database - if (!objectMetadataCollection) { - objectMetadataCollection = - await this.objectMetadataService.findManyWithinWorkspace( - authContext.workspace.id, - ); - - await this.workspaceCacheStorageService.setObjectMetadataCollection( + if (currentCacheVersion === undefined) { + await this.workspaceMetadataCacheService.recomputeMetadataCache( authContext.workspace.id, - objectMetadataCollection, + ); + throw new GraphqlQueryRunnerException( + 'Metadata cache version not found', + GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, + ); + } + + const objectMetadataCollection = + await this.workspaceCacheStorageService.getObjectMetadataCollection( + authContext.workspace.id, + currentCacheVersion, + ); + + if (!objectMetadataCollection) { + await this.workspaceMetadataCacheService.recomputeMetadataCache( + authContext.workspace.id, + ); + throw new GraphqlQueryRunnerException( + 'Object metadata collection not found', + GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, ); } // Get typeDefs from cache let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs( authContext.workspace.id, + currentCacheVersion, ); let usedScalarNames = await this.workspaceCacheStorageService.getGraphQLUsedScalarNames( authContext.workspace.id, + currentCacheVersion, ); // If typeDefs are not cached, generate them @@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory { await this.workspaceCacheStorageService.setGraphQLTypeDefs( authContext.workspace.id, + currentCacheVersion, typeDefs, ); await this.workspaceCacheStorageService.setGraphQLUsedScalarNames( authContext.workspace.id, + currentCacheVersion, usedScalarNames, ); } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index 5c8e832661..4c227709c9 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener'; import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; -import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts'; @@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service'; BillingModule, FileModule, FileUploadModule, - WorkspaceMetadataVersionModule, + WorkspaceMetadataCacheModule, NestjsQueryTypeOrmModule.forFeature( [User, Workspace, UserWorkspace, FeatureFlagEntity], 'core', diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts index bc995017ba..4b9012a389 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts @@ -26,7 +26,6 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { assert } from 'src/utils/assert'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; @@ -39,7 +38,6 @@ import { WorkspaceService } from './services/workspace.service'; export class WorkspaceResolver { constructor( private readonly workspaceService: WorkspaceService, - private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly userWorkspaceService: UserWorkspaceService, private readonly fileUploadService: FileUploadService, private readonly fileService: FileService, diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index 6dc4122aad..1234062b5b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -41,7 +41,7 @@ import { NameTooLongException } from 'src/engine/metadata-modules/utils/exceptio import { exceedsDatabaseIdentifierMaximumLength } from 'src/engine/metadata-modules/utils/validate-database-identifier-length.utils'; import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils'; import { validateMetadataNameValidityOrThrow as validateFieldNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index b41f760a48..136f2c5026 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -34,7 +34,7 @@ import { import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete'; import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service'; import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, diff --git a/packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts index a08006adbf..627a9a4ea2 100644 --- a/packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts @@ -21,7 +21,7 @@ import { import { InvalidStringException } from 'src/engine/metadata-modules/utils/exceptions/invalid-string.exception'; import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils'; import { validateMetadataNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts index 4fed42713f..55e545ed83 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts @@ -11,7 +11,7 @@ import { } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception'; import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { ReferencedTable, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 76127e49f8..d7d3b985af 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -36,7 +36,7 @@ import { mapUdtNameToFieldType, } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspaceMigrationColumnAction, WorkspaceMigrationColumnActionType, diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts new file mode 100644 index 0000000000..04dc3e0137 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts @@ -0,0 +1,12 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class WorkspaceMetadataCacheException extends CustomException { + code: WorkspaceMetadataCacheExceptionCode; + constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) { + super(message, code); + } +} + +export enum WorkspaceMetadataCacheExceptionCode { + METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts new file mode 100644 index 0000000000..26bed85b0e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts @@ -0,0 +1,112 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { + WorkspaceMetadataCacheException, + WorkspaceMetadataCacheExceptionCode, +} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; + +@Injectable() +export class WorkspaceMetadataCacheService { + logger = new Logger(WorkspaceMetadataCacheService.name); + + constructor( + @InjectRepository(Workspace, 'core') + private readonly workspaceRepository: Repository, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + ) {} + + async recomputeMetadataCache( + workspaceId: string, + force = false, + ): Promise { + const currentCacheVersion = + await this.getMetadataVersionFromCache(workspaceId); + + const currentDatabaseVersion = + await this.getMetadataVersionFromDatabase(workspaceId); + + if (currentDatabaseVersion === undefined) { + throw new WorkspaceMetadataCacheException( + 'Metadata version not found in the database', + WorkspaceMetadataCacheExceptionCode.METADATA_VERSION_NOT_FOUND, + ); + } + + if (!force && currentCacheVersion === currentDatabaseVersion) { + return; + } + + const isAlreadyCaching = + await this.workspaceCacheStorageService.getObjectMetadataCollectionOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); + + if (isAlreadyCaching) { + return; + } + + if (currentCacheVersion !== undefined) { + this.workspaceCacheStorageService.flush(workspaceId, currentCacheVersion); + } + + await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); + + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + currentDatabaseVersion, + ); + + const freshObjectMetadataCollection = + await this.objectMetadataRepository.find({ + where: { workspaceId }, + relations: [ + 'fields.object', + 'fields', + 'fields.fromRelationMetadata', + 'fields.toRelationMetadata', + 'fields.fromRelationMetadata.toObjectMetadata', + ], + }); + + await this.workspaceCacheStorageService.setObjectMetadataCollection( + workspaceId, + currentDatabaseVersion, + freshObjectMetadataCollection, + ); + + await this.workspaceCacheStorageService.removeObjectMetadataCollectionOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); + } + + private async getMetadataVersionFromDatabase( + workspaceId: string, + ): Promise { + const workspace = await this.workspaceRepository.findOne({ + where: { id: workspaceId }, + }); + + return workspace?.metadataVersion; + } + + private async getMetadataVersionFromCache( + workspaceId: string, + ): Promise { + return await this.workspaceCacheStorageService.getMetadataVersion( + workspaceId, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts new file mode 100644 index 0000000000..a0dd5058bc --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), + WorkspaceCacheStorageModule, + ], + exports: [WorkspaceMetadataCacheService], + providers: [WorkspaceMetadataCacheService], +}) +export class WorkspaceMetadataCacheModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts new file mode 100644 index 0000000000..76a3df1673 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception.ts @@ -0,0 +1,12 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class WorkspaceMetadataVersionException extends CustomException { + code: WorkspaceMetadataVersionExceptionCode; + constructor(message: string, code: WorkspaceMetadataVersionExceptionCode) { + super(message, code); + } +} + +export enum WorkspaceMetadataVersionExceptionCode { + METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts new file mode 100644 index 0000000000..2d1b0e383d --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts @@ -0,0 +1,52 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { + WorkspaceMetadataVersionException, + WorkspaceMetadataVersionExceptionCode, +} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; + +@Injectable() +export class WorkspaceMetadataVersionService { + logger = new Logger(WorkspaceMetadataCacheService.name); + + constructor( + @InjectRepository(Workspace, 'core') + private readonly workspaceRepository: Repository, + private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) {} + + async incrementMetadataVersion(workspaceId: string): Promise { + const workspace = await this.workspaceRepository.findOne({ + where: { id: workspaceId }, + }); + + const metadataVersion = workspace?.metadataVersion; + + if (metadataVersion === undefined) { + throw new WorkspaceMetadataVersionException( + 'Metadata version not found', + WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND, + ); + } + + const newMetadataVersion = metadataVersion + 1; + + await this.workspaceRepository.update( + { id: workspaceId }, + { metadataVersion: newMetadataVersion }, + ); + + await this.workspaceMetadataCacheService.recomputeMetadataCache( + workspaceId, + ); + + await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts index 4064872a17..350d22b56b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts @@ -2,13 +2,15 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @Module({ imports: [ TypeOrmModule.forFeature([Workspace], 'core'), WorkspaceCacheStorageModule, + WorkspaceMetadataCacheModule, ], exports: [WorkspaceMetadataVersionService], providers: [WorkspaceMetadataVersionService], diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts deleted file mode 100644 index d3986e8db2..0000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import { Repository } from 'typeorm'; - -import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; - -@Injectable() -export class WorkspaceMetadataVersionService { - logger = new Logger(WorkspaceMetadataVersionService.name); - - constructor( - @InjectRepository(Workspace, 'core') - private readonly workspaceRepository: Repository, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - ) {} - - async flushCacheIfMetadataVersionIsOutdated( - workspaceId: string, - ): Promise { - const currentVersion = - (await this.workspaceCacheStorageService.getMetadataVersion( - workspaceId, - )) ?? 1; - - let latestVersion = await this.getMetadataVersion(workspaceId); - - if (latestVersion === undefined || currentVersion !== latestVersion) { - this.logger.log( - `Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, - ); - - await this.workspaceCacheStorageService.flush(workspaceId); - - latestVersion = await this.incrementMetadataVersion(workspaceId); - - await this.workspaceCacheStorageService.setMetadataVersion( - workspaceId, - latestVersion, - ); - } - } - - async incrementMetadataVersion(workspaceId: string): Promise { - const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0; - const newMetadataVersion = metadataVersion + 1; - - await this.workspaceRepository.update( - { id: workspaceId }, - { metadataVersion: newMetadataVersion }, - ); - - await this.workspaceCacheStorageService.flush(workspaceId); - - await this.workspaceCacheStorageService.setMetadataVersion( - workspaceId, - newMetadataVersion, - ); - - return newMetadataVersion; - } - - async getMetadataVersion(workspaceId: string): Promise { - const workspace = await this.workspaceRepository.findOne({ - where: { id: workspaceId }, - }); - - return workspace?.metadataVersion; - } - - async resetMetadataVersion(workspaceId: string): Promise { - await this.workspaceRepository.update( - { id: workspaceId }, - { metadataVersion: 1 }, - ); - - await this.workspaceCacheStorageService.flush(workspaceId); - await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1); - } -} diff --git a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts index 8da8119927..f8ea8cea99 100644 --- a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts +++ b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts @@ -6,8 +6,8 @@ import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filt import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { handleExceptionAndConvertToGraphQLError } from 'src/engine/utils/global-exception-handler.util'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; class GraphqlTokenValidationProxy { private tokenService: TokenService; @@ -33,7 +33,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware { constructor( private readonly tokenService: TokenService, - private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, + private readonly workspaceStorageCacheService: WorkspaceCacheStorageService, private readonly exceptionHandlerService: ExceptionHandlerService, ) {} @@ -73,7 +73,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware data = await graphqlTokenValidationProxy.validateToken(req); const metadataVersion = - await this.workspaceMetadataVersionService.getMetadataVersion( + await this.workspaceStorageCacheService.getMetadataVersion( data.workspace.id, ); diff --git a/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts b/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts new file mode 100644 index 0000000000..25a717af09 --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts @@ -0,0 +1,15 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class TwentyORMException extends CustomException { + code: TwentyORMExceptionCode; + constructor(message: string, code: TwentyORMExceptionCode) { + super(message, code); + } +} + +export enum TwentyORMExceptionCode { + METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND', + METADATA_VERSION_MISMATCH = 'METADATA_VERSION_MISMATCH', + METADATA_COLLECTION_NOT_FOUND = 'METADATA_COLLECTION_NOT_FOUND', + WORKSPACE_SCHEMA_NOT_FOUND = 'WORKSPACE_SCHEMA_NOT_FOUND', +} diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts index fc8755ecc5..d36c6b97d7 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts @@ -19,6 +19,7 @@ export class EntitySchemaRelationFactory { async create( workspaceId: string, + metadataVersion: number, fieldMetadataCollection: FieldMetadataEntity[], ): Promise { const entitySchemaRelationMap: EntitySchemaRelationMap = {}; @@ -40,6 +41,7 @@ export class EntitySchemaRelationFactory { const objectMetadataCollection = await this.workspaceCacheStorageService.getObjectMetadataCollection( workspaceId, + metadataVersion, ); if (!objectMetadataCollection) { diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts index 249f140911..494c892cf5 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts @@ -17,6 +17,7 @@ export class EntitySchemaFactory { async create( workspaceId: string, + metadataVersion: number, objectMetadata: ObjectMetadataEntity, ): Promise { const columns = this.entitySchemaColumnFactory.create( @@ -26,6 +27,7 @@ export class EntitySchemaFactory { const relations = await this.entitySchemaRelationFactory.create( workspaceId, + metadataVersion, objectMetadata.fields, ); diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts index bf70156a94..44b6cd1ed0 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts @@ -11,11 +11,11 @@ export class ScopedWorkspaceContextFactory { public create(): { workspaceId: string | null; - workspaceMetadataVersion: string | null; + workspaceMetadataVersion: number | null; } { const workspaceId: string | undefined = this.request?.['req']?.['workspaceId']; - const workspaceMetadataVersion: string | undefined = + const workspaceMetadataVersion: number | undefined = this.request?.['req']?.['workspaceMetadataVersion']; return { diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 32065b91e9..051406a456 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -1,13 +1,15 @@ import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { EntitySchema, Repository } from 'typeorm'; +import { EntitySchema } from 'typeorm'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; +import { + TwentyORMException, + TwentyORMExceptionCode, +} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @@ -20,63 +22,56 @@ export class WorkspaceDatasourceFactory { private readonly dataSourceService: DataSourceService, private readonly environmentService: EnvironmentService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, - @InjectRepository(ObjectMetadataEntity, 'metadata') - private readonly objectMetadataRepository: Repository, + private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, private readonly entitySchemaFactory: EntitySchemaFactory, ) {} public async create( workspaceId: string, - workspaceMetadataVersion: string | null, + workspaceMetadataVersion: number | null, ): Promise { const latestWorkspaceMetadataVersion = - await this.workspaceMetadataVersionService.getMetadataVersion( + await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); + + if (latestWorkspaceMetadataVersion === undefined) { + await this.workspaceMetadataCacheService.recomputeMetadataCache( workspaceId, ); + throw new TwentyORMException( + `Metadata version not found for workspace ${workspaceId}`, + TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND, + ); + } const desiredWorkspaceMetadataVersion = workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; - if (!desiredWorkspaceMetadataVersion) { - throw new Error( - `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`, - ); - } - if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { - throw new Error( + throw new TwentyORMException( `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, + TwentyORMExceptionCode.METADATA_VERSION_MISMATCH, ); } const workspaceDataSource = await this.cacheManager.execute( - `${workspaceId}-${latestWorkspaceMetadataVersion}`, + `${workspaceId}-${desiredWorkspaceMetadataVersion}`, async () => { - let cachedObjectMetadataCollection = + const cachedObjectMetadataCollection = await this.workspaceCacheStorageService.getObjectMetadataCollection( workspaceId, + desiredWorkspaceMetadataVersion, ); if (!cachedObjectMetadataCollection) { - const freshObjectMetadataCollection = - await this.objectMetadataRepository.find({ - where: { workspaceId }, - relations: [ - 'fields.object', - 'fields', - 'fields.fromRelationMetadata', - 'fields.toRelationMetadata', - 'fields.fromRelationMetadata.toObjectMetadata', - ], - }); - - await this.workspaceCacheStorageService.setObjectMetadataCollection( + await this.workspaceMetadataCacheService.recomputeMetadataCache( workspaceId, - freshObjectMetadataCollection, + true, ); - cachedObjectMetadataCollection = freshObjectMetadataCollection; + throw new TwentyORMException( + `Object metadata collection not found for workspace ${workspaceId}`, + TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND, + ); } const dataSourceMetadata = @@ -85,20 +80,16 @@ export class WorkspaceDatasourceFactory { ); if (!dataSourceMetadata) { - throw new Error( - `Data source metadata not found for workspace ${workspaceId}`, - ); - } - - if (!cachedObjectMetadataCollection) { - throw new Error( - `Object metadata collection not found for workspace ${workspaceId}`, + throw new TwentyORMException( + `Workspace Schema not found for workspace ${workspaceId}`, + TwentyORMExceptionCode.WORKSPACE_SCHEMA_NOT_FOUND, ); } const cachedEntitySchemaOptions = await this.workspaceCacheStorageService.getORMEntitySchema( workspaceId, + desiredWorkspaceMetadataVersion, ); let cachedEntitySchemas: EntitySchema[]; @@ -110,12 +101,17 @@ export class WorkspaceDatasourceFactory { } else { const entitySchemas = await Promise.all( cachedObjectMetadataCollection.map((objectMetadata) => - this.entitySchemaFactory.create(workspaceId, objectMetadata), + this.entitySchemaFactory.create( + workspaceId, + desiredWorkspaceMetadataVersion, + objectMetadata, + ), ), ); await this.workspaceCacheStorageService.setORMEntitySchema( workspaceId, + desiredWorkspaceMetadataVersion, entitySchemas.map((entitySchema) => entitySchema.options), ); diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts index f4d8190c55..c073fa885a 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts @@ -47,6 +47,10 @@ export class TwentyORMGlobalManager { } async getDataSourceForWorkspace(workspaceId: string) { - return this.workspaceDataSourceFactory.create(workspaceId, null); + return await this.workspaceDataSourceFactory.create(workspaceId, null); + } + + async loadDataSourceForWorkspace(workspaceId: string) { + await this.workspaceDataSourceFactory.create(workspaceId, null); } } diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts index 923400c44e..7e514fb646 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts @@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { entitySchemaFactories } from 'src/engine/twenty-orm/factories'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; @@ -16,7 +16,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/ TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), DataSourceModule, WorkspaceCacheStorageModule, - WorkspaceMetadataVersionModule, + WorkspaceMetadataCacheModule, ], providers: [ ...entitySchemaFactories, diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts index b20f880e4d..34612e7097 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts @@ -13,6 +13,7 @@ enum WorkspaceCacheKeys { GraphQLOperations = 'graphql:operations', ORMEntitySchemas = 'orm:entity-schemas', MetadataObjectMetadataCollection = 'metadata:object-metadata-collection', + MetadataObjectMetadataCollectionOngoingCachingLock = 'metadata:object-metadata-collection-ongoing-caching-lock', MetadataVersion = 'metadata:workspace-metadata-version', } @@ -25,26 +26,31 @@ export class WorkspaceCacheStorageService { setORMEntitySchema( workspaceId: string, + metadataVersion: number, entitySchemas: EntitySchemaOptions[], ) { return this.cacheStorageService.set[]>( - `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, entitySchemas, ); } getORMEntitySchema( workspaceId: string, + metadataVersion: number, ): Promise[] | undefined> { return this.cacheStorageService.get[]>( - `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, ); } - setMetadataVersion(workspaceId: string, version: number): Promise { + setMetadataVersion( + workspaceId: string, + metadataVersion: number, + ): Promise { return this.cacheStorageService.set( `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, - version, + metadataVersion, ); } @@ -54,70 +60,113 @@ export class WorkspaceCacheStorageService { ); } + addObjectMetadataCollectionOngoingCachingLock( + workspaceId: string, + metadataVersion: number, + ) { + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, + true, + ); + } + + removeObjectMetadataCollectionOngoingCachingLock( + workspaceId: string, + metadataVersion: number, + ) { + return this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, + ); + } + + getObjectMetadataCollectionOngoingCachingLock( + workspaceId: string, + metadataVersion: number, + ): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, + ); + } + setObjectMetadataCollection( workspaceId: string, + metadataVersion: number, objectMetadataCollection: ObjectMetadataEntity[], ) { return this.cacheStorageService.set( - `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, objectMetadataCollection, ); } getObjectMetadataCollection( workspaceId: string, + metadataVersion: number, ): Promise { return this.cacheStorageService.get( - `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, ); } - setGraphQLTypeDefs(workspaceId: string, typeDefs: string): Promise { + setGraphQLTypeDefs( + workspaceId: string, + metadataVersion: number, + typeDefs: string, + ): Promise { return this.cacheStorageService.set( - `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, typeDefs, ); } - getGraphQLTypeDefs(workspaceId: string): Promise { + getGraphQLTypeDefs( + workspaceId: string, + metadataVersion: number, + ): Promise { return this.cacheStorageService.get( - `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, ); } setGraphQLUsedScalarNames( workspaceId: string, + metadataVersion: number, usedScalarNames: string[], ): Promise { return this.cacheStorageService.set( - `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, usedScalarNames, ); } getGraphQLUsedScalarNames( workspaceId: string, + metadataVersion: number, ): Promise { return this.cacheStorageService.get( - `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, ); } - async flush(workspaceId: string): Promise { + async flush(workspaceId: string, metadataVersion: number): Promise { await this.cacheStorageService.del( - `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`, ); await this.cacheStorageService.del( - `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}:${metadataVersion}`, ); await this.cacheStorageService.del( - `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`, ); await this.cacheStorageService.del( - `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`, ); await this.cacheStorageService.del( - `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`, + ); + + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`, ); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index bd85f4c209..6e035b99c0 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -6,7 +6,7 @@ import { DataSource, QueryFailedError } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; -import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';