mirror of
https://github.com/twentyhq/twenty.git
synced 2024-10-03 20:38:02 +03:00
Refactor metadata caching (#7011)
This PR introduces the following changes: - add the metadataVersion to all our metadata cache keys to ease troubleshooting: <img width="1146" alt="image" src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12"> - introduce a cache recompute lock to avoid overloading the database to recompute the cache many time
This commit is contained in:
Charles Bochet
Charles Bochet
parent
9b46e8c663
commit
3c4168759a
@ -20,9 +20,9 @@ import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphq
|
||||
import { RestApiModule } from 'src/engine/api/rest/rest-api.module';
|
||||
import { MessageQueueDriverType } from 'src/engine/core-modules/message-queue/interfaces';
|
||||
import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module';
|
||||
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
|
||||
import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware';
|
||||
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
import { ModulesModule } from 'src/modules/modules.module';
|
||||
|
||||
import { CoreEngineModule } from './engine/core-modules/core-engine.module';
|
||||
@ -51,7 +51,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module';
|
||||
// Modules module, contains all business logic modules
|
||||
ModulesModule,
|
||||
// Needed for the user workspace middleware
|
||||
WorkspaceMetadataVersionModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
// Api modules
|
||||
CoreGraphQLApiModule,
|
||||
MetadataGraphQLApiModule,
|
||||
|
||||
@ -1,11 +1,17 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
import { DataSeedDemoWorkspaceService } from 'src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service';
|
||||
import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
|
||||
@Module({
|
||||
imports: [WorkspaceManagerModule, EnvironmentModule],
|
||||
imports: [
|
||||
WorkspaceManagerModule,
|
||||
EnvironmentModule,
|
||||
TypeOrmModule.forFeature([Workspace], 'core'),
|
||||
],
|
||||
providers: [DataSeedDemoWorkspaceService],
|
||||
exports: [DataSeedDemoWorkspaceService],
|
||||
})
|
||||
|
||||
@ -1,18 +1,24 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
|
||||
import {
|
||||
deleteCoreSchema,
|
||||
seedCoreSchema,
|
||||
} from 'src/database/typeorm-seeds/core/demo';
|
||||
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
|
||||
|
||||
@Injectable()
|
||||
export class DataSeedDemoWorkspaceService {
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly workspaceManagerService: WorkspaceManagerService,
|
||||
@InjectRepository(Workspace, 'core')
|
||||
protected readonly workspaceRepository: Repository<Workspace>,
|
||||
) {}
|
||||
|
||||
async seedDemo(): Promise<void> {
|
||||
@ -27,8 +33,14 @@ export class DataSeedDemoWorkspaceService {
|
||||
);
|
||||
}
|
||||
for (const workspaceId of demoWorkspaceIds) {
|
||||
await deleteCoreSchema(rawDataSource, workspaceId);
|
||||
await this.workspaceManagerService.delete(workspaceId);
|
||||
const existingWorkspaces = await this.workspaceRepository.findBy({
|
||||
id: workspaceId,
|
||||
});
|
||||
|
||||
if (existingWorkspaces.length > 0) {
|
||||
await this.workspaceManagerService.delete(workspaceId);
|
||||
await deleteCoreSchema(rawDataSource, workspaceId);
|
||||
}
|
||||
|
||||
await seedCoreSchema(rawDataSource, workspaceId);
|
||||
await this.workspaceManagerService.initDemo(workspaceId);
|
||||
|
||||
@ -9,7 +9,7 @@ import {
|
||||
Workspace,
|
||||
WorkspaceActivationStatus,
|
||||
} from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
|
||||
@ -6,7 +6,7 @@ import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-
|
||||
import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
|
||||
describe('WorkspaceSchemaFactory', () => {
|
||||
@ -41,7 +41,7 @@ describe('WorkspaceSchemaFactory', () => {
|
||||
useValue: {},
|
||||
},
|
||||
{
|
||||
provide: WorkspaceMetadataVersionService,
|
||||
provide: WorkspaceMetadataCacheService,
|
||||
useValue: {},
|
||||
},
|
||||
],
|
||||
|
||||
@ -4,25 +4,18 @@ import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-
|
||||
import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module';
|
||||
import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
|
||||
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
|
||||
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
|
||||
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
|
||||
import { WorkspaceSchemaFactory } from './workspace-schema.factory';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
// TODO: Seems like it's breaking /metadata query and mutation arguments
|
||||
// we should investigate this issue
|
||||
// GraphQLModule.forRootAsync<YogaDriverConfig>({
|
||||
// driver: YogaDriver,
|
||||
// imports: [CoreEngineModule, GraphQLConfigModule],
|
||||
// useClass: GraphQLConfigService,
|
||||
// }),
|
||||
MetadataEngineModule,
|
||||
WorkspaceSchemaBuilderModule,
|
||||
WorkspaceResolverBuilderModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
WorkspaceMetadataVersionModule,
|
||||
WorkspaceMetadataCacheModule,
|
||||
],
|
||||
providers: [WorkspaceSchemaFactory, ScalarsExplorerService],
|
||||
exports: [WorkspaceSchemaFactory],
|
||||
|
||||
@ -19,4 +19,5 @@ export enum GraphqlQueryRunnerExceptionCode {
|
||||
RECORD_NOT_FOUND = 'RECORD_NOT_FOUND',
|
||||
INVALID_ARGS_FIRST = 'INVALID_ARGS_FIRST',
|
||||
INVALID_ARGS_LAST = 'INVALID_ARGS_LAST',
|
||||
METADATA_CACHE_VERSION_NOT_FOUND = 'METADATA_CACHE_VERSION_NOT_FOUND',
|
||||
}
|
||||
|
||||
@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
|
||||
import { GraphQLSchema, printSchema } from 'graphql';
|
||||
import { gql } from 'graphql-tag';
|
||||
|
||||
import {
|
||||
GraphqlQueryRunnerException,
|
||||
GraphqlQueryRunnerExceptionCode,
|
||||
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
|
||||
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
|
||||
import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
|
||||
import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
|
||||
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
|
||||
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
@Injectable()
|
||||
export class WorkspaceSchemaFactory {
|
||||
constructor(
|
||||
private readonly dataSourceService: DataSourceService,
|
||||
private readonly objectMetadataService: ObjectMetadataService,
|
||||
private readonly scalarsExplorerService: ScalarsExplorerService,
|
||||
private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory,
|
||||
private readonly workspaceResolverFactory: WorkspaceResolverFactory,
|
||||
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
|
||||
) {}
|
||||
|
||||
async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
|
||||
@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory {
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
// Can'f find any data sources for this workspace
|
||||
if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) {
|
||||
return new GraphQLSchema({});
|
||||
}
|
||||
|
||||
// Validate cache version
|
||||
await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated(
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
// Get object metadata from cache
|
||||
let objectMetadataCollection =
|
||||
await this.workspaceCacheStorageService.getObjectMetadataCollection(
|
||||
const currentCacheVersion =
|
||||
await this.workspaceCacheStorageService.getMetadataVersion(
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
// If object metadata is not cached, get it from the database
|
||||
if (!objectMetadataCollection) {
|
||||
objectMetadataCollection =
|
||||
await this.objectMetadataService.findManyWithinWorkspace(
|
||||
authContext.workspace.id,
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.setObjectMetadataCollection(
|
||||
if (currentCacheVersion === undefined) {
|
||||
await this.workspaceMetadataCacheService.recomputeMetadataCache(
|
||||
authContext.workspace.id,
|
||||
objectMetadataCollection,
|
||||
);
|
||||
throw new GraphqlQueryRunnerException(
|
||||
'Metadata cache version not found',
|
||||
GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const objectMetadataCollection =
|
||||
await this.workspaceCacheStorageService.getObjectMetadataCollection(
|
||||
authContext.workspace.id,
|
||||
currentCacheVersion,
|
||||
);
|
||||
|
||||
if (!objectMetadataCollection) {
|
||||
await this.workspaceMetadataCacheService.recomputeMetadataCache(
|
||||
authContext.workspace.id,
|
||||
);
|
||||
throw new GraphqlQueryRunnerException(
|
||||
'Object metadata collection not found',
|
||||
GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
// Get typeDefs from cache
|
||||
let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
|
||||
authContext.workspace.id,
|
||||
currentCacheVersion,
|
||||
);
|
||||
let usedScalarNames =
|
||||
await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
|
||||
authContext.workspace.id,
|
||||
currentCacheVersion,
|
||||
);
|
||||
|
||||
// If typeDefs are not cached, generate them
|
||||
@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory {
|
||||
|
||||
await this.workspaceCacheStorageService.setGraphQLTypeDefs(
|
||||
authContext.workspace.id,
|
||||
currentCacheVersion,
|
||||
typeDefs,
|
||||
);
|
||||
await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
|
||||
authContext.workspace.id,
|
||||
currentCacheVersion,
|
||||
usedScalarNames,
|
||||
);
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity';
|
||||
import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener';
|
||||
import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
|
||||
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
|
||||
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
|
||||
|
||||
import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts';
|
||||
@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service';
|
||||
BillingModule,
|
||||
FileModule,
|
||||
FileUploadModule,
|
||||
WorkspaceMetadataVersionModule,
|
||||
WorkspaceMetadataCacheModule,
|
||||
NestjsQueryTypeOrmModule.forFeature(
|
||||
[User, Workspace, UserWorkspace, FeatureFlagEntity],
|
||||
'core',
|
||||
|
||||
@ -26,7 +26,6 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
|
||||
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
|
||||
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
|
||||
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { assert } from 'src/utils/assert';
|
||||
import { streamToBuffer } from 'src/utils/stream-to-buffer';
|
||||
|
||||
@ -39,7 +38,6 @@ import { WorkspaceService } from './services/workspace.service';
|
||||
export class WorkspaceResolver {
|
||||
constructor(
|
||||
private readonly workspaceService: WorkspaceService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
private readonly userWorkspaceService: UserWorkspaceService,
|
||||
private readonly fileUploadService: FileUploadService,
|
||||
private readonly fileService: FileService,
|
||||
|
||||
@ -41,7 +41,7 @@ import { NameTooLongException } from 'src/engine/metadata-modules/utils/exceptio
|
||||
import { exceedsDatabaseIdentifierMaximumLength } from 'src/engine/metadata-modules/utils/validate-database-identifier-length.utils';
|
||||
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
|
||||
import { validateMetadataNameValidityOrThrow as validateFieldNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
|
||||
import {
|
||||
WorkspaceMigrationColumnActionType,
|
||||
|
||||
@ -34,7 +34,7 @@ import {
|
||||
import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete';
|
||||
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
|
||||
import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
|
||||
import {
|
||||
WorkspaceMigrationColumnActionType,
|
||||
|
||||
@ -21,7 +21,7 @@ import {
|
||||
import { InvalidStringException } from 'src/engine/metadata-modules/utils/exceptions/invalid-string.exception';
|
||||
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
|
||||
import { validateMetadataNameValidityOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name-validity.utils';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
|
||||
import {
|
||||
WorkspaceMigrationColumnActionType,
|
||||
|
||||
@ -11,7 +11,7 @@ import {
|
||||
} from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception';
|
||||
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util';
|
||||
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
|
||||
import {
|
||||
ReferencedTable,
|
||||
|
||||
@ -36,7 +36,7 @@ import {
|
||||
mapUdtNameToFieldType,
|
||||
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
|
||||
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import {
|
||||
WorkspaceMigrationColumnAction,
|
||||
WorkspaceMigrationColumnActionType,
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export class WorkspaceMetadataCacheException extends CustomException {
|
||||
code: WorkspaceMetadataCacheExceptionCode;
|
||||
constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) {
|
||||
super(message, code);
|
||||
}
|
||||
}
|
||||
|
||||
export enum WorkspaceMetadataCacheExceptionCode {
|
||||
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import {
|
||||
WorkspaceMetadataCacheException,
|
||||
WorkspaceMetadataCacheExceptionCode,
|
||||
} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMetadataCacheService {
|
||||
logger = new Logger(WorkspaceMetadataCacheService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
private readonly workspaceRepository: Repository<Workspace>,
|
||||
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
|
||||
@InjectRepository(ObjectMetadataEntity, 'metadata')
|
||||
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
|
||||
) {}
|
||||
|
||||
async recomputeMetadataCache(
|
||||
workspaceId: string,
|
||||
force = false,
|
||||
): Promise<void> {
|
||||
const currentCacheVersion =
|
||||
await this.getMetadataVersionFromCache(workspaceId);
|
||||
|
||||
const currentDatabaseVersion =
|
||||
await this.getMetadataVersionFromDatabase(workspaceId);
|
||||
|
||||
if (currentDatabaseVersion === undefined) {
|
||||
throw new WorkspaceMetadataCacheException(
|
||||
'Metadata version not found in the database',
|
||||
WorkspaceMetadataCacheExceptionCode.METADATA_VERSION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
if (!force && currentCacheVersion === currentDatabaseVersion) {
|
||||
return;
|
||||
}
|
||||
|
||||
const isAlreadyCaching =
|
||||
await this.workspaceCacheStorageService.getObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId,
|
||||
currentDatabaseVersion,
|
||||
);
|
||||
|
||||
if (isAlreadyCaching) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentCacheVersion !== undefined) {
|
||||
this.workspaceCacheStorageService.flush(workspaceId, currentCacheVersion);
|
||||
}
|
||||
|
||||
await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId,
|
||||
currentDatabaseVersion,
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.setMetadataVersion(
|
||||
workspaceId,
|
||||
currentDatabaseVersion,
|
||||
);
|
||||
|
||||
const freshObjectMetadataCollection =
|
||||
await this.objectMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
relations: [
|
||||
'fields.object',
|
||||
'fields',
|
||||
'fields.fromRelationMetadata',
|
||||
'fields.toRelationMetadata',
|
||||
'fields.fromRelationMetadata.toObjectMetadata',
|
||||
],
|
||||
});
|
||||
|
||||
await this.workspaceCacheStorageService.setObjectMetadataCollection(
|
||||
workspaceId,
|
||||
currentDatabaseVersion,
|
||||
freshObjectMetadataCollection,
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.removeObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId,
|
||||
currentDatabaseVersion,
|
||||
);
|
||||
}
|
||||
|
||||
private async getMetadataVersionFromDatabase(
|
||||
workspaceId: string,
|
||||
): Promise<number | undefined> {
|
||||
const workspace = await this.workspaceRepository.findOne({
|
||||
where: { id: workspaceId },
|
||||
});
|
||||
|
||||
return workspace?.metadataVersion;
|
||||
}
|
||||
|
||||
private async getMetadataVersionFromCache(
|
||||
workspaceId: string,
|
||||
): Promise<number | undefined> {
|
||||
return await this.workspaceCacheStorageService.getMetadataVersion(
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Workspace], 'core'),
|
||||
TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
|
||||
WorkspaceCacheStorageModule,
|
||||
],
|
||||
exports: [WorkspaceMetadataCacheService],
|
||||
providers: [WorkspaceMetadataCacheService],
|
||||
})
|
||||
export class WorkspaceMetadataCacheModule {}
|
||||
@ -0,0 +1,12 @@
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export class WorkspaceMetadataVersionException extends CustomException {
|
||||
code: WorkspaceMetadataVersionExceptionCode;
|
||||
constructor(message: string, code: WorkspaceMetadataVersionExceptionCode) {
|
||||
super(message, code);
|
||||
}
|
||||
}
|
||||
|
||||
export enum WorkspaceMetadataVersionExceptionCode {
|
||||
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import {
|
||||
WorkspaceMetadataVersionException,
|
||||
WorkspaceMetadataVersionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMetadataVersionService {
|
||||
logger = new Logger(WorkspaceMetadataCacheService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
private readonly workspaceRepository: Repository<Workspace>,
|
||||
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
|
||||
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
) {}
|
||||
|
||||
async incrementMetadataVersion(workspaceId: string): Promise<void> {
|
||||
const workspace = await this.workspaceRepository.findOne({
|
||||
where: { id: workspaceId },
|
||||
});
|
||||
|
||||
const metadataVersion = workspace?.metadataVersion;
|
||||
|
||||
if (metadataVersion === undefined) {
|
||||
throw new WorkspaceMetadataVersionException(
|
||||
'Metadata version not found',
|
||||
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const newMetadataVersion = metadataVersion + 1;
|
||||
|
||||
await this.workspaceRepository.update(
|
||||
{ id: workspaceId },
|
||||
{ metadataVersion: newMetadataVersion },
|
||||
);
|
||||
|
||||
await this.workspaceMetadataCacheService.recomputeMetadataCache(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.twentyORMGlobalManager.loadDataSourceForWorkspace(workspaceId);
|
||||
}
|
||||
}
|
||||
@ -2,13 +2,15 @@ import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Workspace], 'core'),
|
||||
WorkspaceCacheStorageModule,
|
||||
WorkspaceMetadataCacheModule,
|
||||
],
|
||||
exports: [WorkspaceMetadataVersionService],
|
||||
providers: [WorkspaceMetadataVersionService],
|
||||
|
||||
@ -1,81 +0,0 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMetadataVersionService {
|
||||
logger = new Logger(WorkspaceMetadataVersionService.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
private readonly workspaceRepository: Repository<Workspace>,
|
||||
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
|
||||
) {}
|
||||
|
||||
async flushCacheIfMetadataVersionIsOutdated(
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
const currentVersion =
|
||||
(await this.workspaceCacheStorageService.getMetadataVersion(
|
||||
workspaceId,
|
||||
)) ?? 1;
|
||||
|
||||
let latestVersion = await this.getMetadataVersion(workspaceId);
|
||||
|
||||
if (latestVersion === undefined || currentVersion !== latestVersion) {
|
||||
this.logger.log(
|
||||
`Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`,
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.flush(workspaceId);
|
||||
|
||||
latestVersion = await this.incrementMetadataVersion(workspaceId);
|
||||
|
||||
await this.workspaceCacheStorageService.setMetadataVersion(
|
||||
workspaceId,
|
||||
latestVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async incrementMetadataVersion(workspaceId: string): Promise<number> {
|
||||
const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0;
|
||||
const newMetadataVersion = metadataVersion + 1;
|
||||
|
||||
await this.workspaceRepository.update(
|
||||
{ id: workspaceId },
|
||||
{ metadataVersion: newMetadataVersion },
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.flush(workspaceId);
|
||||
|
||||
await this.workspaceCacheStorageService.setMetadataVersion(
|
||||
workspaceId,
|
||||
newMetadataVersion,
|
||||
);
|
||||
|
||||
return newMetadataVersion;
|
||||
}
|
||||
|
||||
async getMetadataVersion(workspaceId: string): Promise<number | undefined> {
|
||||
const workspace = await this.workspaceRepository.findOne({
|
||||
where: { id: workspaceId },
|
||||
});
|
||||
|
||||
return workspace?.metadataVersion;
|
||||
}
|
||||
|
||||
async resetMetadataVersion(workspaceId: string): Promise<void> {
|
||||
await this.workspaceRepository.update(
|
||||
{ id: workspaceId },
|
||||
{ metadataVersion: 1 },
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.flush(workspaceId);
|
||||
await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1);
|
||||
}
|
||||
}
|
||||
@ -6,8 +6,8 @@ import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filt
|
||||
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
|
||||
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
|
||||
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { handleExceptionAndConvertToGraphQLError } from 'src/engine/utils/global-exception-handler.util';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
|
||||
class GraphqlTokenValidationProxy {
|
||||
private tokenService: TokenService;
|
||||
@ -33,7 +33,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware
|
||||
{
|
||||
constructor(
|
||||
private readonly tokenService: TokenService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
private readonly workspaceStorageCacheService: WorkspaceCacheStorageService,
|
||||
private readonly exceptionHandlerService: ExceptionHandlerService,
|
||||
) {}
|
||||
|
||||
@ -73,7 +73,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware
|
||||
|
||||
data = await graphqlTokenValidationProxy.validateToken(req);
|
||||
const metadataVersion =
|
||||
await this.workspaceMetadataVersionService.getMetadataVersion(
|
||||
await this.workspaceStorageCacheService.getMetadataVersion(
|
||||
data.workspace.id,
|
||||
);
|
||||
|
||||
|
||||
@ -0,0 +1,15 @@
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export class TwentyORMException extends CustomException {
|
||||
code: TwentyORMExceptionCode;
|
||||
constructor(message: string, code: TwentyORMExceptionCode) {
|
||||
super(message, code);
|
||||
}
|
||||
}
|
||||
|
||||
export enum TwentyORMExceptionCode {
|
||||
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
|
||||
METADATA_VERSION_MISMATCH = 'METADATA_VERSION_MISMATCH',
|
||||
METADATA_COLLECTION_NOT_FOUND = 'METADATA_COLLECTION_NOT_FOUND',
|
||||
WORKSPACE_SCHEMA_NOT_FOUND = 'WORKSPACE_SCHEMA_NOT_FOUND',
|
||||
}
|
||||
@ -19,6 +19,7 @@ export class EntitySchemaRelationFactory {
|
||||
|
||||
async create(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
fieldMetadataCollection: FieldMetadataEntity[],
|
||||
): Promise<EntitySchemaRelationMap> {
|
||||
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
|
||||
@ -40,6 +41,7 @@ export class EntitySchemaRelationFactory {
|
||||
const objectMetadataCollection =
|
||||
await this.workspaceCacheStorageService.getObjectMetadataCollection(
|
||||
workspaceId,
|
||||
metadataVersion,
|
||||
);
|
||||
|
||||
if (!objectMetadataCollection) {
|
||||
|
||||
@ -17,6 +17,7 @@ export class EntitySchemaFactory {
|
||||
|
||||
async create(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
objectMetadata: ObjectMetadataEntity,
|
||||
): Promise<EntitySchema> {
|
||||
const columns = this.entitySchemaColumnFactory.create(
|
||||
@ -26,6 +27,7 @@ export class EntitySchemaFactory {
|
||||
|
||||
const relations = await this.entitySchemaRelationFactory.create(
|
||||
workspaceId,
|
||||
metadataVersion,
|
||||
objectMetadata.fields,
|
||||
);
|
||||
|
||||
|
||||
@ -11,11 +11,11 @@ export class ScopedWorkspaceContextFactory {
|
||||
|
||||
public create(): {
|
||||
workspaceId: string | null;
|
||||
workspaceMetadataVersion: string | null;
|
||||
workspaceMetadataVersion: number | null;
|
||||
} {
|
||||
const workspaceId: string | undefined =
|
||||
this.request?.['req']?.['workspaceId'];
|
||||
const workspaceMetadataVersion: string | undefined =
|
||||
const workspaceMetadataVersion: number | undefined =
|
||||
this.request?.['req']?.['workspaceMetadataVersion'];
|
||||
|
||||
return {
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { EntitySchema, Repository } from 'typeorm';
|
||||
import { EntitySchema } from 'typeorm';
|
||||
|
||||
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
|
||||
import {
|
||||
TwentyORMException,
|
||||
TwentyORMExceptionCode,
|
||||
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
|
||||
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
|
||||
import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage';
|
||||
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
|
||||
@ -20,63 +22,56 @@ export class WorkspaceDatasourceFactory {
|
||||
private readonly dataSourceService: DataSourceService,
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
@InjectRepository(ObjectMetadataEntity, 'metadata')
|
||||
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
|
||||
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
|
||||
private readonly entitySchemaFactory: EntitySchemaFactory,
|
||||
) {}
|
||||
|
||||
public async create(
|
||||
workspaceId: string,
|
||||
workspaceMetadataVersion: string | null,
|
||||
workspaceMetadataVersion: number | null,
|
||||
): Promise<WorkspaceDataSource> {
|
||||
const latestWorkspaceMetadataVersion =
|
||||
await this.workspaceMetadataVersionService.getMetadataVersion(
|
||||
await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
|
||||
|
||||
if (latestWorkspaceMetadataVersion === undefined) {
|
||||
await this.workspaceMetadataCacheService.recomputeMetadataCache(
|
||||
workspaceId,
|
||||
);
|
||||
throw new TwentyORMException(
|
||||
`Metadata version not found for workspace ${workspaceId}`,
|
||||
TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const desiredWorkspaceMetadataVersion =
|
||||
workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
|
||||
|
||||
if (!desiredWorkspaceMetadataVersion) {
|
||||
throw new Error(
|
||||
`Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
|
||||
throw new Error(
|
||||
throw new TwentyORMException(
|
||||
`Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
|
||||
TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
|
||||
);
|
||||
}
|
||||
|
||||
const workspaceDataSource = await this.cacheManager.execute(
|
||||
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
|
||||
`${workspaceId}-${desiredWorkspaceMetadataVersion}`,
|
||||
async () => {
|
||||
let cachedObjectMetadataCollection =
|
||||
const cachedObjectMetadataCollection =
|
||||
await this.workspaceCacheStorageService.getObjectMetadataCollection(
|
||||
workspaceId,
|
||||
desiredWorkspaceMetadataVersion,
|
||||
);
|
||||
|
||||
if (!cachedObjectMetadataCollection) {
|
||||
const freshObjectMetadataCollection =
|
||||
await this.objectMetadataRepository.find({
|
||||
where: { workspaceId },
|
||||
relations: [
|
||||
'fields.object',
|
||||
'fields',
|
||||
'fields.fromRelationMetadata',
|
||||
'fields.toRelationMetadata',
|
||||
'fields.fromRelationMetadata.toObjectMetadata',
|
||||
],
|
||||
});
|
||||
|
||||
await this.workspaceCacheStorageService.setObjectMetadataCollection(
|
||||
await this.workspaceMetadataCacheService.recomputeMetadataCache(
|
||||
workspaceId,
|
||||
freshObjectMetadataCollection,
|
||||
true,
|
||||
);
|
||||
|
||||
cachedObjectMetadataCollection = freshObjectMetadataCollection;
|
||||
throw new TwentyORMException(
|
||||
`Object metadata collection not found for workspace ${workspaceId}`,
|
||||
TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const dataSourceMetadata =
|
||||
@ -85,20 +80,16 @@ export class WorkspaceDatasourceFactory {
|
||||
);
|
||||
|
||||
if (!dataSourceMetadata) {
|
||||
throw new Error(
|
||||
`Data source metadata not found for workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!cachedObjectMetadataCollection) {
|
||||
throw new Error(
|
||||
`Object metadata collection not found for workspace ${workspaceId}`,
|
||||
throw new TwentyORMException(
|
||||
`Workspace Schema not found for workspace ${workspaceId}`,
|
||||
TwentyORMExceptionCode.WORKSPACE_SCHEMA_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const cachedEntitySchemaOptions =
|
||||
await this.workspaceCacheStorageService.getORMEntitySchema(
|
||||
workspaceId,
|
||||
desiredWorkspaceMetadataVersion,
|
||||
);
|
||||
|
||||
let cachedEntitySchemas: EntitySchema[];
|
||||
@ -110,12 +101,17 @@ export class WorkspaceDatasourceFactory {
|
||||
} else {
|
||||
const entitySchemas = await Promise.all(
|
||||
cachedObjectMetadataCollection.map((objectMetadata) =>
|
||||
this.entitySchemaFactory.create(workspaceId, objectMetadata),
|
||||
this.entitySchemaFactory.create(
|
||||
workspaceId,
|
||||
desiredWorkspaceMetadataVersion,
|
||||
objectMetadata,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
await this.workspaceCacheStorageService.setORMEntitySchema(
|
||||
workspaceId,
|
||||
desiredWorkspaceMetadataVersion,
|
||||
entitySchemas.map((entitySchema) => entitySchema.options),
|
||||
);
|
||||
|
||||
|
||||
@ -47,6 +47,10 @@ export class TwentyORMGlobalManager {
|
||||
}
|
||||
|
||||
async getDataSourceForWorkspace(workspaceId: string) {
|
||||
return this.workspaceDataSourceFactory.create(workspaceId, null);
|
||||
return await this.workspaceDataSourceFactory.create(workspaceId, null);
|
||||
}
|
||||
|
||||
async loadDataSourceForWorkspace(workspaceId: string) {
|
||||
await this.workspaceDataSourceFactory.create(workspaceId, null);
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
|
||||
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
|
||||
import { entitySchemaFactories } from 'src/engine/twenty-orm/factories';
|
||||
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
@ -16,7 +16,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
|
||||
TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
|
||||
DataSourceModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
WorkspaceMetadataVersionModule,
|
||||
WorkspaceMetadataCacheModule,
|
||||
],
|
||||
providers: [
|
||||
...entitySchemaFactories,
|
||||
|
||||
@ -13,6 +13,7 @@ enum WorkspaceCacheKeys {
|
||||
GraphQLOperations = 'graphql:operations',
|
||||
ORMEntitySchemas = 'orm:entity-schemas',
|
||||
MetadataObjectMetadataCollection = 'metadata:object-metadata-collection',
|
||||
MetadataObjectMetadataCollectionOngoingCachingLock = 'metadata:object-metadata-collection-ongoing-caching-lock',
|
||||
MetadataVersion = 'metadata:workspace-metadata-version',
|
||||
}
|
||||
|
||||
@ -25,26 +26,31 @@ export class WorkspaceCacheStorageService {
|
||||
|
||||
setORMEntitySchema(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
entitySchemas: EntitySchemaOptions<any>[],
|
||||
) {
|
||||
return this.cacheStorageService.set<EntitySchemaOptions<any>[]>(
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`,
|
||||
entitySchemas,
|
||||
);
|
||||
}
|
||||
|
||||
getORMEntitySchema(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<EntitySchemaOptions<any>[] | undefined> {
|
||||
return this.cacheStorageService.get<EntitySchemaOptions<any>[]>(
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
setMetadataVersion(workspaceId: string, version: number): Promise<void> {
|
||||
setMetadataVersion(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<void> {
|
||||
return this.cacheStorageService.set<number>(
|
||||
`${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`,
|
||||
version,
|
||||
metadataVersion,
|
||||
);
|
||||
}
|
||||
|
||||
@ -54,70 +60,113 @@ export class WorkspaceCacheStorageService {
|
||||
);
|
||||
}
|
||||
|
||||
addObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
) {
|
||||
return this.cacheStorageService.set<boolean>(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
removeObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
) {
|
||||
return this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
getObjectMetadataCollectionOngoingCachingLock(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<boolean | undefined> {
|
||||
return this.cacheStorageService.get<boolean>(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
setObjectMetadataCollection(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
objectMetadataCollection: ObjectMetadataEntity[],
|
||||
) {
|
||||
return this.cacheStorageService.set<ObjectMetadataEntity[]>(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`,
|
||||
objectMetadataCollection,
|
||||
);
|
||||
}
|
||||
|
||||
getObjectMetadataCollection(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<ObjectMetadataEntity[] | undefined> {
|
||||
return this.cacheStorageService.get<ObjectMetadataEntity[]>(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
setGraphQLTypeDefs(workspaceId: string, typeDefs: string): Promise<void> {
|
||||
setGraphQLTypeDefs(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
typeDefs: string,
|
||||
): Promise<void> {
|
||||
return this.cacheStorageService.set<string>(
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`,
|
||||
typeDefs,
|
||||
);
|
||||
}
|
||||
|
||||
getGraphQLTypeDefs(workspaceId: string): Promise<string | undefined> {
|
||||
getGraphQLTypeDefs(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<string | undefined> {
|
||||
return this.cacheStorageService.get<string>(
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
setGraphQLUsedScalarNames(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
usedScalarNames: string[],
|
||||
): Promise<void> {
|
||||
return this.cacheStorageService.set<string[]>(
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`,
|
||||
usedScalarNames,
|
||||
);
|
||||
}
|
||||
|
||||
getGraphQLUsedScalarNames(
|
||||
workspaceId: string,
|
||||
metadataVersion: number,
|
||||
): Promise<string[] | undefined> {
|
||||
return this.cacheStorageService.get<string[]>(
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
|
||||
async flush(workspaceId: string): Promise<void> {
|
||||
async flush(workspaceId: string, metadataVersion: number): Promise<void> {
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`,
|
||||
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
|
||||
await this.cacheStorageService.del(
|
||||
`${WorkspaceCacheKeys.MetadataObjectMetadataCollectionOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ import { DataSource, QueryFailedError } from 'typeorm';
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
|
||||
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
|
||||
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
|
||||
|
||||
Loading…
Reference in New Issue
Block a user