From 41fe8f7fea8140a3d93738d113180ee7b1b2a106 Mon Sep 17 00:00:00 2001 From: Weiko Date: Wed, 18 Sep 2024 20:06:04 +0200 Subject: [PATCH] Fix nested relations with large dataset in find queries (#7127) ## Before before ## After after --- .../process-nested-relations.helper.ts | 187 ++++++++++++++++++ ...raphql-query-find-many-resolver.service.ts | 42 +++- ...graphql-query-find-one-resolver.service.ts | 27 ++- .../get-relation-object-metadata.util.ts | 25 ++- 4 files changed, 267 insertions(+), 14 deletions(-) create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts new file mode 100644 index 0000000000..66a0b35315 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts @@ -0,0 +1,187 @@ +import { + FindManyOptions, + FindOptionsRelations, + In, + ObjectLiteral, +} from 'typeorm'; + +import { Record as IRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface'; + +import { + getRelationMetadata, + getRelationObjectMetadata, +} from 'src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util'; +import { + ObjectMetadataMap, + ObjectMetadataMapItem, +} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { deduceRelationDirection } from 'src/engine/utils/deduce-relation-direction.util'; + +export class ProcessNestedRelationsHelper { + private readonly twentyORMGlobalManager: TwentyORMGlobalManager; + + constructor(twentyORMGlobalManager: TwentyORMGlobalManager) { + this.twentyORMGlobalManager = twentyORMGlobalManager; + } + + private async processFromRelation( + objectMetadataMap: ObjectMetadataMap, + parentObjectMetadataItem: ObjectMetadataMapItem, + parentObjectRecords: ObjectRecord[], + relationName: string, + nestedRelations: any, + limit: number, + authContext: any, + ) { + const relationFieldMetadata = parentObjectMetadataItem.fields[relationName]; + const relationMetadata = getRelationMetadata(relationFieldMetadata); + + const inverseRelationName = + objectMetadataMap[relationMetadata.toObjectMetadataId]?.fields[ + relationMetadata.toFieldMetadataId + ]?.name; + + const referenceObjectMetadata = getRelationObjectMetadata( + relationFieldMetadata, + objectMetadataMap, + ); + + const referenceObjectMetadataName = referenceObjectMetadata.nameSingular; + + const relationRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + authContext.workspace.id, + referenceObjectMetadataName, + ); + + const relationIds = parentObjectRecords.map((item) => item.id); + + const uniqueRelationIds = [...new Set(relationIds)]; + + const relationFindOptions: FindManyOptions = { + where: { + [`${inverseRelationName}Id`]: In(uniqueRelationIds), + }, + take: limit * parentObjectRecords.length, + }; + + const relationResults = await relationRepository.find(relationFindOptions); + + parentObjectRecords.forEach((item) => { + (item as any)[relationName] = relationResults.filter( + (rel) => rel[`${inverseRelationName}Id`] === item.id, + ); + }); + + if (Object.keys(nestedRelations).length > 0) { + await this.processNestedRelations( + objectMetadataMap, + objectMetadataMap[relationName], + relationResults as ObjectRecord[], + nestedRelations as Record>, + limit, + authContext, + ); + } + } + + private async processToRelation( + objectMetadataMap: ObjectMetadataMap, + parentObjectMetadataItem: ObjectMetadataMapItem, + parentObjectRecords: ObjectRecord[], + relationName: string, + nestedRelations: any, + limit: number, + authContext: any, + ) { + const relationFieldMetadata = parentObjectMetadataItem.fields[relationName]; + + const referenceObjectMetadata = getRelationObjectMetadata( + relationFieldMetadata, + objectMetadataMap, + ); + + const referenceObjectMetadataName = referenceObjectMetadata.nameSingular; + + const relationRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + authContext.workspace.id, + referenceObjectMetadataName, + ); + + const relationIds = parentObjectRecords.map( + (item) => item[`${relationName}Id`], + ); + + const uniqueRelationIds = [...new Set(relationIds)]; + + const relationFindOptions: FindManyOptions = { + where: { + id: In(uniqueRelationIds), + }, + take: limit, + }; + + const relationResults = await relationRepository.find(relationFindOptions); + + parentObjectRecords.forEach((item) => { + (item as any)[relationName] = relationResults.filter( + (rel) => rel.id === item[`${relationName}Id`], + )[0]; + }); + + if (Object.keys(nestedRelations).length > 0) { + await this.processNestedRelations( + objectMetadataMap, + objectMetadataMap[relationName], + relationResults as ObjectRecord[], + nestedRelations as Record>, + limit, + authContext, + ); + } + } + + public async processNestedRelations( + objectMetadataMap: ObjectMetadataMap, + parentObjectMetadataItem: ObjectMetadataMapItem, + parentObjectRecords: ObjectRecord[], + relations: Record>, + limit: number, + authContext: any, + ) { + for (const [relationName, nestedRelations] of Object.entries(relations)) { + const relationFieldMetadata = + parentObjectMetadataItem.fields[relationName]; + const relationMetadata = getRelationMetadata(relationFieldMetadata); + + const relationDirection = deduceRelationDirection( + relationFieldMetadata, + relationMetadata, + ); + + if (relationDirection === 'to') { + await this.processToRelation( + objectMetadataMap, + parentObjectMetadataItem, + parentObjectRecords, + relationName, + nestedRelations, + limit, + authContext, + ); + } else { + await this.processFromRelation( + objectMetadataMap, + parentObjectMetadataItem, + parentObjectRecords, + relationName, + nestedRelations, + limit, + authContext, + ); + } + } + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts index 5a48d71b41..84caa77dba 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts @@ -17,11 +17,15 @@ import { GraphqlQueryRunnerExceptionCode, } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; +import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; import { ObjectRecordsToGraphqlConnectionMapper } from 'src/engine/api/graphql/graphql-query-runner/orm-mappers/object-records-to-graphql-connection.mapper'; import { applyRangeFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/apply-range-filter.util'; import { decodeCursor } from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util'; import { getObjectMetadataOrThrow } from 'src/engine/api/graphql/graphql-query-runner/utils/get-object-metadata-or-throw.util'; -import { generateObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; +import { + generateObjectMetadataMap, + ObjectMetadataMapItem, +} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; export class GraphqlQueryFindManyResolverService { @@ -78,12 +82,12 @@ export class GraphqlQueryFindManyResolverService { const limit = args.first ?? args.last ?? QUERY_MAX_RECORDS; this.addOrderByColumnsToSelect(order, select); + this.addForeingKeyColumnsToSelect(relations, select, objectMetadata); const findOptions: FindManyOptions = { where, order, select, - relations, take: limit + 1, }; @@ -95,7 +99,10 @@ export class GraphqlQueryFindManyResolverService { applyRangeFilter(where, cursor, isForwardPagination); } - const objectRecords = await repository.find(findOptions); + const objectRecords = (await repository.find( + findOptions, + )) as ObjectRecord[]; + const { hasNextPage, hasPreviousPage } = this.getPaginationInfo( objectRecords, limit, @@ -106,11 +113,26 @@ export class GraphqlQueryFindManyResolverService { objectRecords.pop(); } + const processNestedRelationsHelper = new ProcessNestedRelationsHelper( + this.twentyORMGlobalManager, + ); + + if (relations) { + await processNestedRelationsHelper.processNestedRelations( + objectMetadataMap, + objectMetadata, + objectRecords, + relations, + limit, + authContext, + ); + } + const typeORMObjectRecordsParser = new ObjectRecordsToGraphqlConnectionMapper(objectMetadataMap); return typeORMObjectRecordsParser.createConnection( - objectRecords as ObjectRecord[], + objectRecords, objectMetadataItem.nameSingular, limit, totalCount, @@ -179,6 +201,18 @@ export class GraphqlQueryFindManyResolverService { } } + private addForeingKeyColumnsToSelect( + relations: Record, + select: Record, + objectMetadata: ObjectMetadataMapItem, + ) { + for (const column of Object.keys(relations || {})) { + if (!select[`${column}Id`] && objectMetadata.fields[`${column}Id`]) { + select[`${column}Id`] = true; + } + } + } + private getPaginationInfo( objectRecords: any[], limit: number, diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-one-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-one-resolver.service.ts index fc5740af1c..2cbfcb01f0 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-one-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-one-resolver.service.ts @@ -7,11 +7,13 @@ import { import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface'; import { FindOneResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; +import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant'; import { GraphqlQueryRunnerException, GraphqlQueryRunnerExceptionCode, } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; +import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; import { ObjectRecordsToGraphqlConnectionMapper } from 'src/engine/api/graphql/graphql-query-runner/orm-mappers/object-records-to-graphql-connection.mapper'; import { getObjectMetadataOrThrow } from 'src/engine/api/graphql/graphql-query-runner/utils/get-object-metadata-or-throw.util'; import { generateObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; @@ -59,7 +61,11 @@ export class GraphqlQueryFindOneResolverService { ); const where = graphqlQueryParser.parseFilter(args.filter ?? ({} as Filter)); - const objectRecord = await repository.findOne({ where, select, relations }); + const objectRecord = (await repository.findOne({ + where, + select, + })) as ObjectRecord; + const limit = QUERY_MAX_RECORDS; if (!objectRecord) { throw new GraphqlQueryRunnerException( @@ -68,11 +74,28 @@ export class GraphqlQueryFindOneResolverService { ); } + const processNestedRelationsHelper = new ProcessNestedRelationsHelper( + this.twentyORMGlobalManager, + ); + + const objectRecords = [objectRecord]; + + if (relations) { + await processNestedRelationsHelper.processNestedRelations( + objectMetadataMap, + objectMetadata, + objectRecords, + relations, + limit, + authContext, + ); + } + const typeORMObjectRecordsParser = new ObjectRecordsToGraphqlConnectionMapper(objectMetadataMap); return typeORMObjectRecordsParser.processRecord( - objectRecord, + objectRecords[0], objectMetadataItem.nameSingular, 1, 1, diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util.ts index 06e92388e5..d05bdcced2 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util.ts @@ -1,5 +1,6 @@ import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; +import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; import { deduceRelationDirection, @@ -10,14 +11,7 @@ export const getRelationObjectMetadata = ( fieldMetadata: FieldMetadataInterface, objectMetadataMap: ObjectMetadataMap, ) => { - const relationMetadata = - fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata; - - if (!relationMetadata) { - throw new Error( - `Relation metadata not found for field ${fieldMetadata.name}`, - ); - } + const relationMetadata = getRelationMetadata(fieldMetadata); const relationDirection = deduceRelationDirection( fieldMetadata, @@ -37,3 +31,18 @@ export const getRelationObjectMetadata = ( return referencedObjectMetadata; }; + +export const getRelationMetadata = ( + fieldMetadata: FieldMetadataInterface, +): RelationMetadataEntity => { + const relationMetadata = + fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata; + + if (!relationMetadata) { + throw new Error( + `Relation metadata not found for field ${fieldMetadata.name}`, + ); + } + + return relationMetadata; +};