From 24dbabcad7a999a6e36e8d80abb1bc00ab26a2ff Mon Sep 17 00:00:00 2001 From: Weiko Date: Thu, 21 Nov 2024 11:49:19 +0100 Subject: [PATCH 1/2] Improve object metadata maps size (#8635) ## Context The object metadata map is becoming quite large and its structure could be simplified. We are removing byNameSingular/byNamePlural keys, the former can be retrieved through a new helper and the latter is not used in the code base currently. --- .../graphql-query.parser.ts | 8 ++-- ...ct-records-to-graphql-connection.helper.ts | 6 ++- .../process-nested-relations.helper.ts | 41 ++++++++++++++---- ...-query-find-duplicates-resolver.service.ts | 18 ++++++-- .../types/object-metadata-maps.ts | 3 +- ...ata-map-item-by-name-singular.util.spec.ts | 42 +++++++++++++++++++ .../generate-object-metadata-maps.util.ts | 9 ++-- ...metadata-map-item-by-name-singular.util.ts | 11 +++++ .../repository/workspace.repository.ts | 15 ++++--- .../record-crud.workflow-action.ts | 6 ++- 10 files changed, 130 insertions(+), 29 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/utils/__tests__/get-object-metadata-map-item-by-name-singular.util.spec.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util.ts diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser.ts index 99157e10f7..9dc415fa2e 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser.ts @@ -19,6 +19,7 @@ import { import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; export class GraphqlQueryParser { private fieldMetadataMapByName: FieldMetadataMap; @@ -108,9 +109,10 @@ export class GraphqlQueryParser { parentObjectMetadata: ObjectMetadataItemWithFieldMaps, graphqlSelectedFields: Partial>, ): GraphqlQuerySelectedFieldsResult { - const parentFields = - this.objectMetadataMaps.byNameSingular[parentObjectMetadata.nameSingular] - ?.fieldsByName; + const parentFields = getObjectMetadataMapItemByNameSingular( + this.objectMetadataMaps, + parentObjectMetadata.nameSingular, + )?.fieldsByName; if (!parentFields) { throw new Error( diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts index 0553c2e736..b91b0301b2 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper.ts @@ -17,6 +17,7 @@ import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-meta import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory'; import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util'; import { isPlainObject } from 'src/utils/is-plain-object'; @@ -143,7 +144,10 @@ export class ObjectRecordsToGraphqlConnectionHelper { ); } - const objectMetadata = this.objectMetadataMaps.byNameSingular[objectName]; + const objectMetadata = getObjectMetadataMapItemByNameSingular( + this.objectMetadataMaps, + objectName, + ); if (!objectMetadata) { throw new GraphqlQueryRunnerException( diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts index 63fa227279..add94baa3e 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts @@ -7,6 +7,10 @@ import { import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; +import { + GraphqlQueryRunnerException, + GraphqlQueryRunnerExceptionCode, +} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { getRelationMetadata, @@ -15,6 +19,7 @@ import { import { AggregationField } from 'src/engine/api/graphql/workspace-schema-builder/utils/get-available-aggregations-from-object-fields.util'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util'; import { deduceRelationDirection } from 'src/engine/utils/deduce-relation-direction.util'; @@ -177,13 +182,23 @@ export class ProcessNestedRelationsHelper { joinField: `${inverseRelationName}Id`, }); + const referenceObjectMetadataItemWithFieldsMaps = + getObjectMetadataMapItemByNameSingular( + objectMetadataMaps, + referenceObjectMetadata.nameSingular, + ); + + if (!referenceObjectMetadataItemWithFieldsMaps) { + throw new GraphqlQueryRunnerException( + `Object ${referenceObjectMetadata.nameSingular} not found`, + GraphqlQueryRunnerExceptionCode.OBJECT_METADATA_NOT_FOUND, + ); + } + if (Object.keys(nestedRelations).length > 0) { await this.processNestedRelations({ objectMetadataMaps, - parentObjectMetadataItem: - objectMetadataMaps.byNameSingular[ - referenceObjectMetadata.nameSingular - ], + parentObjectMetadataItem: referenceObjectMetadataItemWithFieldsMaps, parentObjectRecords: relationResults as ObjectRecord[], parentObjectRecordsAggregatedValues: relationAggregatedFieldsResult, relations: nestedRelations as Record< @@ -258,13 +273,23 @@ export class ProcessNestedRelationsHelper { relationName, }); + const referenceObjectMetadataItemWithFieldsMaps = + getObjectMetadataMapItemByNameSingular( + objectMetadataMaps, + referenceObjectMetadata.nameSingular, + ); + + if (!referenceObjectMetadataItemWithFieldsMaps) { + throw new GraphqlQueryRunnerException( + `Object ${referenceObjectMetadata.nameSingular} not found`, + GraphqlQueryRunnerExceptionCode.OBJECT_METADATA_NOT_FOUND, + ); + } + if (Object.keys(nestedRelations).length > 0) { await this.processNestedRelations({ objectMetadataMaps, - parentObjectMetadataItem: - objectMetadataMaps.byNameSingular[ - referenceObjectMetadata.nameSingular - ], + parentObjectMetadataItem: referenceObjectMetadataItemWithFieldsMaps, parentObjectRecords: relationResults as ObjectRecord[], parentObjectRecordsAggregatedValues: relationAggregatedFieldsResult, relations: nestedRelations as Record< diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts index ebc2200c1c..d4b616d759 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts @@ -22,6 +22,7 @@ import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/g import { settings } from 'src/engine/constants/settings'; import { DUPLICATE_CRITERIA_COLLECTION } from 'src/engine/core-modules/duplicate/constants/duplicate-criteria.constants'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { formatData } from 'src/engine/twenty-orm/utils/format-data.util'; import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util'; @@ -56,10 +57,21 @@ export class GraphqlQueryFindDuplicatesResolverService objectMetadataItemWithFieldMaps.nameSingular, ); + const objectMetadataItemWithFieldsMaps = + getObjectMetadataMapItemByNameSingular( + objectMetadataMaps, + objectMetadataItemWithFieldMaps.nameSingular, + ); + + if (!objectMetadataItemWithFieldsMaps) { + throw new GraphqlQueryRunnerException( + `Object ${objectMetadataItemWithFieldMaps.nameSingular} not found`, + GraphqlQueryRunnerExceptionCode.OBJECT_METADATA_NOT_FOUND, + ); + } + const graphqlQueryParser = new GraphqlQueryParser( - objectMetadataMaps.byNameSingular[ - objectMetadataItemWithFieldMaps.nameSingular - ].fieldsByName, + objectMetadataItemWithFieldsMaps?.fieldsByName, objectMetadataMaps, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/types/object-metadata-maps.ts b/packages/twenty-server/src/engine/metadata-modules/types/object-metadata-maps.ts index a10603c36f..d3ff90d220 100644 --- a/packages/twenty-server/src/engine/metadata-modules/types/object-metadata-maps.ts +++ b/packages/twenty-server/src/engine/metadata-modules/types/object-metadata-maps.ts @@ -2,6 +2,5 @@ import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/typ export type ObjectMetadataMaps = { byId: Record; - byNameSingular: Record; - byNamePlural: Record; + idByNameSingular: Record; }; diff --git a/packages/twenty-server/src/engine/metadata-modules/utils/__tests__/get-object-metadata-map-item-by-name-singular.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/utils/__tests__/get-object-metadata-map-item-by-name-singular.util.spec.ts new file mode 100644 index 0000000000..3102d00546 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/utils/__tests__/get-object-metadata-map-item-by-name-singular.util.spec.ts @@ -0,0 +1,42 @@ +import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; + +describe('getObjectMetadataMapItemByNameSingular', () => { + it('should return the correct metadata item when given a valid singular name', () => { + const mockMetadataItem = { + id: 'test-id', + nameSingular: 'company', + } as ObjectMetadataItemWithFieldMaps; + + const mockObjectMetadataMaps: ObjectMetadataMaps = { + byId: { + 'test-id': mockMetadataItem, + }, + idByNameSingular: { + company: 'test-id', + }, + }; + + const result = getObjectMetadataMapItemByNameSingular( + mockObjectMetadataMaps, + 'company', + ); + + expect(result).toBe(mockMetadataItem); + }); + + it('should return undefined when the singular name does not exist', () => { + const mockObjectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular: {}, + }; + + const result = getObjectMetadataMapItemByNameSingular( + mockObjectMetadataMaps, + 'nonexistent', + ); + + expect(result).toBeUndefined(); + }); +}); diff --git a/packages/twenty-server/src/engine/metadata-modules/utils/generate-object-metadata-maps.util.ts b/packages/twenty-server/src/engine/metadata-modules/utils/generate-object-metadata-maps.util.ts index abaea68e10..e83f75cc33 100644 --- a/packages/twenty-server/src/engine/metadata-modules/utils/generate-object-metadata-maps.util.ts +++ b/packages/twenty-server/src/engine/metadata-modules/utils/generate-object-metadata-maps.util.ts @@ -9,8 +9,7 @@ export const generateObjectMetadataMaps = ( ): ObjectMetadataMaps => { const objectMetadataMaps: ObjectMetadataMaps = { byId: {}, - byNameSingular: {}, - byNamePlural: {}, + idByNameSingular: {}, }; for (const objectMetadata of objectMetadataCollection) { @@ -29,10 +28,8 @@ export const generateObjectMetadataMaps = ( }; objectMetadataMaps.byId[objectMetadata.id] = processedObjectMetadata; - objectMetadataMaps.byNameSingular[objectMetadata.nameSingular] = - processedObjectMetadata; - objectMetadataMaps.byNamePlural[objectMetadata.namePlural] = - processedObjectMetadata; + objectMetadataMaps.idByNameSingular[objectMetadata.nameSingular] = + objectMetadata.id; } return objectMetadataMaps; diff --git a/packages/twenty-server/src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util.ts b/packages/twenty-server/src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util.ts new file mode 100644 index 0000000000..aba7bf14e1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util.ts @@ -0,0 +1,11 @@ +import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; + +export const getObjectMetadataMapItemByNameSingular = ( + objectMetadataMaps: ObjectMetadataMaps, + nameSingular: string, +): ObjectMetadataItemWithFieldMaps | undefined => { + return objectMetadataMaps.byId[ + objectMetadataMaps.idByNameSingular[nameSingular] + ]; +}; diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts index b81eaf1c0d..28e060e30c 100644 --- a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts +++ b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts @@ -23,6 +23,7 @@ import { UpsertOptions } from 'typeorm/repository/UpsertOptions'; import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage'; import { formatData } from 'src/engine/twenty-orm/utils/format-data.util'; import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util'; @@ -630,16 +631,20 @@ export class WorkspaceRepository< throw new Error('Object metadata name is missing'); } - const objectMetadata = - this.internalContext.objectMetadataMaps.byNameSingular[ - objectMetadataName - ]; + const objectMetadata = getObjectMetadataMapItemByNameSingular( + this.internalContext.objectMetadataMaps, + objectMetadataName, + ); if (!objectMetadata) { throw new Error( `Object metadata for object "${objectMetadataName}" is missing ` + `in workspace "${this.internalContext.workspaceId}" ` + - `with object metadata collection length: ${this.internalContext.objectMetadataMaps.byNameSingular.length}`, + `with object metadata collection length: ${ + Object.keys( + this.internalContext.objectMetadataMaps.idByNameSingular, + ).length + }`, ); } diff --git a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud.workflow-action.ts b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud.workflow-action.ts index 39f688de4d..b95de52e12 100644 --- a/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud.workflow-action.ts +++ b/packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud.workflow-action.ts @@ -14,6 +14,7 @@ import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/c import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; @@ -179,7 +180,10 @@ export class RecordCRUDWorkflowAction implements WorkflowAction { } const objectMetadataItemWithFieldsMaps = - objectMetadataMaps.byNameSingular[workflowActionInput.objectName]; + getObjectMetadataMapItemByNameSingular( + objectMetadataMaps, + workflowActionInput.objectName, + ); if (!objectMetadataItemWithFieldsMaps) { throw new RecordCRUDActionException( From 9cb076d9e10d78749b806ea283c8645b8f3ebdf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Thu, 21 Nov 2024 11:51:42 +0100 Subject: [PATCH 2/2] Improve docker compose (#8637) Add a proxy script to use the right install.sh branch/version matching the docker-compose Also stop exposing redis publicly as it's not necessary --- packages/twenty-docker/docker-compose.yml | 2 -- packages/twenty-docker/scripts/1-click.sh | 12 ++++++++++++ .../twenty-docker/scripts/install.sh | 0 .../developers/self-hosting/docker-compose.mdx | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 packages/twenty-docker/scripts/1-click.sh rename install.sh => packages/twenty-docker/scripts/install.sh (100%) diff --git a/packages/twenty-docker/docker-compose.yml b/packages/twenty-docker/docker-compose.yml index fcd485a28d..e44ee876c4 100644 --- a/packages/twenty-docker/docker-compose.yml +++ b/packages/twenty-docker/docker-compose.yml @@ -90,8 +90,6 @@ services: redis: image: redis - ports: - - "6379:6379" restart: always volumes: diff --git a/packages/twenty-docker/scripts/1-click.sh b/packages/twenty-docker/scripts/1-click.sh new file mode 100644 index 0000000000..2e5f935726 --- /dev/null +++ b/packages/twenty-docker/scripts/1-click.sh @@ -0,0 +1,12 @@ +pull_version=${VERSION:-$(curl -s https://api.github.com/repos/twentyhq/twenty/releases/latest | grep '"tag_name":' | cut -d '"' -f 4)} +pull_branch=${BRANCH:-$pull_version} + +version_num=${pull_version#v} +target_version="0.32.4" + +# We moved the install script to a different location in v0.32.4 +if [[ -n "$BRANCH" ]] || [[ "$(printf '%s\n' "$target_version" "$version_num" | sort -V | head -n1)" != "$version_num" ]]; then + curl -sL "https://raw.githubusercontent.com/twentyhq/twenty/$pull_branch/packages/twenty-docker/scripts/install.sh" | bash -s -- "$VERSION" "$BRANCH" +else + curl -sL "https://raw.githubusercontent.com/twentyhq/twenty/$pull_branch/install.sh" | bash -s -- "$VERSION" "$BRANCH" +fi diff --git a/install.sh b/packages/twenty-docker/scripts/install.sh similarity index 100% rename from install.sh rename to packages/twenty-docker/scripts/install.sh diff --git a/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx b/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx index e63a706c7a..9fdfb2fcb2 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/docker-compose.mdx @@ -31,7 +31,7 @@ bash <(curl -sL https://git.new/20) To install a specific version or branch: ```bash -VERSION=x.y.z BRANCH=branch-name bash <(curl -sL https://git.new/20) +VERSION=vx.y.z BRANCH=branch-name bash <(curl -sL https://git.new/20) ``` - Replace x.y.z with the desired version number. - Replace branch-name with the name of the branch you want to install.