diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 88ed589335..3b8e0762f1 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -78,6 +78,7 @@ "lodash.isempty": "^4.4.0", "lodash.isobject": "^3.0.2", "lodash.kebabcase": "^4.1.1", + "lodash.omit": "^4.5.0", "lodash.snakecase": "^4.1.1", "lodash.upperfirst": "^4.3.1", "mailparser": "^3.6.5", @@ -101,6 +102,7 @@ "devDependencies": { "@types/lodash.isempty": "^4.4.7", "@types/lodash.isobject": "^3.0.7", + "@types/lodash.omit": "^4.5.9", "@types/lodash.snakecase": "^4.1.7", "@types/lodash.upperfirst": "^4.3.7", "@types/react": "^18.2.39", diff --git a/packages/twenty-server/src/core/analytics/analytics.resolver.ts b/packages/twenty-server/src/core/analytics/analytics.resolver.ts index c912dd52ca..880708b3c1 100644 --- a/packages/twenty-server/src/core/analytics/analytics.resolver.ts +++ b/packages/twenty-server/src/core/analytics/analytics.resolver.ts @@ -2,8 +2,8 @@ import { Resolver, Mutation, Args } from '@nestjs/graphql'; import { UseGuards } from '@nestjs/common'; import { OptionalJwtAuthGuard } from 'src/guards/optional-jwt.auth.guard'; -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; -import { AuthUser } from 'src/decorators/auth-user.decorator'; +import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator'; +import { AuthUser } from 'src/decorators/auth/auth-user.decorator'; import { Workspace } from 'src/core/workspace/workspace.entity'; import { User } from 'src/core/user/user.entity'; diff --git a/packages/twenty-server/src/core/auth/auth.resolver.ts b/packages/twenty-server/src/core/auth/auth.resolver.ts index 55bad8033a..0283260bc5 100644 --- a/packages/twenty-server/src/core/auth/auth.resolver.ts +++ b/packages/twenty-server/src/core/auth/auth.resolver.ts @@ -11,10 +11,10 @@ import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; -import { AuthUser } from 'src/decorators/auth-user.decorator'; +import { AuthUser } from 'src/decorators/auth/auth-user.decorator'; import { assert } from 'src/utils/assert'; import { Workspace } from 'src/core/workspace/workspace.entity'; -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; +import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator'; import { User } from 'src/core/user/user.entity'; import { ApiKeyTokenInput } from 'src/core/auth/dto/api-key-token.input'; import { ValidatePasswordResetToken } from 'src/core/auth/dto/validate-password-reset-token.entity'; diff --git a/packages/twenty-server/src/core/feature-flag/interfaces/feature-flag-map.interface.ts b/packages/twenty-server/src/core/feature-flag/interfaces/feature-flag-map.interface.ts new file mode 100644 index 0000000000..9ec6c9b73e --- /dev/null +++ b/packages/twenty-server/src/core/feature-flag/interfaces/feature-flag-map.interface.ts @@ -0,0 +1,3 @@ +import { FeatureFlagKeys } from 'src/core/feature-flag/feature-flag.entity'; + +export type FeatureFlagMap = Record<`${FeatureFlagKeys}`, boolean>; diff --git a/packages/twenty-server/src/core/messaging/timeline-messaging.resolver.ts b/packages/twenty-server/src/core/messaging/timeline-messaging.resolver.ts index ef0199cb97..edb5a06d36 100644 --- a/packages/twenty-server/src/core/messaging/timeline-messaging.resolver.ts +++ b/packages/twenty-server/src/core/messaging/timeline-messaging.resolver.ts @@ -13,7 +13,7 @@ import { Max } from 'class-validator'; import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; import { Workspace } from 'src/core/workspace/workspace.entity'; -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; +import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator'; import { TimelineMessagingService } from 'src/core/messaging/timeline-messaging.service'; import { TIMELINE_THREADS_MAX_PAGE_SIZE } from 'src/core/messaging/constants/messaging.constants'; import { TimelineThreadsWithTotal } from 'src/core/messaging/dtos/timeline-threads-with-total.dto'; diff --git a/packages/twenty-server/src/core/user/user.resolver.ts b/packages/twenty-server/src/core/user/user.resolver.ts index ef7ec59ada..a6e6c6cdfb 100644 --- a/packages/twenty-server/src/core/user/user.resolver.ts +++ b/packages/twenty-server/src/core/user/user.resolver.ts @@ -15,7 +15,7 @@ import { FileUpload, GraphQLUpload } from 'graphql-upload'; import { SupportDriver } from 'src/integrations/environment/interfaces/support.interface'; import { FileFolder } from 'src/core/file/interfaces/file-folder.interface'; -import { AuthUser } from 'src/decorators/auth-user.decorator'; +import { AuthUser } from 'src/decorators/auth/auth-user.decorator'; import { EnvironmentService } from 'src/integrations/environment/environment.service'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; import { FileUploadService } from 'src/core/file/services/file-upload.service'; diff --git a/packages/twenty-server/src/core/workspace/workspace.resolver.ts b/packages/twenty-server/src/core/workspace/workspace.resolver.ts index e588fad936..a093feaee4 100644 --- a/packages/twenty-server/src/core/workspace/workspace.resolver.ts +++ b/packages/twenty-server/src/core/workspace/workspace.resolver.ts @@ -7,7 +7,7 @@ import { FileFolder } from 'src/core/file/interfaces/file-folder.interface'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; import { FileUploadService } from 'src/core/file/services/file-upload.service'; -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; +import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator'; import { assert } from 'src/utils/assert'; import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; import { UpdateWorkspaceInput } from 'src/core/workspace/dtos/update-workspace-input'; diff --git a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts index 24c4e90e8b..b2aa57ecd5 100644 --- a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts +++ b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts @@ -11,7 +11,7 @@ import { seedWorkspaceMember } from 'src/database/typeorm-seeds/workspace/worksp import { seedPeople } from 'src/database/typeorm-seeds/workspace/people'; import { seedCoreSchema } from 'src/database/typeorm-seeds/core'; import { EnvironmentService } from 'src/integrations/environment/environment.service'; -import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service'; +import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service'; import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service'; import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service'; @@ -62,8 +62,10 @@ export class DataSeedWorkspaceCommand extends CommandRunner { ); await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata( - dataSourceMetadata.id, - this.workspaceId, + { + workspaceId: this.workspaceId, + dataSourceId: dataSourceMetadata.id, + }, ); } catch (error) { console.error(error); diff --git a/packages/twenty-server/src/decorators/user-ability.decorator.ts b/packages/twenty-server/src/decorators/ability/user-ability.decorator.ts similarity index 100% rename from packages/twenty-server/src/decorators/user-ability.decorator.ts rename to packages/twenty-server/src/decorators/ability/user-ability.decorator.ts diff --git a/packages/twenty-server/src/decorators/auth-user.decorator.ts b/packages/twenty-server/src/decorators/auth/auth-user.decorator.ts similarity index 100% rename from packages/twenty-server/src/decorators/auth-user.decorator.ts rename to packages/twenty-server/src/decorators/auth/auth-user.decorator.ts diff --git a/packages/twenty-server/src/decorators/auth-workspace.decorator.ts b/packages/twenty-server/src/decorators/auth/auth-workspace.decorator.ts similarity index 100% rename from packages/twenty-server/src/decorators/auth-workspace.decorator.ts rename to packages/twenty-server/src/decorators/auth/auth-workspace.decorator.ts diff --git a/packages/twenty-server/src/integrations/environment/environment.validation.ts b/packages/twenty-server/src/integrations/environment/environment.validation.ts index c712632f2a..ba2c3d59b8 100644 --- a/packages/twenty-server/src/integrations/environment/environment.validation.ts +++ b/packages/twenty-server/src/integrations/environment/environment.validation.ts @@ -64,7 +64,11 @@ export class EnvironmentVariables { PORT: number; // Database - @IsUrl({ protocols: ['postgres'], require_tld: false, allow_underscores: true }) + @IsUrl({ + protocols: ['postgres'], + require_tld: false, + allow_underscores: true, + }) PG_DATABASE_URL: string; // Frontend URL diff --git a/packages/twenty-server/src/metadata/field-metadata/field-metadata.resolver.ts b/packages/twenty-server/src/metadata/field-metadata/field-metadata.resolver.ts index 3b712968e9..91b7dd6130 100644 --- a/packages/twenty-server/src/metadata/field-metadata/field-metadata.resolver.ts +++ b/packages/twenty-server/src/metadata/field-metadata/field-metadata.resolver.ts @@ -2,7 +2,7 @@ import { UseGuards } from '@nestjs/common'; import { Args, Mutation, Resolver } from '@nestjs/graphql'; import { Workspace } from 'src/core/workspace/workspace.entity'; -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; +import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator'; import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; import { CreateOneFieldMetadataInput } from 'src/metadata/field-metadata/dtos/create-field.input'; import { FieldMetadataDTO } from 'src/metadata/field-metadata/dtos/field-metadata.dto'; diff --git a/packages/twenty-server/src/metadata/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/metadata/field-metadata/field-metadata.service.ts index 7b5bba5d73..723a8f9981 100644 --- a/packages/twenty-server/src/metadata/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/metadata/field-metadata/field-metadata.service.ts @@ -24,6 +24,7 @@ import { DataSourceService } from 'src/metadata/data-source/data-source.service' import { UpdateFieldInput } from 'src/metadata/field-metadata/dtos/update-field.input'; import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory'; import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; +import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util'; import { FieldMetadataEntity, @@ -111,6 +112,7 @@ export class FieldMetadataService extends TypeOrmQueryService { + let comparator: WorkspaceFieldComparator; + + beforeEach(() => { + // Initialize the comparator before each test + comparator = new WorkspaceFieldComparator(); + }); + + function createMockFieldMetadata(values: any) { + return { + workspaceId: 'some-workspace-id', + type: 'TEXT', + name: 'DefaultFieldName', + label: 'Default Field Label', + targetColumnMap: 'default_column', + defaultValue: null, + description: 'Default description', + isCustom: false, + isSystem: false, + isNullable: true, + ...values, + }; + } + + it('should generate CREATE action for new fields', () => { + const original = { fields: [] } as any; + const standard = { + fields: [createMockFieldMetadata({ name: 'New Field' })], + } as any; + + const result = comparator.compare(original, standard); + + expect(result).toEqual([ + { + action: ComparatorAction.CREATE, + object: expect.objectContaining(standard.fields[0]), + }, + ]); + }); + + it('should generate UPDATE action for modified fields', () => { + const original = { + fields: [ + createMockFieldMetadata({ + id: '1', + isNullable: true, + }), + ], + } as any; + const standard = { + fields: [ + createMockFieldMetadata({ + isNullable: false, + }), + ], + } as any; + + const result = comparator.compare(original, standard); + + expect(result).toEqual([ + { + action: ComparatorAction.UPDATE, + object: expect.objectContaining({ id: '1', isNullable: false }), + }, + ]); + }); + + it('should generate DELETE action for removed fields', () => { + const original = { + fields: [ + createMockFieldMetadata({ + id: '1', + name: 'Removed Field', + isActive: true, + }), + ], + } as any; + const standard = { fields: [] } as any; + + const result = comparator.compare(original, standard); + + expect(result).toEqual([ + { + action: ComparatorAction.DELETE, + object: expect.objectContaining({ name: 'Removed Field' }), + }, + ]); + }); + + it('should not generate any action for identical fields', () => { + const original = { + fields: [createMockFieldMetadata({ id: '1', isActive: true })], + } as any; + const standard = { + fields: [createMockFieldMetadata({})], + } as any; + + const result = comparator.compare(original, standard); + + expect(result).toHaveLength(0); + }); +}); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-object.comparator.spec.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-object.comparator.spec.ts new file mode 100644 index 0000000000..a92a297cca --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-object.comparator.spec.ts @@ -0,0 +1,77 @@ +import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; + +import { WorkspaceObjectComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator'; + +describe('WorkspaceObjectComparator', () => { + let comparator: WorkspaceObjectComparator; + + beforeEach(() => { + // Initialize the comparator before each test + comparator = new WorkspaceObjectComparator(); + }); + + function createMockObjectMetadata(values: any) { + return { + nameSingular: 'TestObject', + namePlural: 'TestObjects', + labelSingular: 'Test Object', + labelPlural: 'Test Objects', + ...values, + }; + } + + it('should generate CREATE action for new objects', () => { + const standardObjectMetadata = createMockObjectMetadata({ + description: 'A standard object', + }); + + const result = comparator.compare(undefined, standardObjectMetadata); + + expect(result).toEqual({ + action: ComparatorAction.CREATE, + object: standardObjectMetadata, + }); + }); + + it('should generate UPDATE action for objects with differences', () => { + const originalObjectMetadata = createMockObjectMetadata({ + id: '1', + description: 'Original description', + }); + const standardObjectMetadata = createMockObjectMetadata({ + description: 'Updated description', + }); + + const result = comparator.compare( + originalObjectMetadata, + standardObjectMetadata, + ); + + expect(result).toEqual({ + action: ComparatorAction.UPDATE, + object: expect.objectContaining({ + id: '1', + description: 'Updated description', + }), + }); + }); + + it('should generate SKIP action for identical objects', () => { + const originalObjectMetadata = createMockObjectMetadata({ + id: '1', + description: 'Same description', + }); + const standardObjectMetadata = createMockObjectMetadata({ + description: 'Same description', + }); + + const result = comparator.compare( + originalObjectMetadata, + standardObjectMetadata, + ); + + expect(result).toEqual({ + action: ComparatorAction.SKIP, + }); + }); +}); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-relation.comparator.spec.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-relation.comparator.spec.ts new file mode 100644 index 0000000000..09b602617d --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-relation.comparator.spec.ts @@ -0,0 +1,60 @@ +import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; + +import { WorkspaceRelationComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator'; + +describe('WorkspaceRelationComparator', () => { + let comparator: WorkspaceRelationComparator; + + beforeEach(() => { + comparator = new WorkspaceRelationComparator(); + }); + + function createMockRelationMetadata(values: any) { + return { + fromObjectMetadataId: 'object-1', + fromFieldMetadataId: 'field-1', + ...values, + }; + } + + it('should generate CREATE action for new relations', () => { + const original = []; + const standard = [createMockRelationMetadata({})]; + + const result = comparator.compare(original, standard); + + expect(result).toEqual([ + { + action: ComparatorAction.CREATE, + object: expect.objectContaining({ + fromObjectMetadataId: 'object-1', + fromFieldMetadataId: 'field-1', + }), + }, + ]); + }); + + it('should generate DELETE action for removed relations', () => { + const original = [createMockRelationMetadata({ id: '1' })]; + const standard = []; + + const result = comparator.compare(original, standard); + + expect(result).toEqual([ + { + action: ComparatorAction.DELETE, + object: expect.objectContaining({ id: '1' }), + }, + ]); + }); + + it('should not generate any action for identical relations', () => { + const relation = createMockRelationMetadata({}); + const original = [{ id: '1', ...relation }]; + const standard = [relation]; + + const result = comparator.compare(original, standard); + + expect(result).toHaveLength(0); + }); +}); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/index.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/index.ts new file mode 100644 index 0000000000..3c95f7babe --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/index.ts @@ -0,0 +1,9 @@ +import { WorkspaceFieldComparator } from './workspace-field.comparator'; +import { WorkspaceObjectComparator } from './workspace-object.comparator'; +import { WorkspaceRelationComparator } from './workspace-relation.comparator'; + +export const workspaceSyncMetadataComparators = [ + WorkspaceFieldComparator, + WorkspaceObjectComparator, + WorkspaceRelationComparator, +]; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/order-object-properties.util.spec.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/order-object-properties.util.spec.ts new file mode 100644 index 0000000000..60309e8b73 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/order-object-properties.util.spec.ts @@ -0,0 +1,52 @@ +import { orderObjectProperties } from 'src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util'; + +describe('orderObjectProperties', () => { + it('orders simple object properties', () => { + const input = { b: 2, a: 1 }; + const expected = { a: 1, b: 2 }; + + expect(orderObjectProperties(input)).toEqual(expected); + }); + + it('orders nested object properties', () => { + const input = { b: { d: 4, c: 3 }, a: 1 }; + const expected = { a: 1, b: { c: 3, d: 4 } }; + + expect(orderObjectProperties(input)).toEqual(expected); + }); + + it('orders properties in an array of objects', () => { + const input = [ + { b: 2, a: 1 }, + { d: 4, c: 3 }, + ]; + const expected = [ + { a: 1, b: 2 }, + { c: 3, d: 4 }, + ]; + + expect(orderObjectProperties(input)).toEqual(expected); + }); + + it('handles nested arrays within objects', () => { + const input = { b: [{ d: 4, c: 3 }], a: 1 }; + const expected = { a: 1, b: [{ c: 3, d: 4 }] }; + + expect(orderObjectProperties(input)).toEqual(expected); + }); + + it('handles complex nested structures', () => { + const input = { + c: 3, + a: { f: [{ j: 10, i: 9 }, 8], e: 5 }, + b: [7, { h: 6, g: 4 }], + }; + const expected = { + a: { e: 5, f: [{ i: 9, j: 10 }, 8] }, + b: [7, { g: 4, h: 6 }], + c: 3, + }; + + expect(orderObjectProperties(input)).toEqual(expected); + }); +}); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/transform-metadata-for-comparison.util.spec.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/transform-metadata-for-comparison.util.spec.ts new file mode 100644 index 0000000000..eac21ff46f --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/transform-metadata-for-comparison.util.spec.ts @@ -0,0 +1,59 @@ +import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; // Adjust the import path as necessary + +describe('transformMetadataForComparison', () => { + // Test for a single object + it('transforms a single object correctly with nested objects', () => { + const input = { name: 'Test', details: { a: 1, nested: { b: 2 } } }; + const result = transformMetadataForComparison(input, { + propertiesToStringify: ['details'], + }); + + expect(result).toEqual({ + name: 'Test', + details: '{"a":1,"nested":{"b":2}}', + }); + }); + + // Test for an array of objects + it('transforms an array of objects correctly, ignoring and stringifying multiple properties', () => { + const input = [ + { name: 'Test1', value: { a: 1 }, ignored: 'ignoreMe' }, + { name: 'Test2', value: { c: 3 }, extra: 'keepMe' }, + ]; + const result = transformMetadataForComparison(input, { + propertiesToIgnore: ['ignored'], + propertiesToStringify: ['value'], + keyFactory: (datum) => datum.name, + }); + + expect(result).toEqual({ + Test1: { name: 'Test1', value: '{"a":1}' }, + Test2: { name: 'Test2', value: '{"c":3}', extra: 'keepMe' }, + }); + }); + + // Test with a custom keyFactory function + it('uses a custom keyFactory function to generate keys', () => { + const input = [{ id: 123, name: 'Test' }]; + const result = transformMetadataForComparison(input, { + keyFactory: (datum) => `key-${datum.id}`, + }); + + expect(result).toHaveProperty('key-123'); + expect(result['key-123']).toEqual({ id: 123, name: 'Test' }); + }); + + // Test with an empty array + it('handles an empty array gracefully', () => { + const result = transformMetadataForComparison([], {}); + + expect(result).toEqual({}); + }); + + // Test with an empty object + it('handles an empty object gracefully', () => { + const result = transformMetadataForComparison({}, {}); + + expect(result).toEqual({}); + }); +}); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util.ts new file mode 100644 index 0000000000..1ac4acdde9 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util.ts @@ -0,0 +1,21 @@ +export function orderObjectProperties(data: T[]): T[]; + +export function orderObjectProperties(data: T): T; + +export function orderObjectProperties | object>( + data: T, +): T { + if (Array.isArray(data)) { + return data.map(orderObjectProperties) as T; + } + + if (data !== null && typeof data === 'object') { + return Object.fromEntries( + Object.entries(data) + .sort() + .map(([key, value]) => [key, orderObjectProperties(value)]), + ) as T; + } + + return data; +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util.ts new file mode 100644 index 0000000000..963b568125 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util.ts @@ -0,0 +1,82 @@ +import { orderObjectProperties } from './order-object-properties.util'; + +type TransformToString = { + [P in keyof T]: P extends Keys ? string : T[P]; +}; + +// Overload for an array of T +export function transformMetadataForComparison( + fieldMetadataCollection: T[], + options: { + propertiesToIgnore?: readonly Keys[]; + propertiesToStringify?: readonly Keys[]; + keyFactory: (datum: T) => string; + }, +): Record>; + +// Overload for a single T object +export function transformMetadataForComparison( + fieldMetadataCollection: T, + options: { + propertiesToIgnore?: readonly Keys[]; + propertiesToStringify?: readonly Keys[]; + }, +): TransformToString; + +export function transformMetadataForComparison( + metadata: T[] | T, + options: { + propertiesToIgnore?: readonly Keys[]; + propertiesToStringify?: readonly Keys[]; + keyFactory?: (datum: T) => string; + }, +): Record> | TransformToString { + const propertiesToIgnore = (options.propertiesToIgnore ?? + []) as readonly string[]; + const propertiesToStringify = (options.propertiesToStringify ?? + []) as readonly string[]; + + const transformProperties = (datum: T): TransformToString => { + const transformedField = {} as TransformToString; + + for (const property in datum) { + if (propertiesToIgnore.includes(property)) { + continue; + } + if ( + propertiesToStringify.includes(property) && + datum[property] !== null && + typeof datum[property] === 'object' + ) { + const orderedValue = orderObjectProperties(datum[property] as object); + + transformedField[property as string] = JSON.stringify( + orderedValue, + ) as T[Keys]; + } else { + transformedField[property as string] = datum[property]; + } + } + + return transformedField; + }; + + if (Array.isArray(metadata)) { + return metadata.reduce>>( + (acc, datum) => { + const key = options.keyFactory?.(datum); + + if (!key) { + throw new Error('keyFactory must be implemented'); + } + + acc[key] = transformProperties(datum); + + return acc; + }, + {}, + ); + } else { + return transformProperties(metadata); + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator.ts new file mode 100644 index 0000000000..8febe3a6d4 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator.ts @@ -0,0 +1,166 @@ +import { Injectable } from '@nestjs/common'; + +import diff from 'microdiff'; + +import { + ComparatorAction, + FieldComparatorResult, +} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; +import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; +import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; + +const fieldPropertiesToIgnore = [ + 'id', + 'createdAt', + 'updatedAt', + 'objectMetadataId', + 'isActive', +] as const; + +const fieldPropertiesToStringify = [ + 'targetColumnMap', + 'defaultValue', + 'options', +] as const; + +@Injectable() +export class WorkspaceFieldComparator { + constructor() {} + + public compare( + originalObjectMetadata: ObjectMetadataEntity, + standardObjectMetadata: PartialObjectMetadata, + ): FieldComparatorResult[] { + const result: FieldComparatorResult[] = []; + const fieldPropertiesToUpdateMap: Record< + string, + Partial + > = {}; + const originalFieldMetadataMap = transformMetadataForComparison( + originalObjectMetadata.fields, + { + propertiesToIgnore: fieldPropertiesToIgnore, + propertiesToStringify: fieldPropertiesToStringify, + keyFactory(datum) { + return datum.name; + }, + }, + ); + const standardFieldMetadataMap = transformMetadataForComparison( + standardObjectMetadata.fields, + { + propertiesToIgnore: fieldPropertiesToStringify, + keyFactory(datum) { + return datum.name; + }, + }, + ); + + // Compare fields + const fieldMetadataDifference = diff( + originalFieldMetadataMap, + standardFieldMetadataMap, + ); + + for (const difference of fieldMetadataDifference) { + const fieldName = difference.path[0]; + // Object shouldn't have thousands of fields, so we can use find here + const standardFieldMetadata = standardObjectMetadata.fields.find( + (field) => field.name === fieldName, + ); + const originalFieldMetadata = originalObjectMetadata.fields.find( + (field) => field.name === fieldName, + ); + + switch (difference.type) { + case 'CREATE': { + if (!standardFieldMetadata) { + throw new Error( + `Field ${fieldName} not found in standardObjectMetadata`, + ); + } + + result.push({ + action: ComparatorAction.CREATE, + object: { + ...standardFieldMetadata, + objectMetadataId: originalObjectMetadata.id, + }, + }); + break; + } + case 'CHANGE': { + if (!originalFieldMetadata) { + throw new Error( + `Field ${fieldName} not found in originalObjectMetadata`, + ); + } + + const id = originalFieldMetadata.id; + const property = difference.path[difference.path.length - 1]; + + // If the old value and the new value are both null, skip + // Database is storing null, and we can get undefined here + if ( + difference.oldValue === null && + (difference.value === null || difference.value === undefined) + ) { + break; + } + + if (typeof property !== 'string') { + break; + } + + if (!fieldPropertiesToUpdateMap[id]) { + fieldPropertiesToUpdateMap[id] = {}; + } + + // If the property is a stringified JSON, parse it + if ( + (fieldPropertiesToStringify as readonly string[]).includes(property) + ) { + fieldPropertiesToUpdateMap[id][property] = JSON.parse( + difference.value, + ); + } else { + fieldPropertiesToUpdateMap[id][property] = difference.value; + } + break; + } + case 'REMOVE': { + if (!originalFieldMetadata) { + throw new Error( + `Field ${fieldName} not found in originalObjectMetadata`, + ); + } + + if (difference.path.length === 1) { + result.push({ + action: ComparatorAction.DELETE, + object: originalFieldMetadata, + }); + } + break; + } + } + } + + for (const [id, fieldPropertiesToUpdate] of Object.entries( + fieldPropertiesToUpdateMap, + )) { + result.push({ + action: ComparatorAction.UPDATE, + object: { + id, + ...fieldPropertiesToUpdate, + }, + }); + } + + return result; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator.ts new file mode 100644 index 0000000000..bcbd562703 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator.ts @@ -0,0 +1,83 @@ +import { Injectable } from '@nestjs/common'; + +import diff from 'microdiff'; +import omit from 'lodash.omit'; + +import { + ComparatorAction, + ObjectComparatorResult, +} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; +import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; + +import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; + +const objectPropertiesToIgnore = [ + 'id', + 'createdAt', + 'updatedAt', + 'labelIdentifierFieldMetadataId', + 'imageIdentifierFieldMetadataId', + 'isActive', + 'fields', +] as const; + +@Injectable() +export class WorkspaceObjectComparator { + constructor() {} + + public compare( + originalObjectMetadata: ObjectMetadataEntity | undefined, + standardObjectMetadata: PartialObjectMetadata, + ): ObjectComparatorResult { + // If the object doesn't exist in the original metadata, we need to create it + if (!originalObjectMetadata) { + return { + action: ComparatorAction.CREATE, + object: standardObjectMetadata, + }; + } + + const objectPropertiesToUpdate: Partial = {}; + + // Only compare properties that are not ignored + const partialOriginalObjectMetadata = transformMetadataForComparison( + originalObjectMetadata, + { + propertiesToIgnore: objectPropertiesToIgnore, + }, + ); + + // Compare objects + const objectMetadataDifference = diff( + partialOriginalObjectMetadata, + omit(standardObjectMetadata, 'fields'), + ); + + // Loop through the differences and create an object with the properties to update + for (const difference of objectMetadataDifference) { + // We only handle CHANGE here as REMOVE and CREATE are handled earlier. + if (difference.type === 'CHANGE') { + const property = difference.path[0]; + + objectPropertiesToUpdate[property] = difference.value; + } + } + + // If there are no properties to update, the objects are equal + if (Object.keys(objectPropertiesToUpdate).length === 0) { + return { + action: ComparatorAction.SKIP, + }; + } + + // If there are properties to update, we need to update the object + return { + action: ComparatorAction.UPDATE, + object: { + id: originalObjectMetadata.id, + ...objectPropertiesToUpdate, + }, + }; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator.ts new file mode 100644 index 0000000000..727a8c5f51 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator.ts @@ -0,0 +1,72 @@ +import { Injectable } from '@nestjs/common'; + +import diff from 'microdiff'; + +import { + ComparatorAction, + RelationComparatorResult, +} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; + +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; +import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; + +const relationPropertiesToIgnore = ['createdAt', 'updatedAt'] as const; + +@Injectable() +export class WorkspaceRelationComparator { + constructor() {} + + compare( + originalRelationMetadataCollection: RelationMetadataEntity[], + standardRelationMetadataCollection: Partial[], + ): RelationComparatorResult[] { + const results: RelationComparatorResult[] = []; + + // Create a map of standard relations + const standardRelationMetadataMap = transformMetadataForComparison( + standardRelationMetadataCollection, + { + keyFactory(relationMetadata) { + return `${relationMetadata.fromObjectMetadataId}->${relationMetadata.fromFieldMetadataId}`; + }, + }, + ); + + // Create a filtered map of original relations + // We filter out 'id' later because we need it to remove the relation from DB + const originalRelationMetadataMap = transformMetadataForComparison( + originalRelationMetadataCollection, + { + propertiesToIgnore: relationPropertiesToIgnore, + keyFactory(relationMetadata) { + return `${relationMetadata.fromObjectMetadataId}->${relationMetadata.fromFieldMetadataId}`; + }, + }, + ); + + // Compare relations + const relationMetadataDifference = diff( + originalRelationMetadataMap, + standardRelationMetadataMap, + ); + + for (const difference of relationMetadataDifference) { + if (difference.type === 'CREATE') { + results.push({ + action: ComparatorAction.CREATE, + object: difference.value, + }); + } else if ( + difference.type === 'REMOVE' && + difference.path[difference.path.length - 1] !== 'id' + ) { + results.push({ + action: ComparatorAction.DELETE, + object: difference.oldValue, + }); + } + } + + return results; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/field-metadata.decorator.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/field-metadata.decorator.ts index a2fc29f606..351d428a50 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/field-metadata.decorator.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/field-metadata.decorator.ts @@ -3,6 +3,7 @@ import { ReflectFieldMetadata, } from 'src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface'; import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface'; +import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface'; import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util'; @@ -14,7 +15,7 @@ export function FieldMetadata( ): PropertyDecorator { return (target: object, fieldKey: string) => { const existingFieldMetadata = - TypedReflect.getMetadata('fieldMetadata', target.constructor) ?? {}; + TypedReflect.getMetadata('fieldMetadataMap', target.constructor) ?? {}; const isNullable = TypedReflect.getMetadata('isNullable', target, fieldKey) ?? false; const isSystem = @@ -23,7 +24,7 @@ export function FieldMetadata( const { joinColumn, ...restParams } = params; TypedReflect.defineMetadata( - 'fieldMetadata', + 'fieldMetadataMap', { ...existingFieldMetadata, [fieldKey]: generateFieldMetadata( @@ -65,19 +66,22 @@ function generateFieldMetadata( gate: GateDecoratorParams | undefined = undefined, ): ReflectFieldMetadata[string] { const targetColumnMap = generateTargetColumnMap(params.type, false, fieldKey); - const defaultValue = params.defaultValue ?? generateDefaultValue(params.type); + const defaultValue = (params.defaultValue ?? + generateDefaultValue( + params.type, + )) as FieldMetadataDefaultValue<'default'> | null; return { name: fieldKey, ...params, - targetColumnMap: JSON.stringify(targetColumnMap), + targetColumnMap, isNullable: params.type === FieldMetadataType.RELATION ? true : isNullable, isSystem, isCustom: false, - options: params.options ? JSON.stringify(params.options) : null, + options: params.options, description: params.description, icon: params.icon, - defaultValue: defaultValue ? JSON.stringify(defaultValue) : null, + defaultValue, gate, }; } diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/relation-metadata.decorator.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/relation-metadata.decorator.ts index 142061671c..62c5b11a25 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/relation-metadata.decorator.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/relation-metadata.decorator.ts @@ -9,17 +9,20 @@ export function RelationMetadata( params: RelationMetadataDecoratorParams, ): PropertyDecorator { return (target: object, fieldKey: string) => { - const existingRelationMetadata = - TypedReflect.getMetadata('relationMetadata', target.constructor) ?? []; + const relationMetadataCollection = + TypedReflect.getMetadata( + 'relationMetadataCollection', + target.constructor, + ) ?? []; const gate = TypedReflect.getMetadata('gate', target, fieldKey); const objectName = convertClassNameToObjectMetadataName( target.constructor.name, ); Reflect.defineMetadata( - 'relationMetadata', + 'relationMetadataCollection', [ - ...existingRelationMetadata, + ...relationMetadataCollection, { type: params.type, fromObjectNameSingular: objectName, diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/feature-flags.factory.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/feature-flags.factory.ts new file mode 100644 index 0000000000..82904e5d8f --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/feature-flags.factory.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface'; + +import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; + +@Injectable() +export class FeatureFlagFactory { + constructor( + @InjectRepository(FeatureFlagEntity, 'core') + private readonly featureFlagRepository: Repository, + ) {} + + async create(context: WorkspaceSyncContext): Promise { + const workspaceFeatureFlags = await this.featureFlagRepository.find({ + where: { workspaceId: context.workspaceId }, + }); + + const workspaceFeatureFlagsMap = workspaceFeatureFlags.reduce( + (result, currentFeatureFlag) => { + result[currentFeatureFlag.key] = currentFeatureFlag.value; + + return result; + }, + {} as FeatureFlagMap, + ); + + return workspaceFeatureFlagsMap; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/index.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/index.ts new file mode 100644 index 0000000000..df6a4bfdb9 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/index.ts @@ -0,0 +1,11 @@ +import { FeatureFlagFactory } from './feature-flags.factory'; +import { StandardObjectFactory } from './standard-object.factory'; +import { StandardRelationFactory } from './standard-relation.factory'; +import { WorkspaceSyncFactory } from './workspace-sync.factory'; + +export const workspaceSyncMetadataFactories = [ + FeatureFlagFactory, + StandardObjectFactory, + StandardRelationFactory, + WorkspaceSyncFactory, +]; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-object.factory.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-object.factory.ts new file mode 100644 index 0000000000..fd76c4e06a --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-object.factory.ts @@ -0,0 +1,70 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; +import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface'; +import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; + +import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata'; +import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects'; +import { TypedReflect } from 'src/utils/typed-reflect'; +import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util'; + +@Injectable() +export class StandardObjectFactory { + create( + context: WorkspaceSyncContext, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): PartialObjectMetadata[] { + return standardObjectMetadata + .map((metadata) => + this.createObjectMetadata(metadata, context, workspaceFeatureFlagsMap), + ) + .filter((metadata): metadata is PartialObjectMetadata => !!metadata); + } + + private createObjectMetadata( + metadata: typeof BaseObjectMetadata, + context: WorkspaceSyncContext, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): PartialObjectMetadata | undefined { + const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata); + const fieldMetadataMap = + TypedReflect.getMetadata('fieldMetadataMap', metadata) ?? []; + + if (!objectMetadata) { + throw new Error( + `Object metadata decorator not found, can\'t parse ${metadata.name}`, + ); + } + + if (isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap)) { + return undefined; + } + + const fields = Object.values(fieldMetadataMap).reduce( + // Omit gate as we don't want to store it in the DB + (acc, { gate, ...fieldMetadata }) => { + if (isGatedAndNotEnabled(gate, workspaceFeatureFlagsMap)) { + return acc; + } + + acc.push({ + ...fieldMetadata, + workspaceId: context.workspaceId, + isSystem: objectMetadata.isSystem || fieldMetadata.isSystem, + }); + + return acc; + }, + [] as PartialFieldMetadata[], + ); + + return { + ...objectMetadata, + workspaceId: context.workspaceId, + dataSourceId: context.dataSourceId, + fields, + }; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-relation.factory.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-relation.factory.ts new file mode 100644 index 0000000000..8d98da4bcb --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-relation.factory.ts @@ -0,0 +1,116 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface'; + +import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata'; +import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects'; +import { TypedReflect } from 'src/utils/typed-reflect'; +import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util'; +import { assert } from 'src/utils/assert'; +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; + +@Injectable() +export class StandardRelationFactory { + create( + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + return standardObjectMetadata.flatMap((standardObjectMetadata) => + this.createRelationMetadata( + standardObjectMetadata, + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ), + ); + } + + private createRelationMetadata( + standardObjectMetadata: typeof BaseObjectMetadata, + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + const objectMetadata = TypedReflect.getMetadata( + 'objectMetadata', + standardObjectMetadata, + ); + const relationMetadataCollection = TypedReflect.getMetadata( + 'relationMetadataCollection', + standardObjectMetadata, + ); + + if (!objectMetadata) { + throw new Error( + `Object metadata decorator not found, can\'t parse ${standardObjectMetadata.name}`, + ); + } + + if ( + !relationMetadataCollection || + isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap) + ) { + return []; + } + + return relationMetadataCollection + .filter( + (relationMetadata) => + !isGatedAndNotEnabled( + relationMetadata.gate, + workspaceFeatureFlagsMap, + ), + ) + .map((relationMetadata) => { + const fromObjectMetadata = + originalObjectMetadataMap[relationMetadata.fromObjectNameSingular]; + + assert( + fromObjectMetadata, + `Object ${relationMetadata.fromObjectNameSingular} not found in DB + for relation FROM defined in class ${objectMetadata.nameSingular}`, + ); + + const toObjectMetadata = + originalObjectMetadataMap[relationMetadata.toObjectNameSingular]; + + assert( + toObjectMetadata, + `Object ${relationMetadata.toObjectNameSingular} not found in DB + for relation TO defined in class ${objectMetadata.nameSingular}`, + ); + + const fromFieldMetadata = fromObjectMetadata?.fields.find( + (field) => field.name === relationMetadata.fromFieldMetadataName, + ); + + assert( + fromFieldMetadata, + `Field ${relationMetadata.fromFieldMetadataName} not found in object ${relationMetadata.fromObjectNameSingular} + for relation FROM defined in class ${objectMetadata.nameSingular}`, + ); + + const toFieldMetadata = toObjectMetadata?.fields.find( + (field) => field.name === relationMetadata.toFieldMetadataName, + ); + + assert( + toFieldMetadata, + `Field ${relationMetadata.toFieldMetadataName} not found in object ${relationMetadata.toObjectNameSingular} + for relation TO defined in class ${objectMetadata.nameSingular}`, + ); + + return { + relationType: relationMetadata.type, + fromObjectMetadataId: fromObjectMetadata?.id, + toObjectMetadataId: toObjectMetadata?.id, + fromFieldMetadataId: fromFieldMetadata?.id, + toFieldMetadataId: toFieldMetadata?.id, + workspaceId: context.workspaceId, + }; + }); + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/workspace-sync.factory.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/workspace-sync.factory.ts new file mode 100644 index 0000000000..3c893571a1 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/factories/workspace-sync.factory.ts @@ -0,0 +1,264 @@ +import { Injectable } from '@nestjs/common'; + +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/metadata/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { + WorkspaceMigrationColumnActionType, + WorkspaceMigrationColumnRelation, + WorkspaceMigrationEntity, + WorkspaceMigrationTableAction, +} from 'src/metadata/workspace-migration/workspace-migration.entity'; +import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; +import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory'; +import { + RelationMetadataEntity, + RelationMetadataType, +} from 'src/metadata/relation-metadata/relation-metadata.entity'; +import { camelCase } from 'src/utils/camel-case'; +import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util'; + +@Injectable() +export class WorkspaceSyncFactory { + constructor( + private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, + ) {} + + async createObjectMigration( + originalObjectMetadataCollection: ObjectMetadataEntity[], + createdObjectMetadataCollection: ObjectMetadataEntity[], + objectMetadataDeleteCollection: ObjectMetadataEntity[], + createdFieldMetadataCollection: FieldMetadataEntity[], + fieldMetadataDeleteCollection: FieldMetadataEntity[], + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + /** + * Create object migrations + */ + if (createdObjectMetadataCollection.length > 0) { + for (const objectMetadata of createdObjectMetadataCollection) { + const migrations = [ + { + name: computeObjectTargetTable(objectMetadata), + action: 'create', + } satisfies WorkspaceMigrationTableAction, + ...objectMetadata.fields + .filter((field) => field.type !== FieldMetadataType.RELATION) + .map( + (field) => + ({ + name: computeObjectTargetTable(objectMetadata), + action: 'alter', + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.CREATE, + field, + ), + }) satisfies WorkspaceMigrationTableAction, + ), + ]; + + workspaceMigrations.push({ + workspaceId: objectMetadata.workspaceId, + name: generateMigrationName(`create-${objectMetadata.nameSingular}`), + isCustom: false, + migrations, + }); + } + } + + /** + * Delete object migrations + * TODO: handle object delete migrations. + * Note: we need to delete the relation first due to the DB constraint. + */ + // if (objectMetadataDeleteCollection.length > 0) { + // for (const objectMetadata of objectMetadataDeleteCollection) { + // const migrations = [ + // { + // name: computeObjectTargetTable(objectMetadata), + // action: 'drop', + // columns: [], + // } satisfies WorkspaceMigrationTableAction, + // ]; + + // workspaceMigrations.push({ + // workspaceId: objectMetadata.workspaceId, + // isCustom: false, + // migrations, + // }); + // } + // } + + /** + * Create field migrations + */ + const originalObjectMetadataMap = originalObjectMetadataCollection.reduce( + (result, currentObject) => { + result[currentObject.id] = currentObject; + + return result; + }, + {} as Record, + ); + + if (createdFieldMetadataCollection.length > 0) { + for (const fieldMetadata of createdFieldMetadataCollection) { + const migrations = [ + { + name: computeObjectTargetTable( + originalObjectMetadataMap[fieldMetadata.objectMetadataId], + ), + action: 'alter', + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.CREATE, + fieldMetadata, + ), + } satisfies WorkspaceMigrationTableAction, + ]; + + workspaceMigrations.push({ + workspaceId: fieldMetadata.workspaceId, + name: generateMigrationName(`create-${fieldMetadata.name}`), + isCustom: false, + migrations, + }); + } + } + + /** + * Delete field migrations + */ + if (fieldMetadataDeleteCollection.length > 0) { + for (const fieldMetadata of fieldMetadataDeleteCollection) { + const migrations = [ + { + name: computeObjectTargetTable( + originalObjectMetadataMap[fieldMetadata.objectMetadataId], + ), + action: 'alter', + columns: [ + { + action: WorkspaceMigrationColumnActionType.DROP, + columnName: fieldMetadata.name, + }, + ], + } satisfies WorkspaceMigrationTableAction, + ]; + + workspaceMigrations.push({ + workspaceId: fieldMetadata.workspaceId, + name: generateMigrationName(`delete-${fieldMetadata.name}`), + isCustom: false, + migrations, + }); + } + } + + return workspaceMigrations; + } + + async createRelationMigration( + originalObjectMetadataCollection: ObjectMetadataEntity[], + createdRelationMetadataCollection: RelationMetadataEntity[], + // TODO: handle relation deletion + // eslint-disable-next-line @typescript-eslint/no-unused-vars + relationMetadataDeleteCollection: RelationMetadataEntity[], + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + if (createdRelationMetadataCollection.length > 0) { + for (const relationMetadata of createdRelationMetadataCollection) { + const toObjectMetadata = originalObjectMetadataCollection.find( + (object) => object.id === relationMetadata.toObjectMetadataId, + ); + + const fromObjectMetadata = originalObjectMetadataCollection.find( + (object) => object.id === relationMetadata.fromObjectMetadataId, + ); + + if (!toObjectMetadata) { + throw new Error( + `ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`, + ); + } + + if (!fromObjectMetadata) { + throw new Error( + `ObjectMetadata with id ${relationMetadata.fromObjectMetadataId} not found`, + ); + } + + const toFieldMetadata = toObjectMetadata.fields.find( + (field) => field.id === relationMetadata.toFieldMetadataId, + ); + + if (!toFieldMetadata) { + throw new Error( + `FieldMetadata with id ${relationMetadata.toFieldMetadataId} not found`, + ); + } + + const migrations = [ + { + name: computeObjectTargetTable(toObjectMetadata), + action: 'alter', + columns: [ + { + action: WorkspaceMigrationColumnActionType.RELATION, + columnName: `${camelCase(toFieldMetadata.name)}Id`, + referencedTableName: + computeObjectTargetTable(fromObjectMetadata), + referencedTableColumnName: 'id', + isUnique: + relationMetadata.relationType === + RelationMetadataType.ONE_TO_ONE, + } satisfies WorkspaceMigrationColumnRelation, + ], + } satisfies WorkspaceMigrationTableAction, + ]; + + workspaceMigrations.push({ + workspaceId: relationMetadata.workspaceId, + name: generateMigrationName( + `create-relation-from-${fromObjectMetadata.nameSingular}-to-${toObjectMetadata.nameSingular}`, + ), + isCustom: false, + migrations, + }); + } + } + + // if (relationMetadataDeleteCollection.length > 0) { + // for (const relationMetadata of relationMetadataDeleteCollection) { + // const toObjectMetadata = originalObjectMetadataCollection.find( + // (object) => object.id === relationMetadata.toObjectMetadataId, + // ); + + // if (!toObjectMetadata) { + // throw new Error( + // `ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`, + // ); + // } + + // const migrations = [ + // { + // name: computeObjectTargetTable(toObjectMetadata), + // action: 'drop', + // columns: [], + // } satisfies WorkspaceMigrationTableAction, + // ]; + + // workspaceMigrations.push({ + // workspaceId: relationMetadata.workspaceId, + // isCustom: false, + // migrations, + // }); + // } + // } + + return workspaceMigrations; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/comparator.interface.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/comparator.interface.ts new file mode 100644 index 0000000000..a90201192b --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/comparator.interface.ts @@ -0,0 +1,46 @@ +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; + +import { PartialFieldMetadata } from './partial-field-metadata.interface'; +import { PartialObjectMetadata } from './partial-object-metadata.interface'; + +export const enum ComparatorAction { + SKIP = 'SKIP', + CREATE = 'CREATE', + UPDATE = 'UPDATE', + DELETE = 'DELETE', +} + +export interface ComparatorSkipResult { + action: ComparatorAction.SKIP; +} + +export interface ComparatorCreateResult { + action: ComparatorAction.CREATE; + object: T; +} + +export interface ComparatorUpdateResult { + action: ComparatorAction.UPDATE; + object: T; +} + +export interface ComparatorDeleteResult { + action: ComparatorAction.DELETE; + object: T; +} + +export type ObjectComparatorResult = + | ComparatorSkipResult + | ComparatorCreateResult + | ComparatorUpdateResult>; + +export type FieldComparatorResult = + | ComparatorSkipResult + | ComparatorCreateResult + | ComparatorUpdateResult & { id: string }> + | ComparatorDeleteResult; + +export type RelationComparatorResult = + | ComparatorCreateResult> + | ComparatorDeleteResult; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface.ts index 0e12d1bcd5..1080b34cf7 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface.ts @@ -2,6 +2,7 @@ import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/inte import { ReflectObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/reflect-object-metadata.interface'; export type PartialObjectMetadata = ReflectObjectMetadata & { + id?: string; workspaceId: string; dataSourceId: string; fields: PartialFieldMetadata[]; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface.ts index 6d4c66ca21..873ed5b75e 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface.ts @@ -1,6 +1,7 @@ import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface'; import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface'; import { FieldMetadataOptions } from 'src/metadata/field-metadata/interfaces/field-metadata-options.interface'; +import { FieldMetadataTargetColumnMap } from 'src/metadata/field-metadata/interfaces/field-metadata-target-column-map.interface'; import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; @@ -23,13 +24,13 @@ export interface ReflectFieldMetadata { > & { name: string; type: FieldMetadataType; - targetColumnMap: string; + targetColumnMap: FieldMetadataTargetColumnMap<'default'>; isNullable: boolean; isSystem: boolean; isCustom: boolean; description?: string; - defaultValue: string | null; + defaultValue: FieldMetadataDefaultValue<'default'> | null; gate?: GateDecoratorParams; - options?: string | null; + options?: FieldMetadataOptions<'default'> | null; }; } diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface.ts new file mode 100644 index 0000000000..fc0c7b5c55 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface.ts @@ -0,0 +1,4 @@ +export interface WorkspaceSyncContext { + workspaceId: string; + dataSourceId: string; +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts deleted file mode 100644 index ab10138f30..0000000000 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import assert from 'assert'; - -import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; -import { MappedObjectMetadataEntity } from 'src/workspace/workspace-sync-metadata/interfaces/mapped-metadata.interface'; - -import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata'; -import { TypedReflect } from 'src/utils/typed-reflect'; -import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util'; - -@Injectable() -export class ReflectiveMetadataFactory { - async createObjectMetadata( - metadata: typeof BaseObjectMetadata, - workspaceId: string, - defaultDataSourceId: string, - workspaceFeatureFlagsMap: Record, - ): Promise { - const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata); - const fieldMetadata = - TypedReflect.getMetadata('fieldMetadata', metadata) ?? {}; - - if (!objectMetadata) { - throw new Error( - `Object metadata decorator not found, can\'t parse ${metadata.name}`, - ); - } - - if (isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap)) { - return undefined; - } - - const fields = Object.values(fieldMetadata).filter( - (field) => !isGatedAndNotEnabled(field, workspaceFeatureFlagsMap), - ); - - return { - ...objectMetadata, - workspaceId, - dataSourceId: defaultDataSourceId, - fields: fields.map((field) => ({ - ...field, - workspaceId, - isSystem: objectMetadata.isSystem || field.isSystem, - })), - }; - } - - async createObjectMetadataCollection( - metadataCollection: (typeof BaseObjectMetadata)[], - workspaceId: string, - dataSourceId: string, - workspaceFeatureFlagsMap: Record, - ) { - const metadataPromises = metadataCollection.map((metadata) => - this.createObjectMetadata( - metadata, - workspaceId, - dataSourceId, - workspaceFeatureFlagsMap, - ), - ); - const resolvedMetadata = await Promise.all(metadataPromises); - - return resolvedMetadata.filter( - (metadata): metadata is PartialObjectMetadata => !!metadata, - ); - } - - createRelationMetadata( - metadata: typeof BaseObjectMetadata, - workspaceId: string, - objectMetadataFromDB: Record, - workspaceFeatureFlagsMap: Record, - ) { - const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata); - const relationMetadata = TypedReflect.getMetadata( - 'relationMetadata', - metadata, - ); - - if (!objectMetadata) { - throw new Error( - `Object metadata decorator not found, can\'t parse ${metadata.name}`, - ); - } - - if ( - !relationMetadata || - isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap) - ) { - return []; - } - - return relationMetadata - .filter( - (relation) => !isGatedAndNotEnabled(relation, workspaceFeatureFlagsMap), - ) - .map((relation) => { - const fromObjectMetadata = - objectMetadataFromDB[relation.fromObjectNameSingular]; - - assert( - fromObjectMetadata, - `Object ${relation.fromObjectNameSingular} not found in DB - for relation FROM defined in class ${objectMetadata.nameSingular}`, - ); - - const toObjectMetadata = - objectMetadataFromDB[relation.toObjectNameSingular]; - - assert( - toObjectMetadata, - `Object ${relation.toObjectNameSingular} not found in DB - for relation TO defined in class ${objectMetadata.nameSingular}`, - ); - - const fromFieldMetadata = - fromObjectMetadata?.fields[relation.fromFieldMetadataName]; - - assert( - fromFieldMetadata, - `Field ${relation.fromFieldMetadataName} not found in object ${relation.fromObjectNameSingular} - for relation FROM defined in class ${objectMetadata.nameSingular}`, - ); - - const toFieldMetadata = - toObjectMetadata?.fields[relation.toFieldMetadataName]; - - assert( - toFieldMetadata, - `Field ${relation.toFieldMetadataName} not found in object ${relation.toObjectNameSingular} - for relation TO defined in class ${objectMetadata.nameSingular}`, - ); - - return { - relationType: relation.type, - fromObjectMetadataId: fromObjectMetadata?.id, - toObjectMetadataId: toObjectMetadata?.id, - fromFieldMetadataId: fromFieldMetadata?.id, - toFieldMetadataId: toFieldMetadata?.id, - workspaceId, - }; - }); - } - - createRelationMetadataCollection( - metadataCollection: (typeof BaseObjectMetadata)[], - workspaceId: string, - objectMetadataFromDB: Record, - workspaceFeatureFlagsMap: Record, - ) { - return metadataCollection.flatMap((metadata) => - this.createRelationMetadata( - metadata, - workspaceId, - objectMetadataFromDB, - workspaceFeatureFlagsMap, - ), - ); - } -} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service.ts new file mode 100644 index 0000000000..1b8f759a04 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service.ts @@ -0,0 +1,189 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EntityManager, In } from 'typeorm'; +import { v4 as uuidV4 } from 'uuid'; +import omit from 'lodash.omit'; + +import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/metadata/field-metadata/field-metadata.entity'; +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; +import { FieldMetadataComplexOption } from 'src/metadata/field-metadata/dtos/options.input'; +import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage'; + +@Injectable() +export class WorkspaceMetadataUpdaterService { + private readonly logger = new Logger(WorkspaceMetadataUpdaterService.name); + + async updateObjectMetadata( + manager: EntityManager, + storage: WorkspaceSyncStorage, + ): Promise<{ + createdObjectMetadataCollection: ObjectMetadataEntity[]; + updatedObjectMetadataCollection: ObjectMetadataEntity[]; + }> { + const objectMetadataRepository = + manager.getRepository(ObjectMetadataEntity); + + /** + * Create object metadata + */ + const createdPartialObjectMetadataCollection = + await objectMetadataRepository.save( + storage.objectMetadataCreateCollection.map((objectMetadata) => ({ + ...objectMetadata, + isActive: true, + fields: objectMetadata.fields.map((field) => + this.prepareFieldMetadataForCreation(field), + ), + })) as DeepPartial[], + ); + const identifiers = createdPartialObjectMetadataCollection.map( + (object) => object.id, + ); + const createdObjectMetadataCollection = await manager.find( + ObjectMetadataEntity, + { + where: { id: In(identifiers) }, + relations: ['dataSource', 'fields'], + }, + ); + + /** + * Update object metadata + */ + const updatedObjectMetadataCollection = await objectMetadataRepository.save( + storage.objectMetadataUpdateCollection.map((objectMetadata) => + omit(objectMetadata, ['fields']), + ), + ); + + /** + * Delete object metadata + */ + if (storage.objectMetadataDeleteCollection.length > 0) { + await objectMetadataRepository.delete( + storage.objectMetadataDeleteCollection.map((object) => object.id), + ); + } + + return { + createdObjectMetadataCollection, + updatedObjectMetadataCollection, + }; + } + + /** + * TODO: Refactor this + */ + private prepareFieldMetadataForCreation(field: PartialFieldMetadata) { + return { + ...field, + ...(field.type === FieldMetadataType.SELECT && field.options + ? { + options: this.generateUUIDForNewSelectFieldOptions( + field.options as FieldMetadataComplexOption[], + ), + } + : {}), + isActive: true, + }; + } + + private generateUUIDForNewSelectFieldOptions( + options: FieldMetadataComplexOption[], + ): FieldMetadataComplexOption[] { + return options.map((option) => ({ + ...option, + id: uuidV4(), + })); + } + + async updateFieldMetadata( + manager: EntityManager, + storage: WorkspaceSyncStorage, + ): Promise<{ + createdFieldMetadataCollection: FieldMetadataEntity[]; + updatedFieldMetadataCollection: FieldMetadataEntity[]; + }> { + const fieldMetadataRepository = manager.getRepository(FieldMetadataEntity); + + /** + * Create field metadata + */ + const createdFieldMetadataCollection = await fieldMetadataRepository.save( + storage.fieldMetadataCreateCollection.map((field) => + this.prepareFieldMetadataForCreation(field), + ) as DeepPartial[], + ); + + /** + * Update field metadata + */ + const updatedFieldMetadataCollection = await fieldMetadataRepository.save( + storage.fieldMetadataUpdateCollection as DeepPartial[], + ); + + /** + * Delete field metadata + */ + // TODO: handle relation fields deletion. We need to delete the relation metadata first due to the DB constraint. + const fieldMetadataDeleteCollectionWithoutRelationType = + storage.fieldMetadataDeleteCollection.filter( + (field) => field.type !== FieldMetadataType.RELATION, + ); + + if (fieldMetadataDeleteCollectionWithoutRelationType.length > 0) { + await fieldMetadataRepository.delete( + fieldMetadataDeleteCollectionWithoutRelationType.map( + (field) => field.id, + ), + ); + } + + return { + createdFieldMetadataCollection: + createdFieldMetadataCollection as FieldMetadataEntity[], + updatedFieldMetadataCollection: + updatedFieldMetadataCollection as FieldMetadataEntity[], + }; + } + + async updateRelationMetadata( + manager: EntityManager, + storage: WorkspaceSyncStorage, + ): Promise<{ + createdRelationMetadataCollection: RelationMetadataEntity[]; + }> { + const relationMetadataRepository = manager.getRepository( + RelationMetadataEntity, + ); + + /** + * Create relation metadata + */ + const createdRelationMetadataCollection = + await relationMetadataRepository.save( + storage.relationMetadataCreateCollection, + ); + + /** + * Delete relation metadata + */ + if (storage.relationMetadataDeleteCollection.length > 0) { + await relationMetadataRepository.delete( + storage.relationMetadataDeleteCollection.map( + (relationMetadata) => relationMetadata.id, + ), + ); + } + + return { + createdRelationMetadataCollection, + }; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts new file mode 100644 index 0000000000..c268b72e51 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts @@ -0,0 +1,154 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; +import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util'; +import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; +import { StandardObjectFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-object.factory'; +import { WorkspaceObjectComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator'; +import { WorkspaceFieldComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator'; +import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service'; +import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory'; +import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage'; + +@Injectable() +export class WorkspaceSyncObjectMetadataService { + private readonly logger = new Logger(WorkspaceSyncObjectMetadataService.name); + + constructor( + private readonly standardObjectFactory: StandardObjectFactory, + private readonly workspaceObjectComparator: WorkspaceObjectComparator, + private readonly workspaceFieldComparator: WorkspaceFieldComparator, + private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService, + private readonly workspaceSyncFactory: WorkspaceSyncFactory, + ) {} + + async synchronize( + context: WorkspaceSyncContext, + manager: EntityManager, + storage: WorkspaceSyncStorage, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Promise { + const objectMetadataRepository = + manager.getRepository(ObjectMetadataEntity); + const workspaceMigrationRepository = manager.getRepository( + WorkspaceMigrationEntity, + ); + + // Retrieve object metadata collection from DB + const originalObjectMetadataCollection = + await objectMetadataRepository.find({ + where: { workspaceId: context.workspaceId, isCustom: false }, + relations: ['dataSource', 'fields'], + }); + + // Create standard object metadata collection + const standardObjectMetadataCollection = this.standardObjectFactory.create( + context, + workspaceFeatureFlagsMap, + ); + + // Create map of original and standard object metadata by unique identifier + const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier( + originalObjectMetadataCollection, + ); + const standardObjectMetadataMap = mapObjectMetadataByUniqueIdentifier( + standardObjectMetadataCollection, + ); + + this.logger.log('Comparing standard objects and fields metadata'); + + // Store object that need to be deleted + for (const originalObjectMetadata of originalObjectMetadataCollection) { + if (!standardObjectMetadataMap[originalObjectMetadata.nameSingular]) { + storage.addDeleteObjectMetadata(originalObjectMetadata); + } + } + + // Loop over all standard objects and compare them with the objects in DB + for (const standardObjectName in standardObjectMetadataMap) { + const originalObjectMetadata = + originalObjectMetadataMap[standardObjectName]; + const standardObjectMetadata = + standardObjectMetadataMap[standardObjectName]; + + /** + * COMPARE OBJECT METADATA + */ + const objectComparatorResult = this.workspaceObjectComparator.compare( + originalObjectMetadata, + standardObjectMetadata, + ); + + if (objectComparatorResult.action === ComparatorAction.CREATE) { + storage.addCreateObjectMetadata(standardObjectMetadata); + continue; + } + + if (objectComparatorResult.action === ComparatorAction.UPDATE) { + storage.addUpdateObjectMetadata(objectComparatorResult.object); + } + + /** + * COMPARE FIELD METADATA + */ + const fieldComparatorResults = this.workspaceFieldComparator.compare( + originalObjectMetadata, + standardObjectMetadata, + ); + + for (const fieldComparatorResult of fieldComparatorResults) { + switch (fieldComparatorResult.action) { + case ComparatorAction.CREATE: { + storage.addCreateFieldMetadata(fieldComparatorResult.object); + break; + } + case ComparatorAction.UPDATE: { + storage.addUpdateFieldMetadata(fieldComparatorResult.object); + break; + } + case ComparatorAction.DELETE: { + storage.addDeleteFieldMetadata(fieldComparatorResult.object); + break; + } + } + } + } + + this.logger.log('Updating workspace metadata'); + + // Apply changes to DB + const metadataObjectUpdaterResult = + await this.workspaceMetadataUpdaterService.updateObjectMetadata( + manager, + storage, + ); + const metadataFieldUpdaterResult = + await this.workspaceMetadataUpdaterService.updateFieldMetadata( + manager, + storage, + ); + + this.logger.log('Generating migrations'); + + // Create migrations + const workspaceObjectMigrations = + await this.workspaceSyncFactory.createObjectMigration( + originalObjectMetadataCollection, + metadataObjectUpdaterResult.createdObjectMetadataCollection, + storage.objectMetadataDeleteCollection, + metadataFieldUpdaterResult.createdFieldMetadataCollection, + storage.fieldMetadataDeleteCollection, + ); + + this.logger.log('Saving migrations'); + + // Save migrations into DB + await workspaceMigrationRepository.save(workspaceObjectMigrations); + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service.ts new file mode 100644 index 0000000000..d4cd980031 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service.ts @@ -0,0 +1,105 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface'; +import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; +import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util'; +import { StandardRelationFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-relation.factory'; +import { WorkspaceRelationComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator'; +import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service'; +import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory'; +import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; +import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage'; + +@Injectable() +export class WorkspaceSyncRelationMetadataService { + private readonly logger = new Logger( + WorkspaceSyncRelationMetadataService.name, + ); + + constructor( + private readonly standardRelationFactory: StandardRelationFactory, + private readonly workspaceRelationComparator: WorkspaceRelationComparator, + private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService, + private readonly workspaceSyncFactory: WorkspaceSyncFactory, + ) {} + + async synchronize( + context: WorkspaceSyncContext, + manager: EntityManager, + storage: WorkspaceSyncStorage, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Promise { + const objectMetadataRepository = + manager.getRepository(ObjectMetadataEntity); + const workspaceMigrationRepository = manager.getRepository( + WorkspaceMigrationEntity, + ); + + // Retrieve object metadata collection from DB + const originalObjectMetadataCollection = + await objectMetadataRepository.find({ + where: { workspaceId: context.workspaceId, isCustom: false }, + relations: ['dataSource', 'fields'], + }); + + // Create map of object metadata & field metadata by unique identifier + const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier( + originalObjectMetadataCollection, + ); + + const relationMetadataRepository = manager.getRepository( + RelationMetadataEntity, + ); + + // Retrieve relation metadata collection from DB + // TODO: filter out custom relations once isCustom has been added to relationMetadata table + const originalRelationMetadataCollection = + await relationMetadataRepository.find({ + where: { workspaceId: context.workspaceId }, + }); + + // Create standard relation metadata collection + const standardRelationMetadataCollection = + this.standardRelationFactory.create( + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ); + + const relationComparatorResults = this.workspaceRelationComparator.compare( + originalRelationMetadataCollection, + standardRelationMetadataCollection, + ); + + for (const relationComparatorResult of relationComparatorResults) { + if (relationComparatorResult.action === ComparatorAction.CREATE) { + storage.addCreateRelationMetadata(relationComparatorResult.object); + } else if (relationComparatorResult.action === ComparatorAction.DELETE) { + storage.addDeleteRelationMetadata(relationComparatorResult.object); + } + } + + const metadataRelationUpdaterResult = + await this.workspaceMetadataUpdaterService.updateRelationMetadata( + manager, + storage, + ); + + // Create migrations + const workspaceRelationMigrations = + await this.workspaceSyncFactory.createRelationMigration( + originalObjectMetadataCollection, + metadataRelationUpdaterResult.createdRelationMetadataCollection, + storage.relationMetadataDeleteCollection, + ); + + // Save migrations into DB + await workspaceMigrationRepository.save(workspaceRelationMigrations); + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/storage/workspace-sync.storage.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/storage/workspace-sync.storage.ts new file mode 100644 index 0000000000..99791b9ac8 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/storage/workspace-sync.storage.ts @@ -0,0 +1,93 @@ +import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; +import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity'; + +export class WorkspaceSyncStorage { + // Object metadata + private readonly _objectMetadataCreateCollection: PartialObjectMetadata[] = + []; + private readonly _objectMetadataUpdateCollection: Partial[] = + []; + private readonly _objectMetadataDeleteCollection: ObjectMetadataEntity[] = []; + + // Field metadata + private readonly _fieldMetadataCreateCollection: PartialFieldMetadata[] = []; + private readonly _fieldMetadataUpdateCollection: Partial[] = + []; + private readonly _fieldMetadataDeleteCollection: FieldMetadataEntity[] = []; + + // Relation metadata + private readonly _relationMetadataCreateCollection: Partial[] = + []; + private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] = + []; + + constructor() {} + + get objectMetadataCreateCollection() { + return this._objectMetadataCreateCollection; + } + + get objectMetadataUpdateCollection() { + return this._objectMetadataUpdateCollection; + } + + get objectMetadataDeleteCollection() { + return this._objectMetadataDeleteCollection; + } + + get fieldMetadataCreateCollection() { + return this._fieldMetadataCreateCollection; + } + + get fieldMetadataUpdateCollection() { + return this._fieldMetadataUpdateCollection; + } + + get fieldMetadataDeleteCollection() { + return this._fieldMetadataDeleteCollection; + } + + get relationMetadataCreateCollection() { + return this._relationMetadataCreateCollection; + } + + get relationMetadataDeleteCollection() { + return this._relationMetadataDeleteCollection; + } + + addCreateObjectMetadata(object: PartialObjectMetadata) { + this._objectMetadataCreateCollection.push(object); + } + + addUpdateObjectMetadata(object: Partial) { + this._objectMetadataUpdateCollection.push(object); + } + + addDeleteObjectMetadata(object: ObjectMetadataEntity) { + this._objectMetadataDeleteCollection.push(object); + } + + addCreateFieldMetadata(field: PartialFieldMetadata) { + this._fieldMetadataCreateCollection.push(field); + } + + addUpdateFieldMetadata(field: Partial) { + this._fieldMetadataUpdateCollection.push(field); + } + + addDeleteFieldMetadata(field: FieldMetadataEntity) { + this._fieldMetadataDeleteCollection.push(field); + } + + addCreateRelationMetadata(relation: Partial) { + this._relationMetadataCreateCollection.push(relation); + } + + addDeleteRelationMetadata(relation: RelationMetadataEntity) { + this._relationMetadataDeleteCollection.push(relation); + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts index 60348141ce..b0fe74feb4 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts @@ -1,10 +1,11 @@ +import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface'; + export const isGatedAndNotEnabled = ( - metadata, + gate: GateDecoratorParams | undefined, workspaceFeatureFlagsMap: Record, ): boolean => { const featureFlagValue = - metadata.gate?.featureFlag && - workspaceFeatureFlagsMap[metadata.gate.featureFlag]; + gate?.featureFlag && workspaceFeatureFlagsMap[gate.featureFlag]; - return metadata.gate?.featureFlag !== undefined && !featureFlagValue; + return gate?.featureFlag !== undefined && !featureFlagValue; }; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.spec.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.spec.ts index 3a4bdaac43..153d92dc87 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.spec.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.spec.ts @@ -1,52 +1,7 @@ import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; -import { - filterIgnoredProperties, - mapObjectMetadataByUniqueIdentifier, -} from './sync-metadata.util'; - -describe('filterIgnoredProperties', () => { - it('should filter out properties based on the ignore list', () => { - const obj = { - name: 'John', - age: 30, - email: 'john@example.com', - address: '123 Main St', - }; - const propertiesToIgnore = ['age', 'address']; - - const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); - - expect(filteredObj).toEqual({ - name: 'John', - email: 'john@example.com', - }); - }); - - it('should return the original object if ignore list is empty', () => { - const obj = { - name: 'John', - age: 30, - email: 'john@example.com', - address: '123 Main St', - }; - const propertiesToIgnore: string[] = []; - - const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); - - expect(filteredObj).toEqual(obj); - }); - - it('should return an empty object if the original object is empty', () => { - const obj = {}; - const propertiesToIgnore = ['age', 'address']; - - const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); - - expect(filteredObj).toEqual({}); - }); -}); +import { mapObjectMetadataByUniqueIdentifier } from './sync-metadata.util'; describe('mapObjectMetadataByUniqueIdentifier', () => { it('should convert an array of ObjectMetadataEntity objects into a map', () => { @@ -75,18 +30,18 @@ describe('mapObjectMetadataByUniqueIdentifier', () => { expect(mappedObject).toEqual({ user: { nameSingular: 'user', - fields: { - id: { name: 'id', type: FieldMetadataType.UUID }, - name: { name: 'name', type: FieldMetadataType.TEXT }, - }, + fields: [ + { name: 'id', type: FieldMetadataType.UUID }, + { name: 'name', type: FieldMetadataType.TEXT }, + ], }, product: { nameSingular: 'product', - fields: { - id: { name: 'id', type: FieldMetadataType.UUID }, - name: { name: 'name', type: FieldMetadataType.TEXT }, - price: { name: 'price', type: FieldMetadataType.UUID }, - }, + fields: [ + { name: 'id', type: FieldMetadataType.UUID }, + { name: 'name', type: FieldMetadataType.TEXT }, + { name: 'price', type: FieldMetadataType.UUID }, + ], }, }); }); diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts index 10daca2808..b095c297e3 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts @@ -1,27 +1,3 @@ -import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface'; -import { FieldMetadataOptions } from 'src/metadata/field-metadata/interfaces/field-metadata-options.interface'; -import { FieldMetadataTargetColumnMap } from 'src/metadata/field-metadata/interfaces/field-metadata-target-column-map.interface'; - -/** - * This utility function filters out properties from an object based on a list of properties to ignore. - * It returns a new object with only the properties that are not in the ignore list. - * - * @param obj - The object to filter. - * @param propertiesToIgnore - An array of property names to ignore. - * @returns A new object with filtered properties. - */ -export const filterIgnoredProperties = ( - obj: any, - propertiesToIgnore: string[], - mapFunction?: (value: any) => any, -) => { - return Object.fromEntries( - Object.entries(obj) - .filter(([key]) => !propertiesToIgnore.includes(key)) - .map(([key, value]) => [key, mapFunction ? mapFunction(value) : value]), - ); -}; - /** * This utility function converts an array of ObjectMetadataEntity objects into a map, * where the keys are the nameSingular properties of the objects. @@ -31,59 +7,18 @@ export const filterIgnoredProperties = ( * @returns A map of object metadata, with nameSingular as the key and the object as the value. */ export const mapObjectMetadataByUniqueIdentifier = < - T extends { nameSingular: string; fields: U[] }, - U extends { name: string }, + T extends { nameSingular: string }, >( arr: T[], -): Record & { fields: Record }> => { +): Record => { return arr.reduce( (acc, curr) => { acc[curr.nameSingular] = { ...curr, - fields: curr.fields.reduce( - (acc, curr) => { - acc[curr.name] = curr; - - return acc; - }, - {} as Record, - ), }; return acc; }, - {} as Record & { fields: Record }>, + {} as Record, ); }; - -export const convertStringifiedFieldsToJSON = < - T extends { - targetColumnMap?: string | null; - defaultValue?: string | null; - options?: string | null; - }, ->( - fieldMetadata: T, -): T & { - targetColumnMap?: FieldMetadataTargetColumnMap; - defaultValue?: FieldMetadataDefaultValue; - options?: FieldMetadataOptions; -} => { - if (fieldMetadata.targetColumnMap) { - fieldMetadata.targetColumnMap = JSON.parse(fieldMetadata.targetColumnMap); - } - - if (fieldMetadata.defaultValue) { - fieldMetadata.defaultValue = JSON.parse(fieldMetadata.defaultValue); - } - - if (fieldMetadata.options) { - fieldMetadata.options = JSON.parse(fieldMetadata.options); - } - - return fieldMetadata as T & { - targetColumnMap?: FieldMetadataTargetColumnMap; - defaultValue?: FieldMetadataDefaultValue; - options?: FieldMetadataOptions; - }; -}; diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.module.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.module.ts index dfbfb6fda1..c7cf9f25da 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.module.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.module.ts @@ -8,8 +8,12 @@ import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation- import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module'; import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module'; -import { ReflectiveMetadataFactory } from 'src/workspace/workspace-sync-metadata/reflective-metadata.factory'; -import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service'; +import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service'; +import { workspaceSyncMetadataFactories } from 'src/workspace/workspace-sync-metadata/factories'; +import { workspaceSyncMetadataComparators } from 'src/workspace/workspace-sync-metadata/comparators'; +import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service'; +import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service'; +import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service'; @Module({ imports: [ @@ -26,7 +30,14 @@ import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metad ), TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), ], - providers: [WorkspaceSyncMetadataService, ReflectiveMetadataFactory], + providers: [ + ...workspaceSyncMetadataFactories, + ...workspaceSyncMetadataComparators, + WorkspaceMetadataUpdaterService, + WorkspaceSyncObjectMetadataService, + WorkspaceSyncRelationMetadataService, + WorkspaceSyncMetadataService, + ], exports: [WorkspaceSyncMetadataService], }) export class WorkspaceSyncMetadataModule {} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts new file mode 100644 index 0000000000..59510304f5 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -0,0 +1,82 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; + +import { DataSource } from 'typeorm'; + +import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; + +import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service'; +import { FeatureFlagFactory } from 'src/workspace/workspace-sync-metadata/factories/feature-flags.factory'; +import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service'; +import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service'; +import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage'; + +@Injectable() +export class WorkspaceSyncMetadataService { + private readonly logger = new Logger(WorkspaceSyncMetadataService.name); + + constructor( + @InjectDataSource('metadata') + private readonly metadataDataSource: DataSource, + private readonly featureFlagFactory: FeatureFlagFactory, + private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService, + private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService, + ) {} + + /** + * + * Sync all standard objects and fields metadata for a given workspace and data source + * This will update the metadata if it has changed and generate migrations based on the diff. + * + * @param dataSourceId + * @param workspaceId + */ + public async syncStandardObjectsAndFieldsMetadata( + context: WorkspaceSyncContext, + ) { + this.logger.log('Syncing standard objects and fields metadata'); + const queryRunner = this.metadataDataSource.createQueryRunner(); + + await queryRunner.connect(); + await queryRunner.startTransaction(); + + const manager = queryRunner.manager; + + try { + const storage = new WorkspaceSyncStorage(); + + // Retrieve feature flags + const workspaceFeatureFlagsMap = + await this.featureFlagFactory.create(context); + + this.logger.log('Syncing standard objects and fields metadata'); + + await this.workspaceSyncObjectMetadataService.synchronize( + context, + manager, + storage, + workspaceFeatureFlagsMap, + ); + + await this.workspaceSyncRelationMetadataService.synchronize( + context, + manager, + storage, + workspaceFeatureFlagsMap, + ); + + await queryRunner.commitTransaction(); + + // Execute migrations + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + context.workspaceId, + ); + } catch (error) { + console.error('Sync of standard objects failed with:', error); + await queryRunner.rollbackTransaction(); + } finally { + await queryRunner.release(); + } + } +} diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync.metadata.service.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync.metadata.service.ts deleted file mode 100644 index 3826c15ab4..0000000000 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync.metadata.service.ts +++ /dev/null @@ -1,584 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import diff from 'microdiff'; -import { In, Repository } from 'typeorm'; -import camelCase from 'lodash.camelcase'; -import { v4 as uuidV4 } from 'uuid'; - -import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; -import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface'; -import { - MappedFieldMetadataEntity, - MappedObjectMetadata, -} from 'src/workspace/workspace-sync-metadata/interfaces/mapped-metadata.interface'; - -import { - FieldMetadataEntity, - FieldMetadataType, -} from 'src/metadata/field-metadata/field-metadata.entity'; -import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; -import { - RelationMetadataEntity, - RelationMetadataType, -} from 'src/metadata/relation-metadata/relation-metadata.entity'; -import { - filterIgnoredProperties, - mapObjectMetadataByUniqueIdentifier, - convertStringifiedFieldsToJSON, -} from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util'; -import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects'; -import { - WorkspaceMigrationColumnActionType, - WorkspaceMigrationColumnRelation, - WorkspaceMigrationEntity, - WorkspaceMigrationTableAction, -} from 'src/metadata/workspace-migration/workspace-migration.entity'; -import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory'; -import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service'; -import { ReflectiveMetadataFactory } from 'src/workspace/workspace-sync-metadata/reflective-metadata.factory'; -import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity'; -import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; -import { FieldMetadataComplexOption } from 'src/metadata/field-metadata/dtos/options.input'; - -@Injectable() -export class WorkspaceSyncMetadataService { - constructor( - private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, - private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, - private readonly reflectiveMetadataFactory: ReflectiveMetadataFactory, - - @InjectRepository(ObjectMetadataEntity, 'metadata') - private readonly objectMetadataRepository: Repository, - @InjectRepository(FieldMetadataEntity, 'metadata') - private readonly fieldMetadataRepository: Repository, - @InjectRepository(RelationMetadataEntity, 'metadata') - private readonly relationMetadataRepository: Repository, - @InjectRepository(WorkspaceMigrationEntity, 'metadata') - private readonly workspaceMigrationRepository: Repository, - @InjectRepository(FeatureFlagEntity, 'core') - private readonly featureFlagRepository: Repository, - ) {} - - /** - * - * Sync all standard objects and fields metadata for a given workspace and data source - * This will update the metadata if it has changed and generate migrations based on the diff. - * - * @param dataSourceId - * @param workspaceId - */ - public async syncStandardObjectsAndFieldsMetadata( - dataSourceId: string, - workspaceId: string, - ) { - try { - const workspaceFeatureFlags = await this.featureFlagRepository.find({ - where: { workspaceId }, - }); - - const workspaceFeatureFlagsMap = workspaceFeatureFlags.reduce( - (result, currentFeatureFlag) => { - result[currentFeatureFlag.key] = currentFeatureFlag.value; - - return result; - }, - {}, - ); - - const standardObjects = - await this.reflectiveMetadataFactory.createObjectMetadataCollection( - standardObjectMetadata, - workspaceId, - dataSourceId, - workspaceFeatureFlagsMap, - ); - - const objectsInDB = await this.objectMetadataRepository.find({ - where: { workspaceId, isCustom: false }, - relations: ['dataSource', 'fields'], - }); - - const objectsInDBByName = mapObjectMetadataByUniqueIdentifier< - ObjectMetadataEntity, - FieldMetadataEntity - >(objectsInDB); - const standardObjectsByName = mapObjectMetadataByUniqueIdentifier< - PartialObjectMetadata, - PartialFieldMetadata - >(standardObjects); - - const objectsToCreate: MappedObjectMetadata[] = []; - const objectsToDelete = objectsInDB.filter( - (objectInDB) => !standardObjectsByName[objectInDB.nameSingular], - ); - const objectsToUpdate: Record = {}; - - const fieldsToCreate: PartialFieldMetadata[] = []; - const fieldsToDelete: FieldMetadataEntity[] = []; - const fieldsToUpdate: Record = {}; - - for (const standardObjectName in standardObjectsByName) { - const standardObject = standardObjectsByName[standardObjectName]; - const objectInDB = objectsInDBByName[standardObjectName]; - - if (!objectInDB) { - objectsToCreate.push(standardObject); - continue; - } - - // Deconstruct fields and compare objects and fields independently - const { fields: objectInDBFields, ...objectInDBWithoutFields } = - objectInDB; - const { fields: standardObjectFields, ...standardObjectWithoutFields } = - standardObject; - - const objectPropertiesToIgnore = [ - 'id', - 'createdAt', - 'updatedAt', - 'labelIdentifierFieldMetadataId', - 'imageIdentifierFieldMetadataId', - 'isActive', - ]; - const objectDiffWithoutIgnoredProperties = filterIgnoredProperties( - objectInDBWithoutFields, - objectPropertiesToIgnore, - ); - - const fieldPropertiesToIgnore = [ - 'id', - 'createdAt', - 'updatedAt', - 'objectMetadataId', - 'isActive', - ]; - const objectInDBFieldsWithoutDefaultFields = Object.fromEntries( - Object.entries(objectInDBFields).map(([key, value]) => { - if (value === null || typeof value !== 'object') { - return [key, value]; - } - - return [ - key, - filterIgnoredProperties( - value, - fieldPropertiesToIgnore, - (property) => { - if (property !== null && typeof property === 'object') { - return JSON.stringify(property); - } - - return property; - }, - ), - ]; - }), - ); - - // Compare objects - const objectDiff = diff( - objectDiffWithoutIgnoredProperties, - standardObjectWithoutFields, - ); - - // Compare fields - const fieldsDiff = diff( - objectInDBFieldsWithoutDefaultFields, - standardObjectFields, - ); - - for (const diff of objectDiff) { - // We only handle CHANGE here as REMOVE and CREATE are handled earlier. - if (diff.type === 'CHANGE') { - const property = diff.path[0]; - - objectsToUpdate[objectInDB.id] = { - ...objectsToUpdate[objectInDB.id], - [property]: diff.value, - }; - } - } - - for (const diff of fieldsDiff) { - const fieldName = diff.path[0]; - - if (diff.type === 'CREATE') { - fieldsToCreate.push({ - ...standardObjectFields[fieldName], - objectMetadataId: objectInDB.id, - }); - } - if (diff.type === 'REMOVE' && diff.path.length === 1) { - fieldsToDelete.push(objectInDBFields[fieldName]); - } - if (diff.type === 'CHANGE') { - const property = diff.path[diff.path.length - 1]; - - fieldsToUpdate[objectInDBFields[fieldName].id] = { - ...fieldsToUpdate[objectInDBFields[fieldName].id], - [property]: diff.value, - }; - } - } - } - - // CREATE OBJECTS - const createdObjectMetadataCollection = - await this.objectMetadataRepository.save( - objectsToCreate.map((object) => ({ - ...object, - isActive: true, - fields: Object.values(object.fields).map((field) => - this.prepareFieldMetadataForCreation(field), - ), - })), - ); - const identifiers = createdObjectMetadataCollection.map( - (object) => object.id, - ); - const createdObjects = await this.objectMetadataRepository.find({ - where: { id: In(identifiers) }, - relations: ['dataSource', 'fields'], - }); - - // UPDATE OBJECTS, this is not optimal as we are running n queries here. - for (const [key, value] of Object.entries(objectsToUpdate)) { - await this.objectMetadataRepository.update(key, value); - } - // DELETE OBJECTS - if (objectsToDelete.length > 0) { - await this.objectMetadataRepository.delete( - objectsToDelete.map((object) => object.id), - ); - } - - // CREATE FIELDS - const createdFields = await this.fieldMetadataRepository.save( - fieldsToCreate.map((field) => - this.prepareFieldMetadataForCreation(field), - ), - ); - - // UPDATE FIELDS - for (const [key, value] of Object.entries(fieldsToUpdate)) { - await this.fieldMetadataRepository.update( - key, - convertStringifiedFieldsToJSON(value), - ); - } - // DELETE FIELDS - // TODO: handle relation fields deletion. We need to delete the relation metadata first due to the DB constraint. - const fieldsToDeleteWithoutRelationType = fieldsToDelete.filter( - (field) => field.type !== FieldMetadataType.RELATION, - ); - - if (fieldsToDeleteWithoutRelationType.length > 0) { - await this.fieldMetadataRepository.delete( - fieldsToDeleteWithoutRelationType.map((field) => field.id), - ); - } - - // Generate migrations - await this.generateMigrationsFromSync( - createdObjects, - objectsToDelete, - createdFields, - fieldsToDelete, - objectsInDB, - ); - - // We run syncRelationMetadata after everything to ensure that all objects and fields are - // in the DB before creating relations. - await this.syncRelationMetadata( - workspaceId, - dataSourceId, - workspaceFeatureFlagsMap, - ); - - // Execute migrations - await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( - workspaceId, - ); - } catch (error) { - console.error('Sync of standard objects failed with:', error); - } - } - - private prepareFieldMetadataForCreation(field: PartialFieldMetadata) { - const convertedField = convertStringifiedFieldsToJSON(field); - - return { - ...convertedField, - ...(convertedField.type === FieldMetadataType.SELECT && - convertedField.options - ? { - options: this.generateUUIDForNewSelectFieldOptions( - convertedField.options as FieldMetadataComplexOption[], - ), - } - : {}), - isActive: true, - }; - } - - private generateUUIDForNewSelectFieldOptions( - options: FieldMetadataComplexOption[], - ): FieldMetadataComplexOption[] { - return options.map((option) => ({ - ...option, - id: uuidV4(), - })); - } - - private async syncRelationMetadata( - workspaceId: string, - dataSourceId: string, - workspaceFeatureFlagsMap: Record, - ) { - const objectsInDB = await this.objectMetadataRepository.find({ - where: { workspaceId, isCustom: false }, - relations: ['dataSource', 'fields'], - }); - const objectsInDBByName = mapObjectMetadataByUniqueIdentifier< - ObjectMetadataEntity, - FieldMetadataEntity - >(objectsInDB); - const standardRelations = - this.reflectiveMetadataFactory.createRelationMetadataCollection( - standardObjectMetadata, - workspaceId, - objectsInDBByName, - workspaceFeatureFlagsMap, - ); - - // TODO: filter out custom relations once isCustom has been added to relationMetadata table - const relationsInDB = await this.relationMetadataRepository.find({ - where: { workspaceId }, - }); - - // We filter out 'id' later because we need it to remove the relation from DB - const relationsInDBWithoutIgnoredProperties = relationsInDB - .map((relation) => - filterIgnoredProperties(relation, ['createdAt', 'updatedAt']), - ) - .reduce((result, currentObject) => { - const key = `${currentObject.fromObjectMetadataId}->${currentObject.fromFieldMetadataId}`; - - result[key] = currentObject; - - return result; - }, {}); - - // Compare relations - const relationsDiff = diff( - relationsInDBWithoutIgnoredProperties, - standardRelations, - ); - - const relationsToCreate: RelationMetadataEntity[] = []; - const relationsToDelete: RelationMetadataEntity[] = []; - - for (const diff of relationsDiff) { - if (diff.type === 'CREATE') { - relationsToCreate.push(diff.value); - } - if (diff.type === 'REMOVE' && diff.path[diff.path.length - 1] !== 'id') { - relationsToDelete.push(diff.oldValue); - } - } - - try { - // CREATE RELATIONS - await this.relationMetadataRepository.save(relationsToCreate); - // DELETE RELATIONS - if (relationsToDelete.length > 0) { - await this.relationMetadataRepository.delete( - relationsToDelete.map((relation) => relation.id), - ); - } - - await this.generateRelationMigrationsFromSync( - relationsToCreate, - relationsToDelete, - objectsInDB, - ); - } catch (error) { - console.error('Sync of standard relations failed with:', error); - } - } - - private async generateMigrationsFromSync( - objectsToCreate: ObjectMetadataEntity[], - _objectsToDelete: ObjectMetadataEntity[], - fieldsToCreate: FieldMetadataEntity[], - fieldsToDelete: FieldMetadataEntity[], - objectsInDB: ObjectMetadataEntity[], - ) { - const migrationsToSave: Partial[] = []; - - if (objectsToCreate.length > 0) { - objectsToCreate.map((object) => { - const migrations = [ - { - name: computeObjectTargetTable(object), - action: 'create', - } satisfies WorkspaceMigrationTableAction, - ...Object.values(object.fields) - .filter((field) => field.type !== FieldMetadataType.RELATION) - .map( - (field) => - ({ - name: computeObjectTargetTable(object), - action: 'alter', - columns: this.workspaceMigrationFactory.createColumnActions( - WorkspaceMigrationColumnActionType.CREATE, - field, - ), - }) satisfies WorkspaceMigrationTableAction, - ), - ]; - - migrationsToSave.push({ - workspaceId: object.workspaceId, - isCustom: false, - migrations, - }); - }); - } - - // TODO: handle object delete migrations. - // Note: we need to delete the relation first due to the DB constraint. - - const objectsInDbById = objectsInDB.reduce( - (result, currentObject) => { - result[currentObject.id] = currentObject; - - return result; - }, - {} as Record, - ); - - if (fieldsToCreate.length > 0) { - fieldsToCreate.map((field) => { - const migrations = [ - { - name: computeObjectTargetTable( - objectsInDbById[field.objectMetadataId], - ), - action: 'alter', - columns: this.workspaceMigrationFactory.createColumnActions( - WorkspaceMigrationColumnActionType.CREATE, - field, - ), - } satisfies WorkspaceMigrationTableAction, - ]; - - migrationsToSave.push({ - workspaceId: field.workspaceId, - isCustom: false, - migrations, - }); - }); - } - - if (fieldsToDelete.length > 0) { - fieldsToDelete.map((field) => { - const migrations = [ - { - name: computeObjectTargetTable( - objectsInDbById[field.objectMetadataId], - ), - action: 'alter', - columns: [ - { - action: WorkspaceMigrationColumnActionType.DROP, - columnName: field.name, - }, - ], - } satisfies WorkspaceMigrationTableAction, - ]; - - migrationsToSave.push({ - workspaceId: field.workspaceId, - isCustom: false, - migrations, - }); - }); - } - - await this.workspaceMigrationRepository.save(migrationsToSave); - - // TODO: handle delete migrations - } - - private async generateRelationMigrationsFromSync( - relationsToCreate: RelationMetadataEntity[], - _relationsToDelete: RelationMetadataEntity[], - objectsInDB: ObjectMetadataEntity[], - ) { - const relationsMigrationsToSave: Partial[] = []; - - if (relationsToCreate.length > 0) { - relationsToCreate.map((relation) => { - const toObjectMetadata = objectsInDB.find( - (object) => object.id === relation.toObjectMetadataId, - ); - - const fromObjectMetadata = objectsInDB.find( - (object) => object.id === relation.fromObjectMetadataId, - ); - - if (!toObjectMetadata) { - throw new Error( - `ObjectMetadata with id ${relation.toObjectMetadataId} not found`, - ); - } - - if (!fromObjectMetadata) { - throw new Error( - `ObjectMetadata with id ${relation.fromObjectMetadataId} not found`, - ); - } - - const toFieldMetadata = toObjectMetadata.fields.find( - (field) => field.id === relation.toFieldMetadataId, - ); - - if (!toFieldMetadata) { - throw new Error( - `FieldMetadata with id ${relation.toFieldMetadataId} not found`, - ); - } - - const migrations = [ - { - name: computeObjectTargetTable(toObjectMetadata), - action: 'alter', - columns: [ - { - action: WorkspaceMigrationColumnActionType.RELATION, - columnName: `${camelCase(toFieldMetadata.name)}Id`, - referencedTableName: - computeObjectTargetTable(fromObjectMetadata), - referencedTableColumnName: 'id', - isUnique: - relation.relationType === RelationMetadataType.ONE_TO_ONE, - } satisfies WorkspaceMigrationColumnRelation, - ], - } satisfies WorkspaceMigrationTableAction, - ]; - - relationsMigrationsToSave.push({ - workspaceId: relation.workspaceId, - isCustom: false, - migrations, - }); - }); - } - - await this.workspaceMigrationRepository.save(relationsMigrationsToSave); - - // TODO: handle delete migrations - } -} diff --git a/yarn.lock b/yarn.lock index f49727562e..f29e29e4ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15371,6 +15371,15 @@ __metadata: languageName: node linkType: hard +"@types/lodash.omit@npm:^4.5.9": + version: 4.5.9 + resolution: "@types/lodash.omit@npm:4.5.9" + dependencies: + "@types/lodash": "npm:*" + checksum: 3b60c8ee8e9a691392d9a3ceabb32c85f888784bd3307eac3de01aeb7ff37383dc8899f027fe852641f5e0f56158fb19785cc3d20a4922e85b5810f14cba23f6 + languageName: node + linkType: hard + "@types/lodash.snakecase@npm:^4.1.7": version: 4.1.9 resolution: "@types/lodash.snakecase@npm:4.1.9" @@ -43153,6 +43162,7 @@ __metadata: "@sentry/profiling-node": "npm:^1.3.4" "@types/lodash.isempty": "npm:^4.4.7" "@types/lodash.isobject": "npm:^3.0.7" + "@types/lodash.omit": "npm:^4.5.9" "@types/lodash.snakecase": "npm:^4.1.7" "@types/lodash.upperfirst": "npm:^4.3.7" "@types/react": "npm:^18.2.39" @@ -43176,6 +43186,7 @@ __metadata: lodash.isempty: "npm:^4.4.0" lodash.isobject: "npm:^3.0.2" lodash.kebabcase: "npm:^4.1.1" + lodash.omit: "npm:^4.5.0" lodash.snakecase: "npm:^4.1.1" lodash.upperfirst: "npm:^4.3.1" mailparser: "npm:^3.6.5"