From 990cb107a15277401a3aa99520fbfe2200691d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20M?= Date: Thu, 15 Feb 2024 18:04:12 +0100 Subject: [PATCH] feat: workspace health target column map fix (#3932) * feat: workspace health fix target column map * fix: remove log * feat: refactor health fixer * fix: default-value issue and health check not working with composite * fix: enhance target column map fix * feat: create workspace migrations for target-column-map issues * feat: enhance workspace-health issue detection --- packages/twenty-server/package.json | 2 + .../object-metadata.service.ts | 6 +- .../utils/compute-custom-name.util.ts | 5 + .../utils/compute-field-target-column.util.ts | 21 +++ .../utils/compute-object-target-table.util.ts | 6 +- .../commands/workspace-health.command.ts | 39 ++-- .../fixer/abstract-workspace.fixer.ts | 68 +++++++ .../workspace/workspace-health/fixer/index.ts | 11 ++ .../workspace-default-value.fixer.ts} | 31 +--- .../workspace-nullable.fixer.ts} | 33 ++-- .../workspace-target-column-map.fixer.ts | 174 ++++++++++++++++++ .../workspace-type.fixer.ts} | 34 ++-- .../workspace-health-issue.interface.ts | 105 ++++++----- .../services/database-structure.service.ts | 18 ++ .../services/field-metadata-health.service.ts | 94 ++++------ .../services/workspace-fix.service.ts | 113 ++++++++---- .../compute-composite-field-metadata.util.ts | 15 ++ .../is-workspace-health-issue-type.util.ts | 6 + .../workspace-health.module.ts | 9 +- .../workspace-health.service.ts | 26 ++- .../workspace-migration-field.factory.ts | 1 + .../workspace-migration-runner.service.ts | 2 +- .../sync-workspace-metadata.command.ts | 41 ++++- ...workspace-sync-metadata-commands.module.ts | 7 +- yarn.lock | 4 +- 25 files changed, 625 insertions(+), 246 deletions(-) create mode 100644 packages/twenty-server/src/workspace/utils/compute-custom-name.util.ts create mode 100644 packages/twenty-server/src/workspace/utils/compute-field-target-column.util.ts create mode 100644 packages/twenty-server/src/workspace/workspace-health/fixer/abstract-workspace.fixer.ts create mode 100644 packages/twenty-server/src/workspace/workspace-health/fixer/index.ts rename packages/twenty-server/src/workspace/workspace-health/{services/workspace-fix-default-value.service.ts => fixer/workspace-default-value.fixer.ts} (75%) rename packages/twenty-server/src/workspace/workspace-health/{services/workspace-fix-nullable.service.ts => fixer/workspace-nullable.fixer.ts} (66%) create mode 100644 packages/twenty-server/src/workspace/workspace-health/fixer/workspace-target-column-map.fixer.ts rename packages/twenty-server/src/workspace/workspace-health/{services/workspace-fix-type.service.ts => fixer/workspace-type.fixer.ts} (69%) create mode 100644 packages/twenty-server/src/workspace/workspace-health/utils/compute-composite-field-metadata.util.ts diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 54fdb68419..e81b701b85 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -39,12 +39,14 @@ "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch", "class-validator": "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch", "graphql-middleware": "^6.1.35", + "lodash.isequal": "^4.5.0", "passport": "^0.7.0" }, "devDependencies": { "@nestjs/cli": "10.3.0", "@nx/js": "17.2.8", "@types/lodash.isempty": "^4.4.7", + "@types/lodash.isequal": "^4.5.8", "@types/lodash.isobject": "^3.0.7", "@types/lodash.omit": "^4.5.9", "@types/lodash.snakecase": "^4.1.7", diff --git a/packages/twenty-server/src/metadata/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/metadata/object-metadata/object-metadata.service.ts index 852be0a8e5..0563bd040a 100644 --- a/packages/twenty-server/src/metadata/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/metadata/object-metadata/object-metadata.service.ts @@ -29,10 +29,8 @@ import { RelationMetadataEntity, RelationMetadataType, } from 'src/metadata/relation-metadata/relation-metadata.entity'; -import { - computeCustomName, - computeObjectTargetTable, -} from 'src/workspace/utils/compute-object-target-table.util'; +import { computeCustomName } from 'src/workspace/utils/compute-custom-name.util'; +import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; import { DeleteOneObjectInput } from 'src/metadata/object-metadata/dtos/delete-object.input'; import { RelationToDelete } from 'src/metadata/relation-metadata/types/relation-to-delete'; import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util'; diff --git a/packages/twenty-server/src/workspace/utils/compute-custom-name.util.ts b/packages/twenty-server/src/workspace/utils/compute-custom-name.util.ts new file mode 100644 index 0000000000..3e6c10d96e --- /dev/null +++ b/packages/twenty-server/src/workspace/utils/compute-custom-name.util.ts @@ -0,0 +1,5 @@ +export const customNamePrefix = '_'; + +export const computeCustomName = (name: string, isCustom: boolean) => { + return isCustom ? `${customNamePrefix}${name}` : name; +}; diff --git a/packages/twenty-server/src/workspace/utils/compute-field-target-column.util.ts b/packages/twenty-server/src/workspace/utils/compute-field-target-column.util.ts new file mode 100644 index 0000000000..aad29d0ec9 --- /dev/null +++ b/packages/twenty-server/src/workspace/utils/compute-field-target-column.util.ts @@ -0,0 +1,21 @@ +import { FieldMetadataInterface } from 'src/metadata/field-metadata/interfaces/field-metadata.interface'; + +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { isCompositeFieldMetadataType } from 'src/metadata/field-metadata/utils/is-composite-field-metadata-type.util'; +import { BasicFieldMetadataType } from 'src/metadata/workspace-migration/factories/basic-column-action.factory'; + +import { computeCustomName } from './compute-custom-name.util'; + +export const computeFieldTargetColumn = ( + fieldMetadata: + | FieldMetadataEntity + | FieldMetadataInterface, +) => { + if (isCompositeFieldMetadataType(fieldMetadata.type)) { + throw new Error( + "Composite field metadata should not be computed here, as they're split into multiple fields.", + ); + } + + return computeCustomName(fieldMetadata.name, fieldMetadata.isCustom ?? false); +}; diff --git a/packages/twenty-server/src/workspace/utils/compute-object-target-table.util.ts b/packages/twenty-server/src/workspace/utils/compute-object-target-table.util.ts index fd241f152a..98f49dacc4 100644 --- a/packages/twenty-server/src/workspace/utils/compute-object-target-table.util.ts +++ b/packages/twenty-server/src/workspace/utils/compute-object-target-table.util.ts @@ -1,5 +1,7 @@ import { ObjectMetadataInterface } from 'src/metadata/field-metadata/interfaces/object-metadata.interface'; +import { computeCustomName } from './compute-custom-name.util'; + export const computeObjectTargetTable = ( objectMetadata: ObjectMetadataInterface, ) => { @@ -8,7 +10,3 @@ export const computeObjectTargetTable = ( objectMetadata.isCustom, ); }; - -export const computeCustomName = (name: string, isCustom: boolean) => { - return isCustom ? `_${name}` : name; -}; diff --git a/packages/twenty-server/src/workspace/workspace-health/commands/workspace-health.command.ts b/packages/twenty-server/src/workspace/workspace-health/commands/workspace-health.command.ts index 7f49352c52..bbfe1d6e64 100644 --- a/packages/twenty-server/src/workspace/workspace-health/commands/workspace-health.command.ts +++ b/packages/twenty-server/src/workspace/workspace-health/commands/workspace-health.command.ts @@ -11,7 +11,6 @@ import { CommandLogger } from 'src/commands/command-logger'; interface WorkspaceHealthCommandOptions { workspaceId: string; - verbose?: boolean; mode?: WorkspaceHealthMode; fix?: WorkspaceHealthFixKind; dryRun?: boolean; @@ -49,7 +48,7 @@ export class WorkspaceHealthCommand extends CommandRunner { chalk.red(`Workspace is not healthy, found ${issues.length} issues`), ); - if (options.verbose) { + if (options.dryRun) { await this.commandLogger.writeLog( `workspace-health-issues-${options.workspaceId}`, issues, @@ -61,25 +60,30 @@ export class WorkspaceHealthCommand extends CommandRunner { if (options.fix) { this.logger.log(chalk.yellow('Fixing issues')); - const workspaceMigrations = await this.workspaceHealthService.fixIssues( - options.workspaceId, - issues, - { - type: options.fix, - applyChanges: !options.dryRun, - }, - ); + const { workspaceMigrations, metadataEntities } = + await this.workspaceHealthService.fixIssues( + options.workspaceId, + issues, + { + type: options.fix, + applyChanges: !options.dryRun, + }, + ); + const totalCount = workspaceMigrations.length + metadataEntities.length; if (options.dryRun) { await this.commandLogger.writeLog( `workspace-health-${options.fix}-migrations`, workspaceMigrations, ); + + await this.commandLogger.writeLog( + `workspace-health-${options.fix}-metadata-entities`, + metadataEntities, + ); } else { this.logger.log( - chalk.green( - `Fixed ${workspaceMigrations.length}/${issues.length} issues`, - ), + chalk.green(`Fixed ${totalCount}/${issues.length} issues`), ); } } @@ -107,15 +111,6 @@ export class WorkspaceHealthCommand extends CommandRunner { return value as WorkspaceHealthFixKind; } - @Option({ - flags: '-v, --verbose', - description: 'Detailed output', - required: false, - }) - parseVerbose(): boolean { - return true; - } - @Option({ flags: '-m, --mode [mode]', description: 'Mode of the health check [structure, metadata, all]', diff --git a/packages/twenty-server/src/workspace/workspace-health/fixer/abstract-workspace.fixer.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/abstract-workspace.fixer.ts new file mode 100644 index 0000000000..4434317a5c --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/abstract-workspace.fixer.ts @@ -0,0 +1,68 @@ +import { EntityManager } from 'typeorm'; + +import { + WorkspaceHealthIssue, + WorkspaceHealthIssueType, + WorkspaceIssueTypeToInterface, +} from 'src/workspace/workspace-health/interfaces/workspace-health-issue.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; + +export class CompareEntity { + current: T | null; + altered: T | null; +} + +export abstract class AbstractWorkspaceFixer< + IssueTypes extends WorkspaceHealthIssueType, + UpdateRecordEntities = unknown, +> { + private issueTypes: IssueTypes[]; + + constructor(...issueTypes: IssueTypes[]) { + this.issueTypes = issueTypes; + } + + filterIssues( + issues: WorkspaceHealthIssue[], + ): WorkspaceIssueTypeToInterface[] { + return issues.filter( + (issue): issue is WorkspaceIssueTypeToInterface => + this.issueTypes.includes(issue.type as IssueTypes), + ); + } + + protected splitIssuesByType( + issues: WorkspaceIssueTypeToInterface[], + ): Record[]> { + return issues.reduce( + ( + acc: Record[]>, + issue, + ) => { + const type = issue.type as IssueTypes; + + if (!acc[type]) { + acc[type] = []; + } + acc[type].push(issue); + + return acc; + }, + {} as Record[]>, + ); + } + + async createWorkspaceMigrations?( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + issues: WorkspaceIssueTypeToInterface[], + ): Promise[]>; + + async createMetadataUpdates?( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + issues: WorkspaceIssueTypeToInterface[], + ): Promise[]>; +} diff --git a/packages/twenty-server/src/workspace/workspace-health/fixer/index.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/index.ts new file mode 100644 index 0000000000..ca9d93aac0 --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/index.ts @@ -0,0 +1,11 @@ +import { WorkspaceNullableFixer } from './workspace-nullable.fixer'; +import { WorkspaceDefaultValueFixer } from './workspace-default-value.fixer'; +import { WorkspaceTypeFixer } from './workspace-type.fixer'; +import { WorkspaceTargetColumnMapFixer } from './workspace-target-column-map.fixer'; + +export const workspaceFixers = [ + WorkspaceNullableFixer, + WorkspaceDefaultValueFixer, + WorkspaceTypeFixer, + WorkspaceTargetColumnMapFixer, +]; diff --git a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-default-value.service.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-default-value.fixer.ts similarity index 75% rename from packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-default-value.service.ts rename to packages/twenty-server/src/workspace/workspace-health/fixer/workspace-default-value.fixer.ts index 2d45eeb8e1..fb95e1e020 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-default-value.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-default-value.fixer.ts @@ -13,37 +13,26 @@ import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metada import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; -type WorkspaceHealthDefaultValueIssue = - WorkspaceHealthColumnIssue; +import { AbstractWorkspaceFixer } from './abstract-workspace.fixer'; @Injectable() -export class WorkspaceFixDefaultValueService { +export class WorkspaceDefaultValueFixer extends AbstractWorkspaceFixer { constructor( private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory, - ) {} + ) { + super(WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT); + } - async fix( + async createWorkspaceMigrations( manager: EntityManager, objectMetadataCollection: ObjectMetadataEntity[], - issues: WorkspaceHealthDefaultValueIssue[], + issues: WorkspaceHealthColumnIssue[], ): Promise[]> { - const workspaceMigrations: Partial[] = []; - const defaultValueIssues = issues.filter( - (issue) => - issue.type === WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT, - ) as WorkspaceHealthColumnIssue[]; - - if (defaultValueIssues.length > 0) { - const columnDefaultValueWorkspaceMigrations = - await this.fixColumnDefaultValueIssues( - objectMetadataCollection, - defaultValueIssues, - ); - - workspaceMigrations.push(...columnDefaultValueWorkspaceMigrations); + if (issues.length <= 0) { + return []; } - return workspaceMigrations; + return this.fixColumnDefaultValueIssues(objectMetadataCollection, issues); } private async fixColumnDefaultValueIssues( diff --git a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-nullable.service.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-nullable.fixer.ts similarity index 66% rename from packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-nullable.service.ts rename to packages/twenty-server/src/workspace/workspace-health/fixer/workspace-nullable.fixer.ts index b9353e8b66..dda7d94794 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-nullable.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-nullable.fixer.ts @@ -8,41 +8,30 @@ import { } from 'src/workspace/workspace-health/interfaces/workspace-health-issue.interface'; import { WorkspaceMigrationBuilderAction } from 'src/workspace/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; -import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; -type WorkspaceHealthNullableIssue = - WorkspaceHealthColumnIssue; +import { AbstractWorkspaceFixer } from './abstract-workspace.fixer'; @Injectable() -export class WorkspaceFixNullableService { +export class WorkspaceNullableFixer extends AbstractWorkspaceFixer { constructor( private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory, - ) {} + ) { + super(WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT); + } - async fix( + async createWorkspaceMigrations( manager: EntityManager, objectMetadataCollection: ObjectMetadataEntity[], - issues: WorkspaceHealthNullableIssue[], + issues: WorkspaceHealthColumnIssue[], ): Promise[]> { - const workspaceMigrations: Partial[] = []; - const nullabilityIssues = issues.filter( - (issue) => - issue.type === WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT, - ) as WorkspaceHealthColumnIssue[]; - - if (nullabilityIssues.length > 0) { - const columnNullabilityWorkspaceMigrations = - await this.fixColumnNullabilityIssues( - objectMetadataCollection, - nullabilityIssues, - ); - - workspaceMigrations.push(...columnNullabilityWorkspaceMigrations); + if (issues.length <= 0) { + return []; } - return workspaceMigrations; + return this.fixColumnNullabilityIssues(objectMetadataCollection, issues); } private async fixColumnNullabilityIssues( diff --git a/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-target-column-map.fixer.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-target-column-map.fixer.ts new file mode 100644 index 0000000000..1b91ab48af --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-target-column-map.fixer.ts @@ -0,0 +1,174 @@ +import { Injectable } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; +import isEqual from 'lodash.isequal'; + +import { + WorkspaceHealthColumnIssue, + WorkspaceHealthIssueType, +} from 'src/workspace/workspace-health/interfaces/workspace-health-issue.interface'; +import { WorkspaceMigrationBuilderAction } from 'src/workspace/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; + +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util'; +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; +import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; +import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity'; +import { DatabaseStructureService } from 'src/workspace/workspace-health/services/database-structure.service'; +import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; +import { isCompositeFieldMetadataType } from 'src/metadata/field-metadata/utils/is-composite-field-metadata-type.util'; + +import { + AbstractWorkspaceFixer, + CompareEntity, +} from './abstract-workspace.fixer'; + +@Injectable() +export class WorkspaceTargetColumnMapFixer extends AbstractWorkspaceFixer< + WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID, + FieldMetadataEntity +> { + constructor( + private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory, + private readonly databaseStructureService: DatabaseStructureService, + ) { + super(WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID); + } + + async createWorkspaceMigrations( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + issues: WorkspaceHealthColumnIssue[], + ): Promise[]> { + if (issues.length <= 0) { + return []; + } + + return this.fixStructureTargetColumnMapIssues( + manager, + objectMetadataCollection, + issues, + ); + } + + async createMetadataUpdates( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + issues: WorkspaceHealthColumnIssue[], + ): Promise[]> { + if (issues.length <= 0) { + return []; + } + + return this.fixMetadataTargetColumnMapIssues(manager, issues); + } + + private async fixStructureTargetColumnMapIssues( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + issues: WorkspaceHealthColumnIssue[], + ): Promise[]> { + const workspaceMigrationCollection: Partial[] = + []; + const dataSourceRepository = manager.getRepository(DataSourceEntity); + + for (const issue of issues) { + const objectMetadata = objectMetadataCollection.find( + (metadata) => metadata.id === issue.fieldMetadata.objectMetadataId, + ); + const targetColumnMap = generateTargetColumnMap( + issue.fieldMetadata.type, + issue.fieldMetadata.isCustom, + issue.fieldMetadata.name, + ); + + // Skip composite fields, too complicated to fix for now + if (isCompositeFieldMetadataType(issue.fieldMetadata.type)) { + continue; + } + + if (!objectMetadata) { + throw new Error( + `Object metadata with id ${issue.fieldMetadata.objectMetadataId} not found`, + ); + } + + if (!isEqual(issue.fieldMetadata.targetColumnMap, targetColumnMap)) { + // Retrieve the data source to get the schema name + const dataSource = await dataSourceRepository.findOne({ + where: { + id: objectMetadata.dataSourceId, + }, + }); + + if (!dataSource) { + throw new Error( + `Data source with id ${objectMetadata.dataSourceId} not found`, + ); + } + + const columnName = issue.fieldMetadata.targetColumnMap?.value; + const columnExist = + await this.databaseStructureService.workspaceColumnExist( + dataSource.schema, + computeObjectTargetTable(objectMetadata), + columnName, + ); + + if (!columnExist) { + continue; + } + + const workspaceMigration = + await this.workspaceMigrationFieldFactory.create( + objectMetadataCollection, + [ + { + current: issue.fieldMetadata, + altered: { + ...issue.fieldMetadata, + targetColumnMap, + }, + }, + ], + WorkspaceMigrationBuilderAction.UPDATE, + ); + + workspaceMigrationCollection.push(workspaceMigration[0]); + } + } + + return workspaceMigrationCollection; + } + + private async fixMetadataTargetColumnMapIssues( + manager: EntityManager, + issues: WorkspaceHealthColumnIssue[], + ): Promise[]> { + const fieldMetadataRepository = manager.getRepository(FieldMetadataEntity); + const updatedEntities: CompareEntity[] = []; + + for (const issue of issues) { + await fieldMetadataRepository.update(issue.fieldMetadata.id, { + targetColumnMap: generateTargetColumnMap( + issue.fieldMetadata.type, + issue.fieldMetadata.isCustom, + issue.fieldMetadata.name, + ), + }); + const alteredEntity = await fieldMetadataRepository.findOne({ + where: { + id: issue.fieldMetadata.id, + }, + }); + + updatedEntities.push({ + current: issue.fieldMetadata, + altered: alteredEntity as FieldMetadataEntity | null, + }); + } + + return updatedEntities; + } +} diff --git a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-type.service.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts similarity index 69% rename from packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-type.service.ts rename to packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts index f516e071d3..3392cb046c 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix-type.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts @@ -11,41 +11,29 @@ import { WorkspaceMigrationBuilderAction } from 'src/workspace/workspace-migrati import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; +import { DatabaseStructureService } from 'src/workspace/workspace-health/services/database-structure.service'; -import { DatabaseStructureService } from './database-structure.service'; - -type WorkspaceHealthTypeIssue = - WorkspaceHealthColumnIssue; +import { AbstractWorkspaceFixer } from './abstract-workspace.fixer'; @Injectable() -export class WorkspaceFixTypeService { +export class WorkspaceTypeFixer extends AbstractWorkspaceFixer { constructor( private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory, private readonly databaseStructureService: DatabaseStructureService, - ) {} + ) { + super(WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT); + } - async fix( + async createWorkspaceMigrations( manager: EntityManager, objectMetadataCollection: ObjectMetadataEntity[], - issues: WorkspaceHealthTypeIssue[], + issues: WorkspaceHealthColumnIssue[], ): Promise[]> { - const workspaceMigrations: Partial[] = []; - const columnTypeIssues = issues.filter( - (issue) => - issue.type === WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT, - ) as WorkspaceHealthColumnIssue[]; - - if (columnTypeIssues.length > 0) { - const columnNullabilityWorkspaceMigrations = - await this.fixColumnTypeIssues( - objectMetadataCollection, - columnTypeIssues, - ); - - workspaceMigrations.push(...columnNullabilityWorkspaceMigrations); + if (issues.length <= 0) { + return []; } - return workspaceMigrations; + return this.fixColumnTypeIssues(objectMetadataCollection, issues); } private async fixColumnTypeIssues( diff --git a/packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-health-issue.interface.ts b/packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-health-issue.interface.ts index 282a9a5875..44294a01ac 100644 --- a/packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-health-issue.interface.ts +++ b/packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-health-issue.interface.ts @@ -14,6 +14,7 @@ export enum WorkspaceHealthIssueType { MISSING_INDEX = 'MISSING_INDEX', MISSING_FOREIGN_KEY = 'MISSING_FOREIGN_KEY', MISSING_COMPOSITE_TYPE = 'MISSING_COMPOSITE_TYPE', + COLUMN_NAME_SHOULD_NOT_BE_PREFIXED = 'COLUMN_NAME_SHOULD_NOT_BE_PREFIXED', COLUMN_TARGET_COLUMN_MAP_NOT_VALID = 'COLUMN_TARGET_COLUMN_MAP_NOT_VALID', COLUMN_NAME_SHOULD_BE_CUSTOM = 'COLUMN_NAME_SHOULD_BE_CUSTOM', COLUMN_OBJECT_REFERENCE_INVALID = 'COLUMN_OBJECT_REFERENCE_INVALID', @@ -30,61 +31,64 @@ export enum WorkspaceHealthIssueType { RELATION_TYPE_NOT_VALID = 'RELATION_TYPE_NOT_VALID', } -type ConditionalType< - T extends WorkspaceHealthIssueType | null, - U, -> = T extends WorkspaceHealthIssueType ? T : U; +/** + * Table issues + */ +export type WorkspaceTableIssueTypes = + | WorkspaceHealthIssueType.MISSING_TABLE + | WorkspaceHealthIssueType.TABLE_NAME_SHOULD_BE_CUSTOM + | WorkspaceHealthIssueType.TABLE_TARGET_TABLE_NAME_NOT_VALID + | WorkspaceHealthIssueType.TABLE_DATA_SOURCE_ID_NOT_VALID + | WorkspaceHealthIssueType.TABLE_NAME_NOT_VALID; -export interface WorkspaceHealthTableIssue< - T extends WorkspaceHealthIssueType | null = null, -> { - type: ConditionalType< - T, - | WorkspaceHealthIssueType.MISSING_TABLE - | WorkspaceHealthIssueType.TABLE_NAME_SHOULD_BE_CUSTOM - | WorkspaceHealthIssueType.TABLE_TARGET_TABLE_NAME_NOT_VALID - | WorkspaceHealthIssueType.TABLE_DATA_SOURCE_ID_NOT_VALID - | WorkspaceHealthIssueType.TABLE_NAME_NOT_VALID - >; +export interface WorkspaceHealthTableIssue { + type: T; objectMetadata: ObjectMetadataEntity; message: string; } +/** + * Column issues + */ +export type WorkspaceColumnIssueTypes = + | WorkspaceHealthIssueType.MISSING_COLUMN + | WorkspaceHealthIssueType.MISSING_INDEX + | WorkspaceHealthIssueType.MISSING_FOREIGN_KEY + | WorkspaceHealthIssueType.MISSING_COMPOSITE_TYPE + | WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_PREFIXED + | WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID + | WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_BE_CUSTOM + | WorkspaceHealthIssueType.COLUMN_OBJECT_REFERENCE_INVALID + | WorkspaceHealthIssueType.COLUMN_NAME_NOT_VALID + | WorkspaceHealthIssueType.COLUMN_TYPE_NOT_VALID + | WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT + | WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT + | WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT + | WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID + | WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID; + export interface WorkspaceHealthColumnIssue< - T extends WorkspaceHealthIssueType | null = null, + T extends WorkspaceColumnIssueTypes, > { - type: ConditionalType< - T, - | WorkspaceHealthIssueType.MISSING_COLUMN - | WorkspaceHealthIssueType.MISSING_INDEX - | WorkspaceHealthIssueType.MISSING_FOREIGN_KEY - | WorkspaceHealthIssueType.MISSING_COMPOSITE_TYPE - | WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID - | WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_BE_CUSTOM - | WorkspaceHealthIssueType.COLUMN_OBJECT_REFERENCE_INVALID - | WorkspaceHealthIssueType.COLUMN_NAME_NOT_VALID - | WorkspaceHealthIssueType.COLUMN_TYPE_NOT_VALID - | WorkspaceHealthIssueType.COLUMN_DATA_TYPE_CONFLICT - | WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT - | WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT - | WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_NOT_VALID - | WorkspaceHealthIssueType.COLUMN_OPTIONS_NOT_VALID - >; + type: T; fieldMetadata: FieldMetadataEntity; columnStructure?: WorkspaceTableStructure; message: string; } +/** + * Relation issues + */ +export type WorkspaceRelationIssueTypes = + | WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID + | WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID + | WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT + | WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID; + export interface WorkspaceHealthRelationIssue< - T extends WorkspaceHealthIssueType | null = null, + T extends WorkspaceRelationIssueTypes, > { - type: ConditionalType< - T, - | WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID - | WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID - | WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT - | WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID - >; + type: T; fromFieldMetadata: FieldMetadataEntity | undefined; toFieldMetadata: FieldMetadataEntity | undefined; relationMetadata: RelationMetadataEntity; @@ -92,7 +96,20 @@ export interface WorkspaceHealthRelationIssue< message: string; } +/** + * Get the interface for the issue type + */ +export type WorkspaceIssueTypeToInterface = + T extends WorkspaceTableIssueTypes + ? WorkspaceHealthTableIssue + : T extends WorkspaceColumnIssueTypes + ? WorkspaceHealthColumnIssue + : T extends WorkspaceRelationIssueTypes + ? WorkspaceHealthRelationIssue + : never; + +/** + * Union of all issues + */ export type WorkspaceHealthIssue = - | WorkspaceHealthTableIssue - | WorkspaceHealthColumnIssue - | WorkspaceHealthRelationIssue; + WorkspaceIssueTypeToInterface; diff --git a/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts b/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts index d7ff40adce..60803726cd 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts @@ -125,6 +125,24 @@ export class DatabaseStructureService { })); } + async workspaceColumnExist( + schemaName: string, + tableName: string, + columnName: string, + ): Promise { + const mainDataSource = this.typeORMService.getMainDataSource(); + const results = await mainDataSource.query( + `SELECT column_name + FROM information_schema.columns + WHERE table_schema = $1 + AND table_name = $2 + AND column_name = $3`, + [schemaName, tableName, columnName], + ); + + return results.length >= 1; + } + getPostgresDataType(fieldMetadata: FieldMetadataEntity): string { const typeORMType = fieldMetadataTypeToColumnType(fieldMetadata.type); const mainDataSource = this.typeORMService.getMainDataSource(); diff --git a/packages/twenty-server/src/workspace/workspace-health/services/field-metadata-health.service.ts b/packages/twenty-server/src/workspace/workspace-health/services/field-metadata-health.service.ts index 6e14db6e97..40a82a8398 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/field-metadata-health.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/services/field-metadata-health.service.ts @@ -1,5 +1,7 @@ import { Injectable, InternalServerErrorException } from '@nestjs/common'; +import isEqual from 'lodash.isequal'; + import { WorkspaceHealthIssue, WorkspaceHealthIssueType, @@ -23,6 +25,9 @@ import { } from 'src/metadata/field-metadata/utils/is-enum-field-metadata-type.util'; import { validateOptionsForType } from 'src/metadata/field-metadata/utils/validate-options-for-type.util'; import { serializeDefaultValue } from 'src/metadata/field-metadata/utils/serialize-default-value'; +import { computeCompositeFieldMetadata } from 'src/workspace/workspace-health/utils/compute-composite-field-metadata.util'; +import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util'; +import { customNamePrefix } from 'src/workspace/utils/compute-custom-name.util'; @Injectable() export class FieldMetadataHealthService { @@ -52,10 +57,11 @@ export class FieldMetadataHealthService { compositeDefinitions.get(fieldMetadata.type)?.(fieldMetadata) ?? []; if (options.mode === 'metadata' || options.mode === 'all') { - const targetColumnMapIssues = - this.targetColumnMapCheck(fieldMetadata); + const targetColumnMapIssue = this.targetColumnMapCheck(fieldMetadata); - issues.push(...targetColumnMapIssues); + if (targetColumnMapIssue) { + issues.push(targetColumnMapIssue); + } const defaultValueIssues = this.defaultValueHealthCheck(fieldMetadata); @@ -67,7 +73,10 @@ export class FieldMetadataHealthService { const compositeFieldIssues = await this.healthCheckField( tableName, workspaceTableColumns, - compositeFieldMetadata as FieldMetadataEntity, + computeCompositeFieldMetadata( + compositeFieldMetadata, + fieldMetadata, + ), options, ); @@ -169,11 +178,7 @@ export class FieldMetadataHealthService { }); } - if ( - defaultValue && - columnDefaultValue && - isEnumFieldMetadataType(fieldMetadata.type) - ) { + if (columnDefaultValue && isEnumFieldMetadataType(fieldMetadata.type)) { const enumValues = fieldMetadata.options?.map((option) => serializeDefaultValue(option.value), ); @@ -188,11 +193,7 @@ export class FieldMetadataHealthService { } } - if ( - defaultValue && - columnDefaultValue && - columnDefaultValue !== defaultValue - ) { + if (columnDefaultValue !== defaultValue) { issues.push({ type: WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT, fieldMetadata, @@ -210,20 +211,22 @@ export class FieldMetadataHealthService { ): WorkspaceHealthIssue[] { const issues: WorkspaceHealthIssue[] = []; const columnName = fieldMetadata.targetColumnMap.value; - const targetColumnMapIssues = this.targetColumnMapCheck(fieldMetadata); + const targetColumnMapIssue = this.targetColumnMapCheck(fieldMetadata); const defaultValueIssues = this.defaultValueHealthCheck(fieldMetadata); - if (Object.keys(fieldMetadata.targetColumnMap).length !== 1) { + if (targetColumnMapIssue) { + issues.push(targetColumnMapIssue); + } + + if (fieldMetadata.name.startsWith(customNamePrefix)) { issues.push({ - type: WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID, + type: WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_NOT_BE_PREFIXED, fieldMetadata, - message: `Column ${columnName} has more than one target column map, it should only contains "value"`, + message: `Column ${columnName} should not be prefixed with "${customNamePrefix}"`, }); } - issues.push(...targetColumnMapIssues); - - if (fieldMetadata.isCustom && !columnName?.startsWith('_')) { + if (fieldMetadata.isCustom && !columnName?.startsWith(customNamePrefix)) { issues.push({ type: WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_BE_CUSTOM, fieldMetadata, @@ -277,46 +280,29 @@ export class FieldMetadataHealthService { private targetColumnMapCheck( fieldMetadata: FieldMetadataEntity, - ): WorkspaceHealthIssue[] { - const issues: WorkspaceHealthIssue[] = []; - - if (!fieldMetadata.targetColumnMap) { - issues.push({ - type: WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID, - fieldMetadata, - message: `Column targetColumnMap of ${fieldMetadata.name} is empty`, - }); - } - - if (!isCompositeFieldMetadataType(fieldMetadata.type)) { - if ( - Object.keys(fieldMetadata.targetColumnMap).length !== 1 && - !('value' in fieldMetadata.targetColumnMap) - ) { - issues.push({ - type: WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID, - fieldMetadata, - message: `Column targetColumnMap "${fieldMetadata.targetColumnMap}" is not valid or well structured`, - }); - } - - return issues; - } + ): WorkspaceHealthIssue | null { + const targetColumnMap = generateTargetColumnMap( + fieldMetadata.type, + fieldMetadata.isCustom, + fieldMetadata.name, + ); if ( - !this.isCompositeObjectWellStructured( - fieldMetadata.type, - fieldMetadata.targetColumnMap, - ) + !fieldMetadata.targetColumnMap || + !isEqual(targetColumnMap, fieldMetadata.targetColumnMap) ) { - issues.push({ + return { type: WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID, fieldMetadata, - message: `Column targetColumnMap for composite type ${fieldMetadata.type} is not well structured "${fieldMetadata.targetColumnMap}"`, - }); + message: `Column targetColumnMap "${JSON.stringify( + fieldMetadata.targetColumnMap, + )}" is not the same as the generated one "${JSON.stringify( + targetColumnMap, + )}"`, + }; } - return issues; + return null; } private defaultValueHealthCheck( diff --git a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix.service.ts b/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix.service.ts index ce55285ccb..9ed85195e2 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/services/workspace-fix.service.ts @@ -7,55 +7,92 @@ import { WorkspaceHealthIssue } from 'src/workspace/workspace-health/interfaces/ import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; -import { - isWorkspaceHealthDefaultValueIssue, - isWorkspaceHealthNullableIssue, - isWorkspaceHealthTypeIssue, -} from 'src/workspace/workspace-health/utils/is-workspace-health-issue-type.util'; - -import { WorkspaceFixNullableService } from './workspace-fix-nullable.service'; -import { WorkspaceFixTypeService } from './workspace-fix-type.service'; -import { WorkspaceFixDefaultValueService } from './workspace-fix-default-value.service'; +import { WorkspaceNullableFixer } from 'src/workspace/workspace-health/fixer/workspace-nullable.fixer'; +import { WorkspaceDefaultValueFixer } from 'src/workspace/workspace-health/fixer/workspace-default-value.fixer'; +import { WorkspaceTypeFixer } from 'src/workspace/workspace-health/fixer/workspace-type.fixer'; +import { WorkspaceTargetColumnMapFixer } from 'src/workspace/workspace-health/fixer/workspace-target-column-map.fixer'; +import { CompareEntity } from 'src/workspace/workspace-health/fixer/abstract-workspace.fixer'; @Injectable() export class WorkspaceFixService { constructor( - private readonly workspaceFixNullableService: WorkspaceFixNullableService, - private readonly workspaceFixTypeService: WorkspaceFixTypeService, - private readonly workspaceFixDefaultValueService: WorkspaceFixDefaultValueService, + private readonly workspaceNullableFixer: WorkspaceNullableFixer, + private readonly workspaceDefaultValueFixer: WorkspaceDefaultValueFixer, + private readonly workspaceTypeFixer: WorkspaceTypeFixer, + private readonly workspaceTargetColumnMapFixer: WorkspaceTargetColumnMapFixer, ) {} - async fix( + async createWorkspaceMigrations( manager: EntityManager, objectMetadataCollection: ObjectMetadataEntity[], type: WorkspaceHealthFixKind, issues: WorkspaceHealthIssue[], ): Promise[]> { - const services = { - [WorkspaceHealthFixKind.Nullable]: { - service: this.workspaceFixNullableService, - issues: issues.filter((issue) => - isWorkspaceHealthNullableIssue(issue.type), - ), - }, - [WorkspaceHealthFixKind.Type]: { - service: this.workspaceFixTypeService, - issues: issues.filter((issue) => - isWorkspaceHealthTypeIssue(issue.type), - ), - }, - [WorkspaceHealthFixKind.DefaultValue]: { - service: this.workspaceFixDefaultValueService, - issues: issues.filter((issue) => - isWorkspaceHealthDefaultValueIssue(issue.type), - ), - }, - }; + switch (type) { + case WorkspaceHealthFixKind.Nullable: { + const filteredIssues = this.workspaceNullableFixer.filterIssues(issues); - return services[type].service.fix( - manager, - objectMetadataCollection, - services[type].issues, - ); + return this.workspaceNullableFixer.createWorkspaceMigrations( + manager, + objectMetadataCollection, + filteredIssues, + ); + } + case WorkspaceHealthFixKind.DefaultValue: { + const filteredIssues = + this.workspaceDefaultValueFixer.filterIssues(issues); + + return this.workspaceDefaultValueFixer.createWorkspaceMigrations( + manager, + objectMetadataCollection, + filteredIssues, + ); + } + case WorkspaceHealthFixKind.Type: { + const filteredIssues = this.workspaceTypeFixer.filterIssues(issues); + + return this.workspaceTypeFixer.createWorkspaceMigrations( + manager, + objectMetadataCollection, + filteredIssues, + ); + } + case WorkspaceHealthFixKind.TargetColumnMap: { + const filteredIssues = + this.workspaceTargetColumnMapFixer.filterIssues(issues); + + return this.workspaceTargetColumnMapFixer.createWorkspaceMigrations( + manager, + objectMetadataCollection, + filteredIssues, + ); + } + default: { + return []; + } + } + } + + async createMetadataUpdates( + manager: EntityManager, + objectMetadataCollection: ObjectMetadataEntity[], + type: WorkspaceHealthFixKind, + issues: WorkspaceHealthIssue[], + ): Promise[]> { + switch (type) { + case WorkspaceHealthFixKind.TargetColumnMap: { + const filteredIssues = + this.workspaceTargetColumnMapFixer.filterIssues(issues); + + return this.workspaceTargetColumnMapFixer.createMetadataUpdates( + manager, + objectMetadataCollection, + filteredIssues, + ); + } + default: { + return []; + } + } } } diff --git a/packages/twenty-server/src/workspace/workspace-health/utils/compute-composite-field-metadata.util.ts b/packages/twenty-server/src/workspace/workspace-health/utils/compute-composite-field-metadata.util.ts new file mode 100644 index 0000000000..21ddb271be --- /dev/null +++ b/packages/twenty-server/src/workspace/workspace-health/utils/compute-composite-field-metadata.util.ts @@ -0,0 +1,15 @@ +import { FieldMetadataInterface } from 'src/metadata/field-metadata/interfaces/field-metadata.interface'; + +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { camelCase } from 'src/utils/camel-case'; + +// Compute composite field metadata by combining the composite field metadata with the field metadata +export const computeCompositeFieldMetadata = ( + compositeFieldMetadata: FieldMetadataInterface, + fieldMetadata: FieldMetadataEntity, +): FieldMetadataEntity => ({ + ...fieldMetadata, + ...compositeFieldMetadata, + objectMetadataId: fieldMetadata.objectMetadataId, + name: camelCase(`${fieldMetadata.name}-${compositeFieldMetadata.name}`), +}); diff --git a/packages/twenty-server/src/workspace/workspace-health/utils/is-workspace-health-issue-type.util.ts b/packages/twenty-server/src/workspace/workspace-health/utils/is-workspace-health-issue-type.util.ts index f0e1cb7f5a..0d946000f4 100644 --- a/packages/twenty-server/src/workspace/workspace-health/utils/is-workspace-health-issue-type.util.ts +++ b/packages/twenty-server/src/workspace/workspace-health/utils/is-workspace-health-issue-type.util.ts @@ -17,3 +17,9 @@ export const isWorkspaceHealthDefaultValueIssue = ( ): type is WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT => { return type === WorkspaceHealthIssueType.COLUMN_DEFAULT_VALUE_CONFLICT; }; + +export const isWorkspaceHealthTargetColumnMapIssue = ( + type: WorkspaceHealthIssueType, +): type is WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID => { + return type === WorkspaceHealthIssueType.COLUMN_TARGET_COLUMN_MAP_NOT_VALID; +}; diff --git a/packages/twenty-server/src/workspace/workspace-health/workspace-health.module.ts b/packages/twenty-server/src/workspace/workspace-health/workspace-health.module.ts index 7b45f89356..296f92b2f1 100644 --- a/packages/twenty-server/src/workspace/workspace-health/workspace-health.module.ts +++ b/packages/twenty-server/src/workspace/workspace-health/workspace-health.module.ts @@ -12,10 +12,9 @@ import { WorkspaceHealthService } from 'src/workspace/workspace-health/workspace import { WorkspaceMigrationBuilderModule } from 'src/workspace/workspace-migration-builder/workspace-migration-builder.module'; import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module'; +import { workspaceFixers } from './fixer'; + import { WorkspaceFixService } from './services/workspace-fix.service'; -import { WorkspaceFixNullableService } from './services/workspace-fix-nullable.service'; -import { WorkspaceFixTypeService } from './services/workspace-fix-type.service'; -import { WorkspaceFixDefaultValueService } from './services/workspace-fix-default-value.service'; @Module({ imports: [ @@ -27,14 +26,12 @@ import { WorkspaceFixDefaultValueService } from './services/workspace-fix-defaul WorkspaceMigrationBuilderModule, ], providers: [ + ...workspaceFixers, WorkspaceHealthService, DatabaseStructureService, ObjectMetadataHealthService, FieldMetadataHealthService, RelationMetadataHealthService, - WorkspaceFixNullableService, - WorkspaceFixTypeService, - WorkspaceFixDefaultValueService, WorkspaceFixService, ], exports: [WorkspaceHealthService], diff --git a/packages/twenty-server/src/workspace/workspace-health/workspace-health.service.ts b/packages/twenty-server/src/workspace/workspace-health/workspace-health.service.ts index cbd8a146a8..d7c6c87790 100644 --- a/packages/twenty-server/src/workspace/workspace-health/workspace-health.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/workspace-health.service.ts @@ -125,8 +125,12 @@ export class WorkspaceHealthService { type: WorkspaceHealthFixKind; applyChanges?: boolean; }, - ): Promise[]> { + ): Promise<{ + workspaceMigrations: Partial[]; + metadataEntities: unknown[]; + }> { let workspaceMigrations: Partial[] = []; + let metadataEntities: unknown[] = []; // Set default options options.applyChanges ??= true; @@ -145,7 +149,15 @@ export class WorkspaceHealthService { const objectMetadataCollection = await this.objectMetadataService.findManyWithinWorkspace(workspaceId); - workspaceMigrations = await this.workspaceFixService.fix( + workspaceMigrations = + await this.workspaceFixService.createWorkspaceMigrations( + manager, + objectMetadataCollection, + options.type, + issues, + ); + + metadataEntities = await this.workspaceFixService.createMetadataUpdates( manager, objectMetadataCollection, options.type, @@ -161,7 +173,10 @@ export class WorkspaceHealthService { await queryRunner.release(); - return workspaceMigrations; + return { + workspaceMigrations, + metadataEntities, + }; } // Commit the transaction @@ -178,6 +193,9 @@ export class WorkspaceHealthService { await queryRunner.release(); } - return workspaceMigrations; + return { + workspaceMigrations, + metadataEntities, + }; } } diff --git a/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts b/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts index bb16d7a547..aedb33f478 100644 --- a/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts +++ b/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts @@ -120,6 +120,7 @@ export class WorkspaceMigrationFieldFactory { const workspaceMigrations: Partial[] = []; for (const fieldMetadataUpdate of fieldMetadataUpdateCollection) { + // Skip relations, because they're just representation and not real columns if (fieldMetadataUpdate.altered.type === FieldMetadataType.RELATION) { continue; } diff --git a/packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts index 3b976978d0..ac7af163fd 100644 --- a/packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts @@ -268,7 +268,7 @@ export class WorkspaceMigrationRunnerService { // TODO: Maybe we can do something better if we can recreate the old `TableColumn` object if (enumValues) { - // This is returning the old enum values to avoid TypeORM droping the enum type + // This is returning the old enum values to avoid TypeORM dropping the enum type await this.workspaceMigrationEnumService.alterEnum( queryRunner, schemaName, diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts index 74cdf2feb5..d8962e4a04 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts @@ -1,7 +1,10 @@ +import { Logger } from '@nestjs/common'; + import { Command, CommandRunner, Option } from 'nest-commander'; import { DataSourceService } from 'src/metadata/data-source/data-source.service'; import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service'; +import { WorkspaceHealthService } from 'src/workspace/workspace-health/workspace-health.service'; import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; @@ -9,6 +12,7 @@ import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.ser interface RunWorkspaceMigrationsOptions { workspaceId: string; dryRun?: boolean; + force?: boolean; } @Command({ @@ -16,8 +20,11 @@ interface RunWorkspaceMigrationsOptions { description: 'Sync metadata', }) export class SyncWorkspaceMetadataCommand extends CommandRunner { + private readonly logger = new Logger(SyncWorkspaceMetadataCommand.name); + constructor( private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, + private readonly workspaceHealthService: WorkspaceHealthService, private readonly dataSourceService: DataSourceService, private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService, ) { @@ -28,7 +35,30 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { _passedParam: string[], options: RunWorkspaceMigrationsOptions, ): Promise { - // TODO: run in a dedicated job + run queries in a transaction. + const issues = await this.workspaceHealthService.healthCheck( + options.workspaceId, + ); + + // Security: abort if there are issues. + if (issues.length > 0) { + if (!options.force) { + this.logger.error( + `Workspace contains ${issues.length} issues, aborting.`, + ); + + this.logger.log('If you want to force the migration, use --force flag'); + this.logger.log( + 'Please use `workspace:health` command to check issues and fix them before running this command.', + ); + + return; + } + + this.logger.warn( + `Workspace contains ${issues.length} issues, sync has been forced.`, + ); + } + const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( options.workspaceId, @@ -68,4 +98,13 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { dryRun(): boolean { return true; } + + @Option({ + flags: '-f, --force', + description: 'Force migration', + required: false, + }) + force(): boolean { + return true; + } } diff --git a/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts b/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts index a59e66f625..f8ac2f6580 100644 --- a/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts +++ b/packages/twenty-server/src/workspace/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module.ts @@ -2,13 +2,18 @@ import { Module } from '@nestjs/common'; import { DataSourceModule } from 'src/metadata/data-source/data-source.module'; import { WorkspaceSyncMetadataModule } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.module'; +import { WorkspaceHealthModule } from 'src/workspace/workspace-health/workspace-health.module'; import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command'; import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; @Module({ - imports: [WorkspaceSyncMetadataModule, DataSourceModule], + imports: [ + WorkspaceSyncMetadataModule, + WorkspaceHealthModule, + DataSourceModule, + ], providers: [SyncWorkspaceMetadataCommand, SyncWorkspaceLoggerService], }) export class WorkspaceSyncMetadataCommandsModule {} diff --git a/yarn.lock b/yarn.lock index 3cd47a7fb8..f5d481448e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15577,7 +15577,7 @@ __metadata: languageName: node linkType: hard -"@types/lodash.isequal@npm:^4.5.7": +"@types/lodash.isequal@npm:^4.5.7, @types/lodash.isequal@npm:^4.5.8": version: 4.5.8 resolution: "@types/lodash.isequal@npm:4.5.8" dependencies: @@ -44115,6 +44115,7 @@ __metadata: "@nx/js": "npm:17.2.8" "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/nestjs-query-graphql@4.2.0#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch" "@types/lodash.isempty": "npm:^4.4.7" + "@types/lodash.isequal": "npm:^4.5.8" "@types/lodash.isobject": "npm:^3.0.7" "@types/lodash.omit": "npm:^4.5.9" "@types/lodash.snakecase": "npm:^4.1.7" @@ -44122,6 +44123,7 @@ __metadata: "@types/react": "npm:^18.2.39" class-validator: "patch:class-validator@0.14.0#./patches/class-validator+0.14.0.patch" graphql-middleware: "npm:^6.1.35" + lodash.isequal: "npm:^4.5.0" passport: "npm:^0.7.0" rimraf: "npm:^5.0.5" typescript: "npm:^5.3.3"