twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-08-18 02:10:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: workspace sync (#3505)

Browse Source 
* feat: wip workspace sync

* feat: wip lot of debugging

* feat: refactor and fix sync

* fix: clean

fix: clean

* feat: add simple comparator tests

* fix: remove debug

* feat: wip drop table

* fix: main merge

* fix: some issues, and prepare storage system to handle complex deletion

* feat: wip clean and fix

* fix: reflect issue when using array instead of map and clean

* fix: test & sync

* fix: yarn files

* fix: unecesary if-else

* fix: if condition not needed

* fix: remove debug

* fix: replace EQUAL by SKIP

* fix: sync metadata relation not applied properly

* fix: lint issues

* fix: merge issue
This commit is contained in:
Jérémy M 2024-01-30 14:40:55 +01:00 committed by GitHub
parent 3a480f1506
commit 73f6876641
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
59 changed files with 2103 additions and 927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-server/package.json
View File

@ -78,6 +78,7 @@
"lodash.isempty": "^4.4.0",
"lodash.isobject": "^3.0.2",
"lodash.kebabcase": "^4.1.1",
"lodash.omit": "^4.5.0",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"mailparser": "^3.6.5",
@ -101,6 +102,7 @@
"devDependencies": {
"@types/lodash.isempty": "^4.4.7",
"@types/lodash.isobject": "^3.0.7",
"@types/lodash.omit": "^4.5.9",
"@types/lodash.snakecase": "^4.1.7",
"@types/lodash.upperfirst": "^4.3.7",
"@types/react": "^18.2.39",

4
packages/twenty-server/src/core/analytics/analytics.resolver.ts
View File

@ -2,8 +2,8 @@ import { Resolver, Mutation, Args } from '@nestjs/graphql';
import { UseGuards } from '@nestjs/common';
import { OptionalJwtAuthGuard } from 'src/guards/optional-jwt.auth.guard';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthUser } from 'src/decorators/auth-user.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { AuthUser } from 'src/decorators/auth/auth-user.decorator';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { User } from 'src/core/user/user.entity';

4
packages/twenty-server/src/core/auth/auth.resolver.ts
View File

@ -11,10 +11,10 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { AuthUser } from 'src/decorators/auth-user.decorator';
import { AuthUser } from 'src/decorators/auth/auth-user.decorator';
import { assert } from 'src/utils/assert';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { User } from 'src/core/user/user.entity';
import { ApiKeyTokenInput } from 'src/core/auth/dto/api-key-token.input';
import { ValidatePasswordResetToken } from 'src/core/auth/dto/validate-password-reset-token.entity';

3
packages/twenty-server/src/core/feature-flag/interfaces/feature-flag-map.interface.ts Normal file
View File

@ -0,0 +1,3 @@
import { FeatureFlagKeys } from 'src/core/feature-flag/feature-flag.entity';
export type FeatureFlagMap = Record<`${FeatureFlagKeys}`, boolean>;

2
packages/twenty-server/src/core/messaging/timeline-messaging.resolver.ts
View File

@ -13,7 +13,7 @@ import { Max } from 'class-validator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { TimelineMessagingService } from 'src/core/messaging/timeline-messaging.service';
import { TIMELINE_THREADS_MAX_PAGE_SIZE } from 'src/core/messaging/constants/messaging.constants';
import { TimelineThreadsWithTotal } from 'src/core/messaging/dtos/timeline-threads-with-total.dto';

2
packages/twenty-server/src/core/user/user.resolver.ts
View File

@ -15,7 +15,7 @@ import { FileUpload, GraphQLUpload } from 'graphql-upload';
import { SupportDriver } from 'src/integrations/environment/interfaces/support.interface';
import { FileFolder } from 'src/core/file/interfaces/file-folder.interface';
import { AuthUser } from 'src/decorators/auth-user.decorator';
import { AuthUser } from 'src/decorators/auth/auth-user.decorator';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { FileUploadService } from 'src/core/file/services/file-upload.service';

2
packages/twenty-server/src/core/workspace/workspace.resolver.ts
View File

@ -7,7 +7,7 @@ import { FileFolder } from 'src/core/file/interfaces/file-folder.interface';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { FileUploadService } from 'src/core/file/services/file-upload.service';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { assert } from 'src/utils/assert';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { UpdateWorkspaceInput } from 'src/core/workspace/dtos/update-workspace-input';

8
packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts
View File

@ -11,7 +11,7 @@ import { seedWorkspaceMember } from 'src/database/typeorm-seeds/workspace/worksp
import { seedPeople } from 'src/database/typeorm-seeds/workspace/people';
import { seedCoreSchema } from 'src/database/typeorm-seeds/core';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service';
@ -62,8 +62,10 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
);
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
this.workspaceId,
{
workspaceId: this.workspaceId,
dataSourceId: dataSourceMetadata.id,
},
);
} catch (error) {
console.error(error);

0
packages/twenty-server/src/decorators/user-ability.decorator.ts → packages/twenty-server/src/decorators/ability/user-ability.decorator.ts
View File

0
packages/twenty-server/src/decorators/auth-user.decorator.ts → packages/twenty-server/src/decorators/auth/auth-user.decorator.ts
View File

0
packages/twenty-server/src/decorators/auth-workspace.decorator.ts → packages/twenty-server/src/decorators/auth/auth-workspace.decorator.ts
View File

6
packages/twenty-server/src/integrations/environment/environment.validation.ts
View File

@ -64,7 +64,11 @@ export class EnvironmentVariables {
PORT: number;
// Database
@IsUrl({ protocols: ['postgres'], require_tld: false, allow_underscores: true })
@IsUrl({
protocols: ['postgres'],
require_tld: false,
allow_underscores: true,
})
PG_DATABASE_URL: string;
// Frontend URL

2
packages/twenty-server/src/metadata/field-metadata/field-metadata.resolver.ts
View File

@ -2,7 +2,7 @@ import { UseGuards } from '@nestjs/common';
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { CreateOneFieldMetadataInput } from 'src/metadata/field-metadata/dtos/create-field.input';
import { FieldMetadataDTO } from 'src/metadata/field-metadata/dtos/field-metadata.dto';

3
packages/twenty-server/src/metadata/field-metadata/field-metadata.service.ts
View File

@ -24,6 +24,7 @@ import { DataSourceService } from 'src/metadata/data-source/data-source.service'
import { UpdateFieldInput } from 'src/metadata/field-metadata/dtos/update-field.input';
import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util';
import {
FieldMetadataEntity,
@ -111,6 +112,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
});
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
@ -228,6 +230,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
if (fieldMetadataInput.options || fieldMetadataInput.defaultValue) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`update-${updatedFieldMetadata.name}`),
existingFieldMetadata.workspaceId,
[
{

2
packages/twenty-server/src/metadata/object-metadata/object-metadata.resolver.ts
View File

@ -2,7 +2,7 @@ import { UseGuards } from '@nestjs/common';
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { Workspace } from 'src/core/workspace/workspace.entity';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { AuthWorkspace } from 'src/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { ObjectMetadataDTO } from 'src/metadata/object-metadata/dtos/object-metadata.dto';
import { DeleteOneObjectInput } from 'src/metadata/object-metadata/dtos/delete-object.input';

21
packages/twenty-server/src/metadata/object-metadata/object-metadata.service.ts
View File

@ -35,6 +35,7 @@ import {
} from 'src/workspace/utils/compute-object-target-table.util';
import { DeleteOneObjectInput } from 'src/metadata/object-metadata/dtos/delete-object.input';
import { RelationToDelete } from 'src/metadata/relation-metadata/types/relation-to-delete';
import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util';
import { ObjectMetadataEntity } from './object-metadata.entity';
@ -152,6 +153,9 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
if (relationToDelete.direction === 'from') {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`delete-${relationToDelete.fromObjectName}-${relationToDelete.toObjectName}`,
),
workspaceId,
[
{
@ -178,12 +182,16 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.objectMetadataRepository.delete(objectMetadata.id);
// DROP TABLE
await this.workspaceMigrationService.createCustomMigration(workspaceId, [
{
name: computeObjectTargetTable(objectMetadata),
action: 'drop',
},
]);
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`delete-${objectMetadata.nameSingular}`),
workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: 'drop',
},
],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
@ -298,6 +306,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
);
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId,
[
{

2
packages/twenty-server/src/metadata/relation-metadata/relation-metadata.service.ts
View File

@ -21,6 +21,7 @@ import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metada
import { createCustomColumnName } from 'src/metadata/utils/create-custom-column-name.util';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { createRelationForeignKeyColumnName } from 'src/metadata/relation-metadata/utils/create-relation-foreign-key-column-name.util';
import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util';
import {
RelationMetadataEntity,
@ -173,6 +174,7 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
foreignKeyColumnName: string,
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${relationMetadataInput.fromName}`),
relationMetadataInput.workspaceId,
[
// Create the column

2
packages/twenty-server/src/metadata/relation-metadata/types/relation-to-delete.ts
View File

@ -11,4 +11,4 @@ export type RelationToDelete = {
toFieldMetadataIsCustom: boolean;
toObjectMetadataIsCustom: boolean;
direction: string;
};
};

3
packages/twenty-server/src/metadata/workspace-migration/utils/generate-migration-name.util.ts Normal file
View File

@ -0,0 +1,3 @@
export function generateMigrationName(name?: string): string {
return `${new Date().getTime()}${name ? `-${name}` : ''}`;
}

2
packages/twenty-server/src/metadata/workspace-migration/workspace-migration.entity.ts
View File

@ -70,7 +70,7 @@ export class WorkspaceMigrationEntity {
@Column({ nullable: true, type: 'jsonb' })
migrations: WorkspaceMigrationTableAction[];
@Column({ nullable: true })
@Column({ nullable: false })
name: string;
@Column({ default: false })

2
packages/twenty-server/src/metadata/workspace-migration/workspace-migration.service.ts
View File

@ -57,10 +57,12 @@ export class WorkspaceMigrationService {
* @param migrations
*/
public async createCustomMigration(
name: string,
workspaceId: string,
migrations: WorkspaceMigrationTableAction[],
) {
await this.workspaceMigrationRepository.save({
name,
migrations,
workspaceId,
isCustom: true,

4
packages/twenty-server/src/utils/typed-reflect.ts
View File

@ -7,8 +7,8 @@ import { ReflectRelationMetadata } from 'src/workspace/workspace-sync-metadata/i
export interface ReflectMetadataTypeMap {
objectMetadata: ReflectObjectMetadata;
fieldMetadata: ReflectFieldMetadata;
relationMetadata: ReflectRelationMetadata[];
fieldMetadataMap: ReflectFieldMetadata;
relationMetadataCollection: ReflectRelationMetadata[];
gate: GateDecoratorParams;
isNullable: true;
isSystem: true;

14
packages/twenty-server/src/workspace/workspace-manager/workspace-manager.service.ts
View File

@ -7,7 +7,7 @@ import { standardObjectsPrefillData } from 'src/workspace/workspace-manager/stan
import { demoObjectsPrefillData } from 'src/workspace/workspace-manager/demo-objects-prefill-data/demo-objects-prefill-data';
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service';
@Injectable()
export class WorkspaceManagerService {
@ -39,8 +39,10 @@ export class WorkspaceManagerService {
await this.setWorkspaceMaxRow(workspaceId, schemaName);
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
workspaceId,
{
workspaceId,
dataSourceId: dataSourceMetadata.id,
},
);
await this.prefillWorkspaceWithStandardObjects(
@ -69,8 +71,10 @@ export class WorkspaceManagerService {
await this.setWorkspaceMaxRow(workspaceId, schemaName);
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
workspaceId,
{
workspaceId,
dataSourceId: dataSourceMetadata.id,
},
);
await this.prefillWorkspaceWithDemoObjects(dataSourceMetadata, workspaceId);

22
packages/twenty-server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts
View File

@ -63,13 +63,25 @@ export class WorkspaceMigrationRunnerService {
}, []);
const queryRunner = workspaceDataSource?.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
const schemaName =
this.workspaceDataSourceService.getSchemaName(workspaceId);
// Loop over each migration and create or update the table
// TODO: Should be done in a transaction
for (const migration of flattenedPendingMigrations) {
await this.handleTableChanges(queryRunner, schemaName, migration);
try {
// Loop over each migration and create or update the table
for (const migration of flattenedPendingMigrations) {
await this.handleTableChanges(queryRunner, schemaName, migration);
}
await queryRunner.commitTransaction();
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
}
// Update appliedAt date for each migration
@ -81,8 +93,6 @@ export class WorkspaceMigrationRunnerService {
);
}
await queryRunner.release();
// Increment workspace cache version
await this.workspaceCacheVersionService.incrementVersion(workspaceId);

8
packages/twenty-server/src/workspace/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts
View File

@ -1,7 +1,7 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSourceService } from 'src/metadata/data-source/data-source.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service';
// TODO: implement dry-run
interface RunWorkspaceMigrationsOptions {
@ -31,8 +31,10 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
);
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
options.workspaceId,
{
workspaceId: options.workspaceId,
dataSourceId: dataSourceMetadata.id,
},
);
}

106
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-field.comparator.spec.ts Normal file
View File

@ -0,0 +1,106 @@
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceFieldComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator';
describe('WorkspaceFieldComparator', () => {
let comparator: WorkspaceFieldComparator;
beforeEach(() => {
// Initialize the comparator before each test
comparator = new WorkspaceFieldComparator();
});
function createMockFieldMetadata(values: any) {
return {
workspaceId: 'some-workspace-id',
type: 'TEXT',
name: 'DefaultFieldName',
label: 'Default Field Label',
targetColumnMap: 'default_column',
defaultValue: null,
description: 'Default description',
isCustom: false,
isSystem: false,
isNullable: true,
...values,
};
}
it('should generate CREATE action for new fields', () => {
const original = { fields: [] } as any;
const standard = {
fields: [createMockFieldMetadata({ name: 'New Field' })],
} as any;
const result = comparator.compare(original, standard);
expect(result).toEqual([
{
action: ComparatorAction.CREATE,
object: expect.objectContaining(standard.fields[0]),
},
]);
});
it('should generate UPDATE action for modified fields', () => {
const original = {
fields: [
createMockFieldMetadata({
id: '1',
isNullable: true,
}),
],
} as any;
const standard = {
fields: [
createMockFieldMetadata({
isNullable: false,
}),
],
} as any;
const result = comparator.compare(original, standard);
expect(result).toEqual([
{
action: ComparatorAction.UPDATE,
object: expect.objectContaining({ id: '1', isNullable: false }),
},
]);
});
it('should generate DELETE action for removed fields', () => {
const original = {
fields: [
createMockFieldMetadata({
id: '1',
name: 'Removed Field',
isActive: true,
}),
],
} as any;
const standard = { fields: [] } as any;
const result = comparator.compare(original, standard);
expect(result).toEqual([
{
action: ComparatorAction.DELETE,
object: expect.objectContaining({ name: 'Removed Field' }),
},
]);
});
it('should not generate any action for identical fields', () => {
const original = {
fields: [createMockFieldMetadata({ id: '1', isActive: true })],
} as any;
const standard = {
fields: [createMockFieldMetadata({})],
} as any;
const result = comparator.compare(original, standard);
expect(result).toHaveLength(0);
});
});

77
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-object.comparator.spec.ts Normal file
View File

@ -0,0 +1,77 @@
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceObjectComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator';
describe('WorkspaceObjectComparator', () => {
let comparator: WorkspaceObjectComparator;
beforeEach(() => {
// Initialize the comparator before each test
comparator = new WorkspaceObjectComparator();
});
function createMockObjectMetadata(values: any) {
return {
nameSingular: 'TestObject',
namePlural: 'TestObjects',
labelSingular: 'Test Object',
labelPlural: 'Test Objects',
...values,
};
}
it('should generate CREATE action for new objects', () => {
const standardObjectMetadata = createMockObjectMetadata({
description: 'A standard object',
});
const result = comparator.compare(undefined, standardObjectMetadata);
expect(result).toEqual({
action: ComparatorAction.CREATE,
object: standardObjectMetadata,
});
});
it('should generate UPDATE action for objects with differences', () => {
const originalObjectMetadata = createMockObjectMetadata({
id: '1',
description: 'Original description',
});
const standardObjectMetadata = createMockObjectMetadata({
description: 'Updated description',
});
const result = comparator.compare(
originalObjectMetadata,
standardObjectMetadata,
);
expect(result).toEqual({
action: ComparatorAction.UPDATE,
object: expect.objectContaining({
id: '1',
description: 'Updated description',
}),
});
});
it('should generate SKIP action for identical objects', () => {
const originalObjectMetadata = createMockObjectMetadata({
id: '1',
description: 'Same description',
});
const standardObjectMetadata = createMockObjectMetadata({
description: 'Same description',
});
const result = comparator.compare(
originalObjectMetadata,
standardObjectMetadata,
);
expect(result).toEqual({
action: ComparatorAction.SKIP,
});
});
});

60
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/__tests__/workspace-relation.comparator.spec.ts Normal file
View File

@ -0,0 +1,60 @@
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceRelationComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator';
describe('WorkspaceRelationComparator', () => {
let comparator: WorkspaceRelationComparator;
beforeEach(() => {
comparator = new WorkspaceRelationComparator();
});
function createMockRelationMetadata(values: any) {
return {
fromObjectMetadataId: 'object-1',
fromFieldMetadataId: 'field-1',
...values,
};
}
it('should generate CREATE action for new relations', () => {
const original = [];
const standard = [createMockRelationMetadata({})];
const result = comparator.compare(original, standard);
expect(result).toEqual([
{
action: ComparatorAction.CREATE,
object: expect.objectContaining({
fromObjectMetadataId: 'object-1',
fromFieldMetadataId: 'field-1',
}),
},
]);
});
it('should generate DELETE action for removed relations', () => {
const original = [createMockRelationMetadata({ id: '1' })];
const standard = [];
const result = comparator.compare(original, standard);
expect(result).toEqual([
{
action: ComparatorAction.DELETE,
object: expect.objectContaining({ id: '1' }),
},
]);
});
it('should not generate any action for identical relations', () => {
const relation = createMockRelationMetadata({});
const original = [{ id: '1', ...relation }];
const standard = [relation];
const result = comparator.compare(original, standard);
expect(result).toHaveLength(0);
});
});

9
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/index.ts Normal file
View File

@ -0,0 +1,9 @@
import { WorkspaceFieldComparator } from './workspace-field.comparator';
import { WorkspaceObjectComparator } from './workspace-object.comparator';
import { WorkspaceRelationComparator } from './workspace-relation.comparator';
export const workspaceSyncMetadataComparators = [
WorkspaceFieldComparator,
WorkspaceObjectComparator,
WorkspaceRelationComparator,
];

52
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/order-object-properties.util.spec.ts Normal file
View File

@ -0,0 +1,52 @@
import { orderObjectProperties } from 'src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util';
describe('orderObjectProperties', () => {
it('orders simple object properties', () => {
const input = { b: 2, a: 1 };
const expected = { a: 1, b: 2 };
expect(orderObjectProperties(input)).toEqual(expected);
});
it('orders nested object properties', () => {
const input = { b: { d: 4, c: 3 }, a: 1 };
const expected = { a: 1, b: { c: 3, d: 4 } };
expect(orderObjectProperties(input)).toEqual(expected);
});
it('orders properties in an array of objects', () => {
const input = [
{ b: 2, a: 1 },
{ d: 4, c: 3 },
];
const expected = [
{ a: 1, b: 2 },
{ c: 3, d: 4 },
];
expect(orderObjectProperties(input)).toEqual(expected);
});
it('handles nested arrays within objects', () => {
const input = { b: [{ d: 4, c: 3 }], a: 1 };
const expected = { a: 1, b: [{ c: 3, d: 4 }] };
expect(orderObjectProperties(input)).toEqual(expected);
});
it('handles complex nested structures', () => {
const input = {
c: 3,
a: { f: [{ j: 10, i: 9 }, 8], e: 5 },
b: [7, { h: 6, g: 4 }],
};
const expected = {
a: { e: 5, f: [{ i: 9, j: 10 }, 8] },
b: [7, { g: 4, h: 6 }],
c: 3,
};
expect(orderObjectProperties(input)).toEqual(expected);
});
});

59
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/__tests__/transform-metadata-for-comparison.util.spec.ts Normal file
View File

@ -0,0 +1,59 @@
import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; // Adjust the import path as necessary
describe('transformMetadataForComparison', () => {
// Test for a single object
it('transforms a single object correctly with nested objects', () => {
const input = { name: 'Test', details: { a: 1, nested: { b: 2 } } };
const result = transformMetadataForComparison(input, {
propertiesToStringify: ['details'],
});
expect(result).toEqual({
name: 'Test',
details: '{"a":1,"nested":{"b":2}}',
});
});
// Test for an array of objects
it('transforms an array of objects correctly, ignoring and stringifying multiple properties', () => {
const input = [
{ name: 'Test1', value: { a: 1 }, ignored: 'ignoreMe' },
{ name: 'Test2', value: { c: 3 }, extra: 'keepMe' },
];
const result = transformMetadataForComparison(input, {
propertiesToIgnore: ['ignored'],
propertiesToStringify: ['value'],
keyFactory: (datum) => datum.name,
});
expect(result).toEqual({
Test1: { name: 'Test1', value: '{"a":1}' },
Test2: { name: 'Test2', value: '{"c":3}', extra: 'keepMe' },
});
});
// Test with a custom keyFactory function
it('uses a custom keyFactory function to generate keys', () => {
const input = [{ id: 123, name: 'Test' }];
const result = transformMetadataForComparison(input, {
keyFactory: (datum) => `key-${datum.id}`,
});
expect(result).toHaveProperty('key-123');
expect(result['key-123']).toEqual({ id: 123, name: 'Test' });
});
// Test with an empty array
it('handles an empty array gracefully', () => {
const result = transformMetadataForComparison([], {});
expect(result).toEqual({});
});
// Test with an empty object
it('handles an empty object gracefully', () => {
const result = transformMetadataForComparison({}, {});
expect(result).toEqual({});
});
});

21
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/order-object-properties.util.ts Normal file
View File

@ -0,0 +1,21 @@
export function orderObjectProperties<T extends object>(data: T[]): T[];
export function orderObjectProperties<T extends object>(data: T): T;
export function orderObjectProperties<T extends Array<any> | object>(
data: T,
): T {
if (Array.isArray(data)) {
return data.map(orderObjectProperties) as T;
}
if (data !== null && typeof data === 'object') {
return Object.fromEntries(
Object.entries(data)
.sort()
.map(([key, value]) => [key, orderObjectProperties(value)]),
) as T;
}
return data;
}

82
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util.ts Normal file
View File

@ -0,0 +1,82 @@
import { orderObjectProperties } from './order-object-properties.util';
type TransformToString<T, Keys extends keyof T> = {
[P in keyof T]: P extends Keys ? string : T[P];
};
// Overload for an array of T
export function transformMetadataForComparison<T, Keys extends keyof T>(
fieldMetadataCollection: T[],
options: {
propertiesToIgnore?: readonly Keys[];
propertiesToStringify?: readonly Keys[];
keyFactory: (datum: T) => string;
},
): Record<string, TransformToString<T, Keys>>;
// Overload for a single T object
export function transformMetadataForComparison<T, Keys extends keyof T>(
fieldMetadataCollection: T,
options: {
propertiesToIgnore?: readonly Keys[];
propertiesToStringify?: readonly Keys[];
},
): TransformToString<T, Keys>;
export function transformMetadataForComparison<T, Keys extends keyof T>(
metadata: T[] | T,
options: {
propertiesToIgnore?: readonly Keys[];
propertiesToStringify?: readonly Keys[];
keyFactory?: (datum: T) => string;
},
): Record<string, TransformToString<T, Keys>> | TransformToString<T, Keys> {
const propertiesToIgnore = (options.propertiesToIgnore ??
[]) as readonly string[];
const propertiesToStringify = (options.propertiesToStringify ??
[]) as readonly string[];
const transformProperties = (datum: T): TransformToString<T, Keys> => {
const transformedField = {} as TransformToString<T, Keys>;
for (const property in datum) {
if (propertiesToIgnore.includes(property)) {
continue;
}
if (
propertiesToStringify.includes(property) &&
datum[property] !== null &&
typeof datum[property] === 'object'
) {
const orderedValue = orderObjectProperties(datum[property] as object);
transformedField[property as string] = JSON.stringify(
orderedValue,
) as T[Keys];
} else {
transformedField[property as string] = datum[property];
}
}
return transformedField;
};
if (Array.isArray(metadata)) {
return metadata.reduce<Record<string, TransformToString<T, Keys>>>(
(acc, datum) => {
const key = options.keyFactory?.(datum);
if (!key) {
throw new Error('keyFactory must be implemented');
}
acc[key] = transformProperties(datum);
return acc;
},
{},
);
} else {
return transformProperties(metadata);
}
}

166
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator.ts Normal file
View File

@ -0,0 +1,166 @@
import { Injectable } from '@nestjs/common';
import diff from 'microdiff';
import {
ComparatorAction,
FieldComparatorResult,
} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';
const fieldPropertiesToIgnore = [
'id',
'createdAt',
'updatedAt',
'objectMetadataId',
'isActive',
] as const;
const fieldPropertiesToStringify = [
'targetColumnMap',
'defaultValue',
'options',
] as const;
@Injectable()
export class WorkspaceFieldComparator {
constructor() {}
public compare(
originalObjectMetadata: ObjectMetadataEntity,
standardObjectMetadata: PartialObjectMetadata,
): FieldComparatorResult[] {
const result: FieldComparatorResult[] = [];
const fieldPropertiesToUpdateMap: Record<
string,
Partial<PartialFieldMetadata>
> = {};
const originalFieldMetadataMap = transformMetadataForComparison(
originalObjectMetadata.fields,
{
propertiesToIgnore: fieldPropertiesToIgnore,
propertiesToStringify: fieldPropertiesToStringify,
keyFactory(datum) {
return datum.name;
},
},
);
const standardFieldMetadataMap = transformMetadataForComparison(
standardObjectMetadata.fields,
{
propertiesToIgnore: fieldPropertiesToStringify,
keyFactory(datum) {
return datum.name;
},
},
);
// Compare fields
const fieldMetadataDifference = diff(
originalFieldMetadataMap,
standardFieldMetadataMap,
);
for (const difference of fieldMetadataDifference) {
const fieldName = difference.path[0];
// Object shouldn't have thousands of fields, so we can use find here
const standardFieldMetadata = standardObjectMetadata.fields.find(
(field) => field.name === fieldName,
);
const originalFieldMetadata = originalObjectMetadata.fields.find(
(field) => field.name === fieldName,
);
switch (difference.type) {
case 'CREATE': {
if (!standardFieldMetadata) {
throw new Error(
`Field ${fieldName} not found in standardObjectMetadata`,
);
}
result.push({
action: ComparatorAction.CREATE,
object: {
...standardFieldMetadata,
objectMetadataId: originalObjectMetadata.id,
},
});
break;
}
case 'CHANGE': {
if (!originalFieldMetadata) {
throw new Error(
`Field ${fieldName} not found in originalObjectMetadata`,
);
}
const id = originalFieldMetadata.id;
const property = difference.path[difference.path.length - 1];
// If the old value and the new value are both null, skip
// Database is storing null, and we can get undefined here
if (
difference.oldValue === null &&
(difference.value === null || difference.value === undefined)
) {
break;
}
if (typeof property !== 'string') {
break;
}
if (!fieldPropertiesToUpdateMap[id]) {
fieldPropertiesToUpdateMap[id] = {};
}
// If the property is a stringified JSON, parse it
if (
(fieldPropertiesToStringify as readonly string[]).includes(property)
) {
fieldPropertiesToUpdateMap[id][property] = JSON.parse(
difference.value,
);
} else {
fieldPropertiesToUpdateMap[id][property] = difference.value;
}
break;
}
case 'REMOVE': {
if (!originalFieldMetadata) {
throw new Error(
`Field ${fieldName} not found in originalObjectMetadata`,
);
}
if (difference.path.length === 1) {
result.push({
action: ComparatorAction.DELETE,
object: originalFieldMetadata,
});
}
break;
}
}
}
for (const [id, fieldPropertiesToUpdate] of Object.entries(
fieldPropertiesToUpdateMap,
)) {
result.push({
action: ComparatorAction.UPDATE,
object: {
id,
...fieldPropertiesToUpdate,
},
});
}
return result;
}
}

83
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator.ts Normal file
View File

@ -0,0 +1,83 @@
import { Injectable } from '@nestjs/common';
import diff from 'microdiff';
import omit from 'lodash.omit';
import {
ComparatorAction,
ObjectComparatorResult,
} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
const objectPropertiesToIgnore = [
'id',
'createdAt',
'updatedAt',
'labelIdentifierFieldMetadataId',
'imageIdentifierFieldMetadataId',
'isActive',
'fields',
] as const;
@Injectable()
export class WorkspaceObjectComparator {
constructor() {}
public compare(
originalObjectMetadata: ObjectMetadataEntity | undefined,
standardObjectMetadata: PartialObjectMetadata,
): ObjectComparatorResult {
// If the object doesn't exist in the original metadata, we need to create it
if (!originalObjectMetadata) {
return {
action: ComparatorAction.CREATE,
object: standardObjectMetadata,
};
}
const objectPropertiesToUpdate: Partial<PartialObjectMetadata> = {};
// Only compare properties that are not ignored
const partialOriginalObjectMetadata = transformMetadataForComparison(
originalObjectMetadata,
{
propertiesToIgnore: objectPropertiesToIgnore,
},
);
// Compare objects
const objectMetadataDifference = diff(
partialOriginalObjectMetadata,
omit(standardObjectMetadata, 'fields'),
);
// Loop through the differences and create an object with the properties to update
for (const difference of objectMetadataDifference) {
// We only handle CHANGE here as REMOVE and CREATE are handled earlier.
if (difference.type === 'CHANGE') {
const property = difference.path[0];
objectPropertiesToUpdate[property] = difference.value;
}
}
// If there are no properties to update, the objects are equal
if (Object.keys(objectPropertiesToUpdate).length === 0) {
return {
action: ComparatorAction.SKIP,
};
}
// If there are properties to update, we need to update the object
return {
action: ComparatorAction.UPDATE,
object: {
id: originalObjectMetadata.id,
...objectPropertiesToUpdate,
},
};
}
}

72
packages/twenty-server/src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator.ts Normal file
View File

@ -0,0 +1,72 @@
import { Injectable } from '@nestjs/common';
import diff from 'microdiff';
import {
ComparatorAction,
RelationComparatorResult,
} from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
import { transformMetadataForComparison } from 'src/workspace/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';
const relationPropertiesToIgnore = ['createdAt', 'updatedAt'] as const;
@Injectable()
export class WorkspaceRelationComparator {
constructor() {}
compare(
originalRelationMetadataCollection: RelationMetadataEntity[],
standardRelationMetadataCollection: Partial<RelationMetadataEntity>[],
): RelationComparatorResult[] {
const results: RelationComparatorResult[] = [];
// Create a map of standard relations
const standardRelationMetadataMap = transformMetadataForComparison(
standardRelationMetadataCollection,
{
keyFactory(relationMetadata) {
return `${relationMetadata.fromObjectMetadataId}->${relationMetadata.fromFieldMetadataId}`;
},
},
);
// Create a filtered map of original relations
// We filter out 'id' later because we need it to remove the relation from DB
const originalRelationMetadataMap = transformMetadataForComparison(
originalRelationMetadataCollection,
{
propertiesToIgnore: relationPropertiesToIgnore,
keyFactory(relationMetadata) {
return `${relationMetadata.fromObjectMetadataId}->${relationMetadata.fromFieldMetadataId}`;
},
},
);
// Compare relations
const relationMetadataDifference = diff(
originalRelationMetadataMap,
standardRelationMetadataMap,
);
for (const difference of relationMetadataDifference) {
if (difference.type === 'CREATE') {
results.push({
action: ComparatorAction.CREATE,
object: difference.value,
});
} else if (
difference.type === 'REMOVE' &&
difference.path[difference.path.length - 1] !== 'id'
) {
results.push({
action: ComparatorAction.DELETE,
object: difference.oldValue,
});
}
}
return results;
}
}

16
packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/field-metadata.decorator.ts
View File

@ -3,6 +3,7 @@ import {
ReflectFieldMetadata,
} from 'src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface';
import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface';
import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util';
@ -14,7 +15,7 @@ export function FieldMetadata<T extends FieldMetadataType>(
): PropertyDecorator {
return (target: object, fieldKey: string) => {
const existingFieldMetadata =
TypedReflect.getMetadata('fieldMetadata', target.constructor) ?? {};
TypedReflect.getMetadata('fieldMetadataMap', target.constructor) ?? {};
const isNullable =
TypedReflect.getMetadata('isNullable', target, fieldKey) ?? false;
const isSystem =
@ -23,7 +24,7 @@ export function FieldMetadata<T extends FieldMetadataType>(
const { joinColumn, ...restParams } = params;
TypedReflect.defineMetadata(
'fieldMetadata',
'fieldMetadataMap',
{
...existingFieldMetadata,
[fieldKey]: generateFieldMetadata<T>(
@ -65,19 +66,22 @@ function generateFieldMetadata<T extends FieldMetadataType>(
gate: GateDecoratorParams | undefined = undefined,
): ReflectFieldMetadata[string] {
const targetColumnMap = generateTargetColumnMap(params.type, false, fieldKey);
const defaultValue = params.defaultValue ?? generateDefaultValue(params.type);
const defaultValue = (params.defaultValue ??
generateDefaultValue(
params.type,
)) as FieldMetadataDefaultValue<'default'> | null;
return {
name: fieldKey,
...params,
targetColumnMap: JSON.stringify(targetColumnMap),
targetColumnMap,
isNullable: params.type === FieldMetadataType.RELATION ? true : isNullable,
isSystem,
isCustom: false,
options: params.options ? JSON.stringify(params.options) : null,
options: params.options,
description: params.description,
icon: params.icon,
defaultValue: defaultValue ? JSON.stringify(defaultValue) : null,
defaultValue,
gate,
};
}

11
packages/twenty-server/src/workspace/workspace-sync-metadata/decorators/relation-metadata.decorator.ts
View File

@ -9,17 +9,20 @@ export function RelationMetadata(
params: RelationMetadataDecoratorParams,
): PropertyDecorator {
return (target: object, fieldKey: string) => {
const existingRelationMetadata =
TypedReflect.getMetadata('relationMetadata', target.constructor) ?? [];
const relationMetadataCollection =
TypedReflect.getMetadata(
'relationMetadataCollection',
target.constructor,
) ?? [];
const gate = TypedReflect.getMetadata('gate', target, fieldKey);
const objectName = convertClassNameToObjectMetadataName(
target.constructor.name,
);
Reflect.defineMetadata(
'relationMetadata',
'relationMetadataCollection',
[
...existingRelationMetadata,
...relationMetadataCollection,
{
type: params.type,
fromObjectNameSingular: objectName,

34
packages/twenty-server/src/workspace/workspace-sync-metadata/factories/feature-flags.factory.ts Normal file
View File

@ -0,0 +1,34 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
@Injectable()
export class FeatureFlagFactory {
constructor(
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {}
async create(context: WorkspaceSyncContext): Promise<FeatureFlagMap> {
const workspaceFeatureFlags = await this.featureFlagRepository.find({
where: { workspaceId: context.workspaceId },
});
const workspaceFeatureFlagsMap = workspaceFeatureFlags.reduce(
(result, currentFeatureFlag) => {
result[currentFeatureFlag.key] = currentFeatureFlag.value;
return result;
},
{} as FeatureFlagMap,
);
return workspaceFeatureFlagsMap;
}
}

11
packages/twenty-server/src/workspace/workspace-sync-metadata/factories/index.ts Normal file
View File

@ -0,0 +1,11 @@
import { FeatureFlagFactory } from './feature-flags.factory';
import { StandardObjectFactory } from './standard-object.factory';
import { StandardRelationFactory } from './standard-relation.factory';
import { WorkspaceSyncFactory } from './workspace-sync.factory';
export const workspaceSyncMetadataFactories = [
FeatureFlagFactory,
StandardObjectFactory,
StandardRelationFactory,
WorkspaceSyncFactory,
];

70
packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-object.factory.ts Normal file
View File

@ -0,0 +1,70 @@
import { Injectable } from '@nestjs/common';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects';
import { TypedReflect } from 'src/utils/typed-reflect';
import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util';
@Injectable()
export class StandardObjectFactory {
create(
context: WorkspaceSyncContext,
workspaceFeatureFlagsMap: FeatureFlagMap,
): PartialObjectMetadata[] {
return standardObjectMetadata
.map((metadata) =>
this.createObjectMetadata(metadata, context, workspaceFeatureFlagsMap),
)
.filter((metadata): metadata is PartialObjectMetadata => !!metadata);
}
private createObjectMetadata(
metadata: typeof BaseObjectMetadata,
context: WorkspaceSyncContext,
workspaceFeatureFlagsMap: FeatureFlagMap,
): PartialObjectMetadata | undefined {
const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata);
const fieldMetadataMap =
TypedReflect.getMetadata('fieldMetadataMap', metadata) ?? [];
if (!objectMetadata) {
throw new Error(
`Object metadata decorator not found, can\'t parse ${metadata.name}`,
);
}
if (isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap)) {
return undefined;
}
const fields = Object.values(fieldMetadataMap).reduce(
// Omit gate as we don't want to store it in the DB
(acc, { gate, ...fieldMetadata }) => {
if (isGatedAndNotEnabled(gate, workspaceFeatureFlagsMap)) {
return acc;
}
acc.push({
...fieldMetadata,
workspaceId: context.workspaceId,
isSystem: objectMetadata.isSystem || fieldMetadata.isSystem,
});
return acc;
},
[] as PartialFieldMetadata[],
);
return {
...objectMetadata,
workspaceId: context.workspaceId,
dataSourceId: context.dataSourceId,
fields,
};
}
}

116
packages/twenty-server/src/workspace/workspace-sync-metadata/factories/standard-relation.factory.ts Normal file
View File

@ -0,0 +1,116 @@
import { Injectable } from '@nestjs/common';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects';
import { TypedReflect } from 'src/utils/typed-reflect';
import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util';
import { assert } from 'src/utils/assert';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
@Injectable()
export class StandardRelationFactory {
create(
context: WorkspaceSyncContext,
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Partial<RelationMetadataEntity>[] {
return standardObjectMetadata.flatMap((standardObjectMetadata) =>
this.createRelationMetadata(
standardObjectMetadata,
context,
originalObjectMetadataMap,
workspaceFeatureFlagsMap,
),
);
}
private createRelationMetadata(
standardObjectMetadata: typeof BaseObjectMetadata,
context: WorkspaceSyncContext,
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Partial<RelationMetadataEntity>[] {
const objectMetadata = TypedReflect.getMetadata(
'objectMetadata',
standardObjectMetadata,
);
const relationMetadataCollection = TypedReflect.getMetadata(
'relationMetadataCollection',
standardObjectMetadata,
);
if (!objectMetadata) {
throw new Error(
`Object metadata decorator not found, can\'t parse ${standardObjectMetadata.name}`,
);
}
if (
!relationMetadataCollection ||
isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap)
) {
return [];
}
return relationMetadataCollection
.filter(
(relationMetadata) =>
!isGatedAndNotEnabled(
relationMetadata.gate,
workspaceFeatureFlagsMap,
),
)
.map((relationMetadata) => {
const fromObjectMetadata =
originalObjectMetadataMap[relationMetadata.fromObjectNameSingular];
assert(
fromObjectMetadata,
`Object ${relationMetadata.fromObjectNameSingular} not found in DB
for relation FROM defined in class ${objectMetadata.nameSingular}`,
);
const toObjectMetadata =
originalObjectMetadataMap[relationMetadata.toObjectNameSingular];
assert(
toObjectMetadata,
`Object ${relationMetadata.toObjectNameSingular} not found in DB
for relation TO defined in class ${objectMetadata.nameSingular}`,
);
const fromFieldMetadata = fromObjectMetadata?.fields.find(
(field) => field.name === relationMetadata.fromFieldMetadataName,
);
assert(
fromFieldMetadata,
`Field ${relationMetadata.fromFieldMetadataName} not found in object ${relationMetadata.fromObjectNameSingular}
for relation FROM defined in class ${objectMetadata.nameSingular}`,
);
const toFieldMetadata = toObjectMetadata?.fields.find(
(field) => field.name === relationMetadata.toFieldMetadataName,
);
assert(
toFieldMetadata,
`Field ${relationMetadata.toFieldMetadataName} not found in object ${relationMetadata.toObjectNameSingular}
for relation TO defined in class ${objectMetadata.nameSingular}`,
);
return {
relationType: relationMetadata.type,
fromObjectMetadataId: fromObjectMetadata?.id,
toObjectMetadataId: toObjectMetadata?.id,
fromFieldMetadataId: fromFieldMetadata?.id,
toFieldMetadataId: toFieldMetadata?.id,
workspaceId: context.workspaceId,
};
});
}
}

264
packages/twenty-server/src/workspace/workspace-sync-metadata/factories/workspace-sync.factory.ts Normal file
View File

@ -0,0 +1,264 @@
import { Injectable } from '@nestjs/common';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/metadata/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import {
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnRelation,
WorkspaceMigrationEntity,
WorkspaceMigrationTableAction,
} from 'src/metadata/workspace-migration/workspace-migration.entity';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory';
import {
RelationMetadataEntity,
RelationMetadataType,
} from 'src/metadata/relation-metadata/relation-metadata.entity';
import { camelCase } from 'src/utils/camel-case';
import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util';
@Injectable()
export class WorkspaceSyncFactory {
constructor(
private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
) {}
async createObjectMigration(
originalObjectMetadataCollection: ObjectMetadataEntity[],
createdObjectMetadataCollection: ObjectMetadataEntity[],
objectMetadataDeleteCollection: ObjectMetadataEntity[],
createdFieldMetadataCollection: FieldMetadataEntity[],
fieldMetadataDeleteCollection: FieldMetadataEntity[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
/**
* Create object migrations
*/
if (createdObjectMetadataCollection.length > 0) {
for (const objectMetadata of createdObjectMetadataCollection) {
const migrations = [
{
name: computeObjectTargetTable(objectMetadata),
action: 'create',
} satisfies WorkspaceMigrationTableAction,
...objectMetadata.fields
.filter((field) => field.type !== FieldMetadataType.RELATION)
.map(
(field) =>
({
name: computeObjectTargetTable(objectMetadata),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
field,
),
}) satisfies WorkspaceMigrationTableAction,
),
];
workspaceMigrations.push({
workspaceId: objectMetadata.workspaceId,
name: generateMigrationName(`create-${objectMetadata.nameSingular}`),
isCustom: false,
migrations,
});
}
}
/**
* Delete object migrations
* TODO: handle object delete migrations.
* Note: we need to delete the relation first due to the DB constraint.
*/
// if (objectMetadataDeleteCollection.length > 0) {
// for (const objectMetadata of objectMetadataDeleteCollection) {
// const migrations = [
// {
// name: computeObjectTargetTable(objectMetadata),
// action: 'drop',
// columns: [],
// } satisfies WorkspaceMigrationTableAction,
// ];
// workspaceMigrations.push({
// workspaceId: objectMetadata.workspaceId,
// isCustom: false,
// migrations,
// });
// }
// }
/**
* Create field migrations
*/
const originalObjectMetadataMap = originalObjectMetadataCollection.reduce(
(result, currentObject) => {
result[currentObject.id] = currentObject;
return result;
},
{} as Record<string, ObjectMetadataEntity>,
);
if (createdFieldMetadataCollection.length > 0) {
for (const fieldMetadata of createdFieldMetadataCollection) {
const migrations = [
{
name: computeObjectTargetTable(
originalObjectMetadataMap[fieldMetadata.objectMetadataId],
),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
fieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
];
workspaceMigrations.push({
workspaceId: fieldMetadata.workspaceId,
name: generateMigrationName(`create-${fieldMetadata.name}`),
isCustom: false,
migrations,
});
}
}
/**
* Delete field migrations
*/
if (fieldMetadataDeleteCollection.length > 0) {
for (const fieldMetadata of fieldMetadataDeleteCollection) {
const migrations = [
{
name: computeObjectTargetTable(
originalObjectMetadataMap[fieldMetadata.objectMetadataId],
),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.DROP,
columnName: fieldMetadata.name,
},
],
} satisfies WorkspaceMigrationTableAction,
];
workspaceMigrations.push({
workspaceId: fieldMetadata.workspaceId,
name: generateMigrationName(`delete-${fieldMetadata.name}`),
isCustom: false,
migrations,
});
}
}
return workspaceMigrations;
}
async createRelationMigration(
originalObjectMetadataCollection: ObjectMetadataEntity[],
createdRelationMetadataCollection: RelationMetadataEntity[],
// TODO: handle relation deletion
// eslint-disable-next-line @typescript-eslint/no-unused-vars
relationMetadataDeleteCollection: RelationMetadataEntity[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
if (createdRelationMetadataCollection.length > 0) {
for (const relationMetadata of createdRelationMetadataCollection) {
const toObjectMetadata = originalObjectMetadataCollection.find(
(object) => object.id === relationMetadata.toObjectMetadataId,
);
const fromObjectMetadata = originalObjectMetadataCollection.find(
(object) => object.id === relationMetadata.fromObjectMetadataId,
);
if (!toObjectMetadata) {
throw new Error(
`ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`,
);
}
if (!fromObjectMetadata) {
throw new Error(
`ObjectMetadata with id ${relationMetadata.fromObjectMetadataId} not found`,
);
}
const toFieldMetadata = toObjectMetadata.fields.find(
(field) => field.id === relationMetadata.toFieldMetadataId,
);
if (!toFieldMetadata) {
throw new Error(
`FieldMetadata with id ${relationMetadata.toFieldMetadataId} not found`,
);
}
const migrations = [
{
name: computeObjectTargetTable(toObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.RELATION,
columnName: `${camelCase(toFieldMetadata.name)}Id`,
referencedTableName:
computeObjectTargetTable(fromObjectMetadata),
referencedTableColumnName: 'id',
isUnique:
relationMetadata.relationType ===
RelationMetadataType.ONE_TO_ONE,
} satisfies WorkspaceMigrationColumnRelation,
],
} satisfies WorkspaceMigrationTableAction,
];
workspaceMigrations.push({
workspaceId: relationMetadata.workspaceId,
name: generateMigrationName(
`create-relation-from-${fromObjectMetadata.nameSingular}-to-${toObjectMetadata.nameSingular}`,
),
isCustom: false,
migrations,
});
}
}
// if (relationMetadataDeleteCollection.length > 0) {
// for (const relationMetadata of relationMetadataDeleteCollection) {
// const toObjectMetadata = originalObjectMetadataCollection.find(
// (object) => object.id === relationMetadata.toObjectMetadataId,
// );
// if (!toObjectMetadata) {
// throw new Error(
// `ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`,
// );
// }
// const migrations = [
// {
// name: computeObjectTargetTable(toObjectMetadata),
// action: 'drop',
// columns: [],
// } satisfies WorkspaceMigrationTableAction,
// ];
// workspaceMigrations.push({
// workspaceId: relationMetadata.workspaceId,
// isCustom: false,
// migrations,
// });
// }
// }
return workspaceMigrations;
}
}

46
packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/comparator.interface.ts Normal file
View File

@ -0,0 +1,46 @@
import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
import { PartialFieldMetadata } from './partial-field-metadata.interface';
import { PartialObjectMetadata } from './partial-object-metadata.interface';
export const enum ComparatorAction {
SKIP = 'SKIP',
CREATE = 'CREATE',
UPDATE = 'UPDATE',
DELETE = 'DELETE',
}
export interface ComparatorSkipResult {
action: ComparatorAction.SKIP;
}
export interface ComparatorCreateResult<T> {
action: ComparatorAction.CREATE;
object: T;
}
export interface ComparatorUpdateResult<T> {
action: ComparatorAction.UPDATE;
object: T;
}
export interface ComparatorDeleteResult<T> {
action: ComparatorAction.DELETE;
object: T;
}
export type ObjectComparatorResult =
| ComparatorSkipResult
| ComparatorCreateResult<PartialObjectMetadata>
| ComparatorUpdateResult<Partial<PartialObjectMetadata>>;
export type FieldComparatorResult =
| ComparatorSkipResult
| ComparatorCreateResult<PartialFieldMetadata>
| ComparatorUpdateResult<Partial<PartialFieldMetadata> & { id: string }>
| ComparatorDeleteResult<FieldMetadataEntity>;
export type RelationComparatorResult =
| ComparatorCreateResult<Partial<RelationMetadataEntity>>
| ComparatorDeleteResult<RelationMetadataEntity>;

1
packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface.ts
View File

@ -2,6 +2,7 @@ import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/inte
import { ReflectObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/reflect-object-metadata.interface';
export type PartialObjectMetadata = ReflectObjectMetadata & {
id?: string;
workspaceId: string;
dataSourceId: string;
fields: PartialFieldMetadata[];

7
packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/reflect-field-metadata.interface.ts
View File

@ -1,6 +1,7 @@
import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface';
import { FieldMetadataOptions } from 'src/metadata/field-metadata/interfaces/field-metadata-options.interface';
import { FieldMetadataTargetColumnMap } from 'src/metadata/field-metadata/interfaces/field-metadata-target-column-map.interface';
import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
@ -23,13 +24,13 @@ export interface ReflectFieldMetadata {
> & {
name: string;
type: FieldMetadataType;
targetColumnMap: string;
targetColumnMap: FieldMetadataTargetColumnMap<'default'>;
isNullable: boolean;
isSystem: boolean;
isCustom: boolean;
description?: string;
defaultValue: string | null;
defaultValue: FieldMetadataDefaultValue<'default'> | null;
gate?: GateDecoratorParams;
options?: string | null;
options?: FieldMetadataOptions<'default'> | null;
};
}

4
packages/twenty-server/src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface.ts Normal file
View File

@ -0,0 +1,4 @@
export interface WorkspaceSyncContext {
workspaceId: string;
dataSourceId: string;
}

163
packages/twenty-server/src/workspace/workspace-sync-metadata/reflective-metadata.factory.ts
View File

@ -1,163 +0,0 @@
import { Injectable } from '@nestjs/common';
import assert from 'assert';
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import { MappedObjectMetadataEntity } from 'src/workspace/workspace-sync-metadata/interfaces/mapped-metadata.interface';
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
import { TypedReflect } from 'src/utils/typed-reflect';
import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util';
@Injectable()
export class ReflectiveMetadataFactory {
async createObjectMetadata(
metadata: typeof BaseObjectMetadata,
workspaceId: string,
defaultDataSourceId: string,
workspaceFeatureFlagsMap: Record<string, boolean>,
): Promise<PartialObjectMetadata | undefined> {
const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata);
const fieldMetadata =
TypedReflect.getMetadata('fieldMetadata', metadata) ?? {};
if (!objectMetadata) {
throw new Error(
`Object metadata decorator not found, can\'t parse ${metadata.name}`,
);
}
if (isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap)) {
return undefined;
}
const fields = Object.values(fieldMetadata).filter(
(field) => !isGatedAndNotEnabled(field, workspaceFeatureFlagsMap),
);
return {
...objectMetadata,
workspaceId,
dataSourceId: defaultDataSourceId,
fields: fields.map((field) => ({
...field,
workspaceId,
isSystem: objectMetadata.isSystem || field.isSystem,
})),
};
}
async createObjectMetadataCollection(
metadataCollection: (typeof BaseObjectMetadata)[],
workspaceId: string,
dataSourceId: string,
workspaceFeatureFlagsMap: Record<string, boolean>,
) {
const metadataPromises = metadataCollection.map((metadata) =>
this.createObjectMetadata(
metadata,
workspaceId,
dataSourceId,
workspaceFeatureFlagsMap,
),
);
const resolvedMetadata = await Promise.all(metadataPromises);
return resolvedMetadata.filter(
(metadata): metadata is PartialObjectMetadata => !!metadata,
);
}
createRelationMetadata(
metadata: typeof BaseObjectMetadata,
workspaceId: string,
objectMetadataFromDB: Record<string, MappedObjectMetadataEntity>,
workspaceFeatureFlagsMap: Record<string, boolean>,
) {
const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata);
const relationMetadata = TypedReflect.getMetadata(
'relationMetadata',
metadata,
);
if (!objectMetadata) {
throw new Error(
`Object metadata decorator not found, can\'t parse ${metadata.name}`,
);
}
if (
!relationMetadata ||
isGatedAndNotEnabled(objectMetadata, workspaceFeatureFlagsMap)
) {
return [];
}
return relationMetadata
.filter(
(relation) => !isGatedAndNotEnabled(relation, workspaceFeatureFlagsMap),
)
.map((relation) => {
const fromObjectMetadata =
objectMetadataFromDB[relation.fromObjectNameSingular];
assert(
fromObjectMetadata,
`Object ${relation.fromObjectNameSingular} not found in DB
for relation FROM defined in class ${objectMetadata.nameSingular}`,
);
const toObjectMetadata =
objectMetadataFromDB[relation.toObjectNameSingular];
assert(
toObjectMetadata,
`Object ${relation.toObjectNameSingular} not found in DB
for relation TO defined in class ${objectMetadata.nameSingular}`,
);
const fromFieldMetadata =
fromObjectMetadata?.fields[relation.fromFieldMetadataName];
assert(
fromFieldMetadata,
`Field ${relation.fromFieldMetadataName} not found in object ${relation.fromObjectNameSingular}
for relation FROM defined in class ${objectMetadata.nameSingular}`,
);
const toFieldMetadata =
toObjectMetadata?.fields[relation.toFieldMetadataName];
assert(
toFieldMetadata,
`Field ${relation.toFieldMetadataName} not found in object ${relation.toObjectNameSingular}
for relation TO defined in class ${objectMetadata.nameSingular}`,
);
return {
relationType: relation.type,
fromObjectMetadataId: fromObjectMetadata?.id,
toObjectMetadataId: toObjectMetadata?.id,
fromFieldMetadataId: fromFieldMetadata?.id,
toFieldMetadataId: toFieldMetadata?.id,
workspaceId,
};
});
}
createRelationMetadataCollection(
metadataCollection: (typeof BaseObjectMetadata)[],
workspaceId: string,
objectMetadataFromDB: Record<string, MappedObjectMetadataEntity>,
workspaceFeatureFlagsMap: Record<string, boolean>,
) {
return metadataCollection.flatMap((metadata) =>
this.createRelationMetadata(
metadata,
workspaceId,
objectMetadataFromDB,
workspaceFeatureFlagsMap,
),
);
}
}

189
packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service.ts Normal file
View File

@ -0,0 +1,189 @@
import { Injectable, Logger } from '@nestjs/common';
import { EntityManager, In } from 'typeorm';
import { v4 as uuidV4 } from 'uuid';
import omit from 'lodash.omit';
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/metadata/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
import { FieldMetadataComplexOption } from 'src/metadata/field-metadata/dtos/options.input';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
@Injectable()
export class WorkspaceMetadataUpdaterService {
private readonly logger = new Logger(WorkspaceMetadataUpdaterService.name);
async updateObjectMetadata(
manager: EntityManager,
storage: WorkspaceSyncStorage,
): Promise<{
createdObjectMetadataCollection: ObjectMetadataEntity[];
updatedObjectMetadataCollection: ObjectMetadataEntity[];
}> {
const objectMetadataRepository =
manager.getRepository(ObjectMetadataEntity);
/**
* Create object metadata
*/
const createdPartialObjectMetadataCollection =
await objectMetadataRepository.save(
storage.objectMetadataCreateCollection.map((objectMetadata) => ({
...objectMetadata,
isActive: true,
fields: objectMetadata.fields.map((field) =>
this.prepareFieldMetadataForCreation(field),
),
})) as DeepPartial<ObjectMetadataEntity>[],
);
const identifiers = createdPartialObjectMetadataCollection.map(
(object) => object.id,
);
const createdObjectMetadataCollection = await manager.find(
ObjectMetadataEntity,
{
where: { id: In(identifiers) },
relations: ['dataSource', 'fields'],
},
);
/**
* Update object metadata
*/
const updatedObjectMetadataCollection = await objectMetadataRepository.save(
storage.objectMetadataUpdateCollection.map((objectMetadata) =>
omit(objectMetadata, ['fields']),
),
);
/**
* Delete object metadata
*/
if (storage.objectMetadataDeleteCollection.length > 0) {
await objectMetadataRepository.delete(
storage.objectMetadataDeleteCollection.map((object) => object.id),
);
}
return {
createdObjectMetadataCollection,
updatedObjectMetadataCollection,
};
}
/**
* TODO: Refactor this
*/
private prepareFieldMetadataForCreation(field: PartialFieldMetadata) {
return {
...field,
...(field.type === FieldMetadataType.SELECT && field.options
? {
options: this.generateUUIDForNewSelectFieldOptions(
field.options as FieldMetadataComplexOption[],
),
}
: {}),
isActive: true,
};
}
private generateUUIDForNewSelectFieldOptions(
options: FieldMetadataComplexOption[],
): FieldMetadataComplexOption[] {
return options.map((option) => ({
...option,
id: uuidV4(),
}));
}
async updateFieldMetadata(
manager: EntityManager,
storage: WorkspaceSyncStorage,
): Promise<{
createdFieldMetadataCollection: FieldMetadataEntity[];
updatedFieldMetadataCollection: FieldMetadataEntity[];
}> {
const fieldMetadataRepository = manager.getRepository(FieldMetadataEntity);
/**
* Create field metadata
*/
const createdFieldMetadataCollection = await fieldMetadataRepository.save(
storage.fieldMetadataCreateCollection.map((field) =>
this.prepareFieldMetadataForCreation(field),
) as DeepPartial<FieldMetadataEntity>[],
);
/**
* Update field metadata
*/
const updatedFieldMetadataCollection = await fieldMetadataRepository.save(
storage.fieldMetadataUpdateCollection as DeepPartial<FieldMetadataEntity>[],
);
/**
* Delete field metadata
*/
// TODO: handle relation fields deletion. We need to delete the relation metadata first due to the DB constraint.
const fieldMetadataDeleteCollectionWithoutRelationType =
storage.fieldMetadataDeleteCollection.filter(
(field) => field.type !== FieldMetadataType.RELATION,
);
if (fieldMetadataDeleteCollectionWithoutRelationType.length > 0) {
await fieldMetadataRepository.delete(
fieldMetadataDeleteCollectionWithoutRelationType.map(
(field) => field.id,
),
);
}
return {
createdFieldMetadataCollection:
createdFieldMetadataCollection as FieldMetadataEntity[],
updatedFieldMetadataCollection:
updatedFieldMetadataCollection as FieldMetadataEntity[],
};
}
async updateRelationMetadata(
manager: EntityManager,
storage: WorkspaceSyncStorage,
): Promise<{
createdRelationMetadataCollection: RelationMetadataEntity[];
}> {
const relationMetadataRepository = manager.getRepository(
RelationMetadataEntity,
);
/**
* Create relation metadata
*/
const createdRelationMetadataCollection =
await relationMetadataRepository.save(
storage.relationMetadataCreateCollection,
);
/**
* Delete relation metadata
*/
if (storage.relationMetadataDeleteCollection.length > 0) {
await relationMetadataRepository.delete(
storage.relationMetadataDeleteCollection.map(
(relationMetadata) => relationMetadata.id,
),
);
}
return {
createdRelationMetadataCollection,
};
}
}

154
packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts Normal file
View File

@ -0,0 +1,154 @@
import { Injectable, Logger } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
import { StandardObjectFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-object.factory';
import { WorkspaceObjectComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator';
import { WorkspaceFieldComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator';
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
@Injectable()
export class WorkspaceSyncObjectMetadataService {
private readonly logger = new Logger(WorkspaceSyncObjectMetadataService.name);
constructor(
private readonly standardObjectFactory: StandardObjectFactory,
private readonly workspaceObjectComparator: WorkspaceObjectComparator,
private readonly workspaceFieldComparator: WorkspaceFieldComparator,
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
private readonly workspaceSyncFactory: WorkspaceSyncFactory,
) {}
async synchronize(
context: WorkspaceSyncContext,
manager: EntityManager,
storage: WorkspaceSyncStorage,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Promise<void> {
const objectMetadataRepository =
manager.getRepository(ObjectMetadataEntity);
const workspaceMigrationRepository = manager.getRepository(
WorkspaceMigrationEntity,
);
// Retrieve object metadata collection from DB
const originalObjectMetadataCollection =
await objectMetadataRepository.find({
where: { workspaceId: context.workspaceId, isCustom: false },
relations: ['dataSource', 'fields'],
});
// Create standard object metadata collection
const standardObjectMetadataCollection = this.standardObjectFactory.create(
context,
workspaceFeatureFlagsMap,
);
// Create map of original and standard object metadata by unique identifier
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
originalObjectMetadataCollection,
);
const standardObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
standardObjectMetadataCollection,
);
this.logger.log('Comparing standard objects and fields metadata');
// Store object that need to be deleted
for (const originalObjectMetadata of originalObjectMetadataCollection) {
if (!standardObjectMetadataMap[originalObjectMetadata.nameSingular]) {
storage.addDeleteObjectMetadata(originalObjectMetadata);
}
}
// Loop over all standard objects and compare them with the objects in DB
for (const standardObjectName in standardObjectMetadataMap) {
const originalObjectMetadata =
originalObjectMetadataMap[standardObjectName];
const standardObjectMetadata =
standardObjectMetadataMap[standardObjectName];
/**
* COMPARE OBJECT METADATA
*/
const objectComparatorResult = this.workspaceObjectComparator.compare(
originalObjectMetadata,
standardObjectMetadata,
);
if (objectComparatorResult.action === ComparatorAction.CREATE) {
storage.addCreateObjectMetadata(standardObjectMetadata);
continue;
}
if (objectComparatorResult.action === ComparatorAction.UPDATE) {
storage.addUpdateObjectMetadata(objectComparatorResult.object);
}
/**
* COMPARE FIELD METADATA
*/
const fieldComparatorResults = this.workspaceFieldComparator.compare(
originalObjectMetadata,
standardObjectMetadata,
);
for (const fieldComparatorResult of fieldComparatorResults) {
switch (fieldComparatorResult.action) {
case ComparatorAction.CREATE: {
storage.addCreateFieldMetadata(fieldComparatorResult.object);
break;
}
case ComparatorAction.UPDATE: {
storage.addUpdateFieldMetadata(fieldComparatorResult.object);
break;
}
case ComparatorAction.DELETE: {
storage.addDeleteFieldMetadata(fieldComparatorResult.object);
break;
}
}
}
}
this.logger.log('Updating workspace metadata');
// Apply changes to DB
const metadataObjectUpdaterResult =
await this.workspaceMetadataUpdaterService.updateObjectMetadata(
manager,
storage,
);
const metadataFieldUpdaterResult =
await this.workspaceMetadataUpdaterService.updateFieldMetadata(
manager,
storage,
);
this.logger.log('Generating migrations');
// Create migrations
const workspaceObjectMigrations =
await this.workspaceSyncFactory.createObjectMigration(
originalObjectMetadataCollection,
metadataObjectUpdaterResult.createdObjectMetadataCollection,
storage.objectMetadataDeleteCollection,
metadataFieldUpdaterResult.createdFieldMetadataCollection,
storage.fieldMetadataDeleteCollection,
);
this.logger.log('Saving migrations');
// Save migrations into DB
await workspaceMigrationRepository.save(workspaceObjectMigrations);
}
}

105
packages/twenty-server/src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service.ts Normal file
View File

@ -0,0 +1,105 @@
import { Injectable, Logger } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util';
import { StandardRelationFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-relation.factory';
import { WorkspaceRelationComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator';
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
@Injectable()
export class WorkspaceSyncRelationMetadataService {
private readonly logger = new Logger(
WorkspaceSyncRelationMetadataService.name,
);
constructor(
private readonly standardRelationFactory: StandardRelationFactory,
private readonly workspaceRelationComparator: WorkspaceRelationComparator,
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
private readonly workspaceSyncFactory: WorkspaceSyncFactory,
) {}
async synchronize(
context: WorkspaceSyncContext,
manager: EntityManager,
storage: WorkspaceSyncStorage,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Promise<void> {
const objectMetadataRepository =
manager.getRepository(ObjectMetadataEntity);
const workspaceMigrationRepository = manager.getRepository(
WorkspaceMigrationEntity,
);
// Retrieve object metadata collection from DB
const originalObjectMetadataCollection =
await objectMetadataRepository.find({
where: { workspaceId: context.workspaceId, isCustom: false },
relations: ['dataSource', 'fields'],
});
// Create map of object metadata & field metadata by unique identifier
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
originalObjectMetadataCollection,
);
const relationMetadataRepository = manager.getRepository(
RelationMetadataEntity,
);
// Retrieve relation metadata collection from DB
// TODO: filter out custom relations once isCustom has been added to relationMetadata table
const originalRelationMetadataCollection =
await relationMetadataRepository.find({
where: { workspaceId: context.workspaceId },
});
// Create standard relation metadata collection
const standardRelationMetadataCollection =
this.standardRelationFactory.create(
context,
originalObjectMetadataMap,
workspaceFeatureFlagsMap,
);
const relationComparatorResults = this.workspaceRelationComparator.compare(
originalRelationMetadataCollection,
standardRelationMetadataCollection,
);
for (const relationComparatorResult of relationComparatorResults) {
if (relationComparatorResult.action === ComparatorAction.CREATE) {
storage.addCreateRelationMetadata(relationComparatorResult.object);
} else if (relationComparatorResult.action === ComparatorAction.DELETE) {
storage.addDeleteRelationMetadata(relationComparatorResult.object);
}
}
const metadataRelationUpdaterResult =
await this.workspaceMetadataUpdaterService.updateRelationMetadata(
manager,
storage,
);
// Create migrations
const workspaceRelationMigrations =
await this.workspaceSyncFactory.createRelationMigration(
originalObjectMetadataCollection,
metadataRelationUpdaterResult.createdRelationMetadataCollection,
storage.relationMetadataDeleteCollection,
);
// Save migrations into DB
await workspaceMigrationRepository.save(workspaceRelationMigrations);
}
}

93
packages/twenty-server/src/workspace/workspace-sync-metadata/storage/workspace-sync.storage.ts Normal file
View File

@ -0,0 +1,93 @@
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
export class WorkspaceSyncStorage {
// Object metadata
private readonly _objectMetadataCreateCollection: PartialObjectMetadata[] =
[];
private readonly _objectMetadataUpdateCollection: Partial<PartialObjectMetadata>[] =
[];
private readonly _objectMetadataDeleteCollection: ObjectMetadataEntity[] = [];
// Field metadata
private readonly _fieldMetadataCreateCollection: PartialFieldMetadata[] = [];
private readonly _fieldMetadataUpdateCollection: Partial<PartialFieldMetadata>[] =
[];
private readonly _fieldMetadataDeleteCollection: FieldMetadataEntity[] = [];
// Relation metadata
private readonly _relationMetadataCreateCollection: Partial<RelationMetadataEntity>[] =
[];
private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] =
[];
constructor() {}
get objectMetadataCreateCollection() {
return this._objectMetadataCreateCollection;
}
get objectMetadataUpdateCollection() {
return this._objectMetadataUpdateCollection;
}
get objectMetadataDeleteCollection() {
return this._objectMetadataDeleteCollection;
}
get fieldMetadataCreateCollection() {
return this._fieldMetadataCreateCollection;
}
get fieldMetadataUpdateCollection() {
return this._fieldMetadataUpdateCollection;
}
get fieldMetadataDeleteCollection() {
return this._fieldMetadataDeleteCollection;
}
get relationMetadataCreateCollection() {
return this._relationMetadataCreateCollection;
}
get relationMetadataDeleteCollection() {
return this._relationMetadataDeleteCollection;
}
addCreateObjectMetadata(object: PartialObjectMetadata) {
this._objectMetadataCreateCollection.push(object);
}
addUpdateObjectMetadata(object: Partial<PartialObjectMetadata>) {
this._objectMetadataUpdateCollection.push(object);
}
addDeleteObjectMetadata(object: ObjectMetadataEntity) {
this._objectMetadataDeleteCollection.push(object);
}
addCreateFieldMetadata(field: PartialFieldMetadata) {
this._fieldMetadataCreateCollection.push(field);
}
addUpdateFieldMetadata(field: Partial<PartialFieldMetadata>) {
this._fieldMetadataUpdateCollection.push(field);
}
addDeleteFieldMetadata(field: FieldMetadataEntity) {
this._fieldMetadataDeleteCollection.push(field);
}
addCreateRelationMetadata(relation: Partial<RelationMetadataEntity>) {
this._relationMetadataCreateCollection.push(relation);
}
addDeleteRelationMetadata(relation: RelationMetadataEntity) {
this._relationMetadataDeleteCollection.push(relation);
}
}

9
packages/twenty-server/src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util.ts
View File

@ -1,10 +1,11 @@
import { GateDecoratorParams } from 'src/workspace/workspace-sync-metadata/interfaces/gate-decorator.interface';
export const isGatedAndNotEnabled = (
metadata,
gate: GateDecoratorParams | undefined,
workspaceFeatureFlagsMap: Record<string, boolean>,
): boolean => {
const featureFlagValue =
metadata.gate?.featureFlag &&
workspaceFeatureFlagsMap[metadata.gate.featureFlag];
gate?.featureFlag && workspaceFeatureFlagsMap[gate.featureFlag];
return metadata.gate?.featureFlag !== undefined && !featureFlagValue;
return gate?.featureFlag !== undefined && !featureFlagValue;
};

65
packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.spec.ts
View File

@ -1,52 +1,7 @@
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity';
import {
filterIgnoredProperties,
mapObjectMetadataByUniqueIdentifier,
} from './sync-metadata.util';
describe('filterIgnoredProperties', () => {
it('should filter out properties based on the ignore list', () => {
const obj = {
name: 'John',
age: 30,
email: 'john@example.com',
address: '123 Main St',
};
const propertiesToIgnore = ['age', 'address'];
const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
expect(filteredObj).toEqual({
name: 'John',
email: 'john@example.com',
});
});
it('should return the original object if ignore list is empty', () => {
const obj = {
name: 'John',
age: 30,
email: 'john@example.com',
address: '123 Main St',
};
const propertiesToIgnore: string[] = [];
const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
expect(filteredObj).toEqual(obj);
});
it('should return an empty object if the original object is empty', () => {
const obj = {};
const propertiesToIgnore = ['age', 'address'];
const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore);
expect(filteredObj).toEqual({});
});
});
import { mapObjectMetadataByUniqueIdentifier } from './sync-metadata.util';
describe('mapObjectMetadataByUniqueIdentifier', () => {
it('should convert an array of ObjectMetadataEntity objects into a map', () => {
@ -75,18 +30,18 @@ describe('mapObjectMetadataByUniqueIdentifier', () => {
expect(mappedObject).toEqual({
user: {
nameSingular: 'user',
fields: {
id: { name: 'id', type: FieldMetadataType.UUID },
name: { name: 'name', type: FieldMetadataType.TEXT },
},
fields: [
{ name: 'id', type: FieldMetadataType.UUID },
{ name: 'name', type: FieldMetadataType.TEXT },
],
},
product: {
nameSingular: 'product',
fields: {
id: { name: 'id', type: FieldMetadataType.UUID },
name: { name: 'name', type: FieldMetadataType.TEXT },
price: { name: 'price', type: FieldMetadataType.UUID },
},
fields: [
{ name: 'id', type: FieldMetadataType.UUID },
{ name: 'name', type: FieldMetadataType.TEXT },
{ name: 'price', type: FieldMetadataType.UUID },
],
},
});
});

71
packages/twenty-server/src/workspace/workspace-sync-metadata/utils/sync-metadata.util.ts
View File

@ -1,27 +1,3 @@
import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
import { FieldMetadataOptions } from 'src/metadata/field-metadata/interfaces/field-metadata-options.interface';
import { FieldMetadataTargetColumnMap } from 'src/metadata/field-metadata/interfaces/field-metadata-target-column-map.interface';
/**
* This utility function filters out properties from an object based on a list of properties to ignore.
* It returns a new object with only the properties that are not in the ignore list.
*
* @param obj - The object to filter.
* @param propertiesToIgnore - An array of property names to ignore.
* @returns A new object with filtered properties.
*/
export const filterIgnoredProperties = (
obj: any,
propertiesToIgnore: string[],
mapFunction?: (value: any) => any,
) => {
return Object.fromEntries(
Object.entries(obj)
.filter(([key]) => !propertiesToIgnore.includes(key))
.map(([key, value]) => [key, mapFunction ? mapFunction(value) : value]),
);
};
/**
* This utility function converts an array of ObjectMetadataEntity objects into a map,
* where the keys are the nameSingular properties of the objects.
@ -31,59 +7,18 @@ export const filterIgnoredProperties = (
* @returns A map of object metadata, with nameSingular as the key and the object as the value.
*/
export const mapObjectMetadataByUniqueIdentifier = <
T extends { nameSingular: string; fields: U[] },
U extends { name: string },
T extends { nameSingular: string },
>(
arr: T[],
): Record<string, Omit<T, 'fields'> & { fields: Record<string, U> }> => {
): Record<string, T> => {
return arr.reduce(
(acc, curr) => {
acc[curr.nameSingular] = {
...curr,
fields: curr.fields.reduce(
(acc, curr) => {
acc[curr.name] = curr;
return acc;
},
{} as Record<string, U>,
),
};
return acc;
},
{} as Record<string, Omit<T, 'fields'> & { fields: Record<string, U> }>,
{} as Record<string, T>,
);
};
export const convertStringifiedFieldsToJSON = <
T extends {
targetColumnMap?: string | null;
defaultValue?: string | null;
options?: string | null;
},
>(
fieldMetadata: T,
): T & {
targetColumnMap?: FieldMetadataTargetColumnMap;
defaultValue?: FieldMetadataDefaultValue;
options?: FieldMetadataOptions;
} => {
if (fieldMetadata.targetColumnMap) {
fieldMetadata.targetColumnMap = JSON.parse(fieldMetadata.targetColumnMap);
}
if (fieldMetadata.defaultValue) {
fieldMetadata.defaultValue = JSON.parse(fieldMetadata.defaultValue);
}
if (fieldMetadata.options) {
fieldMetadata.options = JSON.parse(fieldMetadata.options);
}
return fieldMetadata as T & {
targetColumnMap?: FieldMetadataTargetColumnMap;
defaultValue?: FieldMetadataDefaultValue;
options?: FieldMetadataOptions;
};
};

17
packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.module.ts
View File

@ -8,8 +8,12 @@ import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module';
import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module';
import { ReflectiveMetadataFactory } from 'src/workspace/workspace-sync-metadata/reflective-metadata.factory';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync.metadata.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service';
import { workspaceSyncMetadataFactories } from 'src/workspace/workspace-sync-metadata/factories';
import { workspaceSyncMetadataComparators } from 'src/workspace/workspace-sync-metadata/comparators';
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
@Module({
imports: [
@ -26,7 +30,14 @@ import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metad
),
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
],
providers: [WorkspaceSyncMetadataService, ReflectiveMetadataFactory],
providers: [
...workspaceSyncMetadataFactories,
...workspaceSyncMetadataComparators,
WorkspaceMetadataUpdaterService,
WorkspaceSyncObjectMetadataService,
WorkspaceSyncRelationMetadataService,
WorkspaceSyncMetadataService,
],
exports: [WorkspaceSyncMetadataService],
})
export class WorkspaceSyncMetadataModule {}

82
packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync-metadata.service.ts Normal file
View File

@ -0,0 +1,82 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { DataSource } from 'typeorm';
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service';
import { FeatureFlagFactory } from 'src/workspace/workspace-sync-metadata/factories/feature-flags.factory';
import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
@Injectable()
export class WorkspaceSyncMetadataService {
private readonly logger = new Logger(WorkspaceSyncMetadataService.name);
constructor(
@InjectDataSource('metadata')
private readonly metadataDataSource: DataSource,
private readonly featureFlagFactory: FeatureFlagFactory,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService,
private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService,
) {}
/**
*
* Sync all standard objects and fields metadata for a given workspace and data source
* This will update the metadata if it has changed and generate migrations based on the diff.
*
* @param dataSourceId
* @param workspaceId
*/
public async syncStandardObjectsAndFieldsMetadata(
context: WorkspaceSyncContext,
) {
this.logger.log('Syncing standard objects and fields metadata');
const queryRunner = this.metadataDataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
const manager = queryRunner.manager;
try {
const storage = new WorkspaceSyncStorage();
// Retrieve feature flags
const workspaceFeatureFlagsMap =
await this.featureFlagFactory.create(context);
this.logger.log('Syncing standard objects and fields metadata');
await this.workspaceSyncObjectMetadataService.synchronize(
context,
manager,
storage,
workspaceFeatureFlagsMap,
);
await this.workspaceSyncRelationMetadataService.synchronize(
context,
manager,
storage,
workspaceFeatureFlagsMap,
);
await queryRunner.commitTransaction();
// Execute migrations
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
context.workspaceId,
);
} catch (error) {
console.error('Sync of standard objects failed with:', error);
await queryRunner.rollbackTransaction();
} finally {
await queryRunner.release();
}
}
}

584
packages/twenty-server/src/workspace/workspace-sync-metadata/workspace-sync.metadata.service.ts
View File

@ -1,584 +0,0 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import diff from 'microdiff';
import { In, Repository } from 'typeorm';
import camelCase from 'lodash.camelcase';
import { v4 as uuidV4 } from 'uuid';
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
import {
MappedFieldMetadataEntity,
MappedObjectMetadata,
} from 'src/workspace/workspace-sync-metadata/interfaces/mapped-metadata.interface';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/metadata/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import {
RelationMetadataEntity,
RelationMetadataType,
} from 'src/metadata/relation-metadata/relation-metadata.entity';
import {
filterIgnoredProperties,
mapObjectMetadataByUniqueIdentifier,
convertStringifiedFieldsToJSON,
} from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util';
import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects';
import {
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnRelation,
WorkspaceMigrationEntity,
WorkspaceMigrationTableAction,
} from 'src/metadata/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory';
import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service';
import { ReflectiveMetadataFactory } from 'src/workspace/workspace-sync-metadata/reflective-metadata.factory';
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { FieldMetadataComplexOption } from 'src/metadata/field-metadata/dtos/options.input';
@Injectable()
export class WorkspaceSyncMetadataService {
constructor(
private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly reflectiveMetadataFactory: ReflectiveMetadataFactory,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
@InjectRepository(FieldMetadataEntity, 'metadata')
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
@InjectRepository(RelationMetadataEntity, 'metadata')
private readonly relationMetadataRepository: Repository<RelationMetadataEntity>,
@InjectRepository(WorkspaceMigrationEntity, 'metadata')
private readonly workspaceMigrationRepository: Repository<WorkspaceMigrationEntity>,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {}
/**
*
* Sync all standard objects and fields metadata for a given workspace and data source
* This will update the metadata if it has changed and generate migrations based on the diff.
*
* @param dataSourceId
* @param workspaceId
*/
public async syncStandardObjectsAndFieldsMetadata(
dataSourceId: string,
workspaceId: string,
) {
try {
const workspaceFeatureFlags = await this.featureFlagRepository.find({
where: { workspaceId },
});
const workspaceFeatureFlagsMap = workspaceFeatureFlags.reduce(
(result, currentFeatureFlag) => {
result[currentFeatureFlag.key] = currentFeatureFlag.value;
return result;
},
{},
);
const standardObjects =
await this.reflectiveMetadataFactory.createObjectMetadataCollection(
standardObjectMetadata,
workspaceId,
dataSourceId,
workspaceFeatureFlagsMap,
);
const objectsInDB = await this.objectMetadataRepository.find({
where: { workspaceId, isCustom: false },
relations: ['dataSource', 'fields'],
});
const objectsInDBByName = mapObjectMetadataByUniqueIdentifier<
ObjectMetadataEntity,
FieldMetadataEntity
>(objectsInDB);
const standardObjectsByName = mapObjectMetadataByUniqueIdentifier<
PartialObjectMetadata,
PartialFieldMetadata
>(standardObjects);
const objectsToCreate: MappedObjectMetadata[] = [];
const objectsToDelete = objectsInDB.filter(
(objectInDB) => !standardObjectsByName[objectInDB.nameSingular],
);
const objectsToUpdate: Record<string, ObjectMetadataEntity> = {};
const fieldsToCreate: PartialFieldMetadata[] = [];
const fieldsToDelete: FieldMetadataEntity[] = [];
const fieldsToUpdate: Record<string, MappedFieldMetadataEntity> = {};
for (const standardObjectName in standardObjectsByName) {
const standardObject = standardObjectsByName[standardObjectName];
const objectInDB = objectsInDBByName[standardObjectName];
if (!objectInDB) {
objectsToCreate.push(standardObject);
continue;
}
// Deconstruct fields and compare objects and fields independently
const { fields: objectInDBFields, ...objectInDBWithoutFields } =
objectInDB;
const { fields: standardObjectFields, ...standardObjectWithoutFields } =
standardObject;
const objectPropertiesToIgnore = [
'id',
'createdAt',
'updatedAt',
'labelIdentifierFieldMetadataId',
'imageIdentifierFieldMetadataId',
'isActive',
];
const objectDiffWithoutIgnoredProperties = filterIgnoredProperties(
objectInDBWithoutFields,
objectPropertiesToIgnore,
);
const fieldPropertiesToIgnore = [
'id',
'createdAt',
'updatedAt',
'objectMetadataId',
'isActive',
];
const objectInDBFieldsWithoutDefaultFields = Object.fromEntries(
Object.entries(objectInDBFields).map(([key, value]) => {
if (value === null || typeof value !== 'object') {
return [key, value];
}
return [
key,
filterIgnoredProperties(
value,
fieldPropertiesToIgnore,
(property) => {
if (property !== null && typeof property === 'object') {
return JSON.stringify(property);
}
return property;
},
),
];
}),
);
// Compare objects
const objectDiff = diff(
objectDiffWithoutIgnoredProperties,
standardObjectWithoutFields,
);
// Compare fields
const fieldsDiff = diff(
objectInDBFieldsWithoutDefaultFields,
standardObjectFields,
);
for (const diff of objectDiff) {
// We only handle CHANGE here as REMOVE and CREATE are handled earlier.
if (diff.type === 'CHANGE') {
const property = diff.path[0];
objectsToUpdate[objectInDB.id] = {
...objectsToUpdate[objectInDB.id],
[property]: diff.value,
};
}
}
for (const diff of fieldsDiff) {
const fieldName = diff.path[0];
if (diff.type === 'CREATE') {
fieldsToCreate.push({
...standardObjectFields[fieldName],
objectMetadataId: objectInDB.id,
});
}
if (diff.type === 'REMOVE' && diff.path.length === 1) {
fieldsToDelete.push(objectInDBFields[fieldName]);
}
if (diff.type === 'CHANGE') {
const property = diff.path[diff.path.length - 1];
fieldsToUpdate[objectInDBFields[fieldName].id] = {
...fieldsToUpdate[objectInDBFields[fieldName].id],
[property]: diff.value,
};
}
}
}
// CREATE OBJECTS
const createdObjectMetadataCollection =
await this.objectMetadataRepository.save(
objectsToCreate.map((object) => ({
...object,
isActive: true,
fields: Object.values(object.fields).map((field) =>
this.prepareFieldMetadataForCreation(field),
),
})),
);
const identifiers = createdObjectMetadataCollection.map(
(object) => object.id,
);
const createdObjects = await this.objectMetadataRepository.find({
where: { id: In(identifiers) },
relations: ['dataSource', 'fields'],
});
// UPDATE OBJECTS, this is not optimal as we are running n queries here.
for (const [key, value] of Object.entries(objectsToUpdate)) {
await this.objectMetadataRepository.update(key, value);
}
// DELETE OBJECTS
if (objectsToDelete.length > 0) {
await this.objectMetadataRepository.delete(
objectsToDelete.map((object) => object.id),
);
}
// CREATE FIELDS
const createdFields = await this.fieldMetadataRepository.save(
fieldsToCreate.map((field) =>
this.prepareFieldMetadataForCreation(field),
),
);
// UPDATE FIELDS
for (const [key, value] of Object.entries(fieldsToUpdate)) {
await this.fieldMetadataRepository.update(
key,
convertStringifiedFieldsToJSON(value),
);
}
// DELETE FIELDS
// TODO: handle relation fields deletion. We need to delete the relation metadata first due to the DB constraint.
const fieldsToDeleteWithoutRelationType = fieldsToDelete.filter(
(field) => field.type !== FieldMetadataType.RELATION,
);
if (fieldsToDeleteWithoutRelationType.length > 0) {
await this.fieldMetadataRepository.delete(
fieldsToDeleteWithoutRelationType.map((field) => field.id),
);
}
// Generate migrations
await this.generateMigrationsFromSync(
createdObjects,
objectsToDelete,
createdFields,
fieldsToDelete,
objectsInDB,
);
// We run syncRelationMetadata after everything to ensure that all objects and fields are
// in the DB before creating relations.
await this.syncRelationMetadata(
workspaceId,
dataSourceId,
workspaceFeatureFlagsMap,
);
// Execute migrations
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
);
} catch (error) {
console.error('Sync of standard objects failed with:', error);
}
}
private prepareFieldMetadataForCreation(field: PartialFieldMetadata) {
const convertedField = convertStringifiedFieldsToJSON(field);
return {
...convertedField,
...(convertedField.type === FieldMetadataType.SELECT &&
convertedField.options
? {
options: this.generateUUIDForNewSelectFieldOptions(
convertedField.options as FieldMetadataComplexOption[],
),
}
: {}),
isActive: true,
};
}
private generateUUIDForNewSelectFieldOptions(
options: FieldMetadataComplexOption[],
): FieldMetadataComplexOption[] {
return options.map((option) => ({
...option,
id: uuidV4(),
}));
}
private async syncRelationMetadata(
workspaceId: string,
dataSourceId: string,
workspaceFeatureFlagsMap: Record<string, boolean>,
) {
const objectsInDB = await this.objectMetadataRepository.find({
where: { workspaceId, isCustom: false },
relations: ['dataSource', 'fields'],
});
const objectsInDBByName = mapObjectMetadataByUniqueIdentifier<
ObjectMetadataEntity,
FieldMetadataEntity
>(objectsInDB);
const standardRelations =
this.reflectiveMetadataFactory.createRelationMetadataCollection(
standardObjectMetadata,
workspaceId,
objectsInDBByName,
workspaceFeatureFlagsMap,
);
// TODO: filter out custom relations once isCustom has been added to relationMetadata table
const relationsInDB = await this.relationMetadataRepository.find({
where: { workspaceId },
});
// We filter out 'id' later because we need it to remove the relation from DB
const relationsInDBWithoutIgnoredProperties = relationsInDB
.map((relation) =>
filterIgnoredProperties(relation, ['createdAt', 'updatedAt']),
)
.reduce((result, currentObject) => {
const key = `${currentObject.fromObjectMetadataId}->${currentObject.fromFieldMetadataId}`;
result[key] = currentObject;
return result;
}, {});
// Compare relations
const relationsDiff = diff(
relationsInDBWithoutIgnoredProperties,
standardRelations,
);
const relationsToCreate: RelationMetadataEntity[] = [];
const relationsToDelete: RelationMetadataEntity[] = [];
for (const diff of relationsDiff) {
if (diff.type === 'CREATE') {
relationsToCreate.push(diff.value);
}
if (diff.type === 'REMOVE' && diff.path[diff.path.length - 1] !== 'id') {
relationsToDelete.push(diff.oldValue);
}
}
try {
// CREATE RELATIONS
await this.relationMetadataRepository.save(relationsToCreate);
// DELETE RELATIONS
if (relationsToDelete.length > 0) {
await this.relationMetadataRepository.delete(
relationsToDelete.map((relation) => relation.id),
);
}
await this.generateRelationMigrationsFromSync(
relationsToCreate,
relationsToDelete,
objectsInDB,
);
} catch (error) {
console.error('Sync of standard relations failed with:', error);
}
}
private async generateMigrationsFromSync(
objectsToCreate: ObjectMetadataEntity[],
_objectsToDelete: ObjectMetadataEntity[],
fieldsToCreate: FieldMetadataEntity[],
fieldsToDelete: FieldMetadataEntity[],
objectsInDB: ObjectMetadataEntity[],
) {
const migrationsToSave: Partial<WorkspaceMigrationEntity>[] = [];
if (objectsToCreate.length > 0) {
objectsToCreate.map((object) => {
const migrations = [
{
name: computeObjectTargetTable(object),
action: 'create',
} satisfies WorkspaceMigrationTableAction,
...Object.values(object.fields)
.filter((field) => field.type !== FieldMetadataType.RELATION)
.map(
(field) =>
({
name: computeObjectTargetTable(object),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
field,
),
}) satisfies WorkspaceMigrationTableAction,
),
];
migrationsToSave.push({
workspaceId: object.workspaceId,
isCustom: false,
migrations,
});
});
}
// TODO: handle object delete migrations.
// Note: we need to delete the relation first due to the DB constraint.
const objectsInDbById = objectsInDB.reduce(
(result, currentObject) => {
result[currentObject.id] = currentObject;
return result;
},
{} as Record<string, ObjectMetadataEntity>,
);
if (fieldsToCreate.length > 0) {
fieldsToCreate.map((field) => {
const migrations = [
{
name: computeObjectTargetTable(
objectsInDbById[field.objectMetadataId],
),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
field,
),
} satisfies WorkspaceMigrationTableAction,
];
migrationsToSave.push({
workspaceId: field.workspaceId,
isCustom: false,
migrations,
});
});
}
if (fieldsToDelete.length > 0) {
fieldsToDelete.map((field) => {
const migrations = [
{
name: computeObjectTargetTable(
objectsInDbById[field.objectMetadataId],
),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.DROP,
columnName: field.name,
},
],
} satisfies WorkspaceMigrationTableAction,
];
migrationsToSave.push({
workspaceId: field.workspaceId,
isCustom: false,
migrations,
});
});
}
await this.workspaceMigrationRepository.save(migrationsToSave);
// TODO: handle delete migrations
}
private async generateRelationMigrationsFromSync(
relationsToCreate: RelationMetadataEntity[],
_relationsToDelete: RelationMetadataEntity[],
objectsInDB: ObjectMetadataEntity[],
) {
const relationsMigrationsToSave: Partial<WorkspaceMigrationEntity>[] = [];
if (relationsToCreate.length > 0) {
relationsToCreate.map((relation) => {
const toObjectMetadata = objectsInDB.find(
(object) => object.id === relation.toObjectMetadataId,
);
const fromObjectMetadata = objectsInDB.find(
(object) => object.id === relation.fromObjectMetadataId,
);
if (!toObjectMetadata) {
throw new Error(
`ObjectMetadata with id ${relation.toObjectMetadataId} not found`,
);
}
if (!fromObjectMetadata) {
throw new Error(
`ObjectMetadata with id ${relation.fromObjectMetadataId} not found`,
);
}
const toFieldMetadata = toObjectMetadata.fields.find(
(field) => field.id === relation.toFieldMetadataId,
);
if (!toFieldMetadata) {
throw new Error(
`FieldMetadata with id ${relation.toFieldMetadataId} not found`,
);
}
const migrations = [
{
name: computeObjectTargetTable(toObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.RELATION,
columnName: `${camelCase(toFieldMetadata.name)}Id`,
referencedTableName:
computeObjectTargetTable(fromObjectMetadata),
referencedTableColumnName: 'id',
isUnique:
relation.relationType === RelationMetadataType.ONE_TO_ONE,
} satisfies WorkspaceMigrationColumnRelation,
],
} satisfies WorkspaceMigrationTableAction,
];
relationsMigrationsToSave.push({
workspaceId: relation.workspaceId,
isCustom: false,
migrations,
});
});
}
await this.workspaceMigrationRepository.save(relationsMigrationsToSave);
// TODO: handle delete migrations
}
}

11
yarn.lock
View File

@ -15371,6 +15371,15 @@ __metadata:
languageName: node
linkType: hard
"@types/lodash.omit@npm:^4.5.9":
version: 4.5.9
resolution: "@types/lodash.omit@npm:4.5.9"
dependencies:
"@types/lodash": "npm:*"
checksum: 3b60c8ee8e9a691392d9a3ceabb32c85f888784bd3307eac3de01aeb7ff37383dc8899f027fe852641f5e0f56158fb19785cc3d20a4922e85b5810f14cba23f6
languageName: node
linkType: hard
"@types/lodash.snakecase@npm:^4.1.7":
version: 4.1.9
resolution: "@types/lodash.snakecase@npm:4.1.9"
@ -43153,6 +43162,7 @@ __metadata:
"@sentry/profiling-node": "npm:^1.3.4"
"@types/lodash.isempty": "npm:^4.4.7"
"@types/lodash.isobject": "npm:^3.0.7"
"@types/lodash.omit": "npm:^4.5.9"
"@types/lodash.snakecase": "npm:^4.1.7"
"@types/lodash.upperfirst": "npm:^4.3.7"
"@types/react": "npm:^18.2.39"
@ -43176,6 +43186,7 @@ __metadata:
lodash.isempty: "npm:^4.4.0"
lodash.isobject: "npm:^3.0.2"
lodash.kebabcase: "npm:^4.1.1"
lodash.omit: "npm:^4.5.0"
lodash.snakecase: "npm:^4.1.1"
lodash.upperfirst: "npm:^4.3.1"
mailparser: "npm:^3.6.5"