[FlexibleSchema] Add IndexMetadata decorator (#5981)

## Context
Our Flexible Schema engine dynamically generates entities/tables/APIs
for us but was not flexible enough to build indexes in the DB. With more
and more features involving heavy queries such as Messaging, we are now
adding a new WorkspaceIndex() decorator for our standard objects (will
come later for custom objects). This decorator will give enough
information to the workspace sync metadata manager to generate the
proper migrations that will create or drop indexes on demand.
To be aligned with the rest of the engine, we are adding 2 new tables:
IndexMetadata and IndexFieldMetadata, that will store the info of our
indexes.

## Implementation

```typescript
@WorkspaceEntity({
  standardId: STANDARD_OBJECT_IDS.person,
  namePlural: 'people',
  labelSingular: 'Person',
  labelPlural: 'People',
  description: 'A person',
  icon: 'IconUser',
})
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
  @WorkspaceField({
    standardId: PERSON_STANDARD_FIELD_IDS.email,
    type: FieldMetadataType.EMAIL,
    label: 'Email',
    description: 'Contact’s Email',
    icon: 'IconMail',
  })
  @WorkspaceIndex()
  email: string;
```
By simply adding the WorkspaceIndex decorator, sync-metadata command
will create a new index for that column.
We can also add composite indexes, note that the order is important for
PSQL.
```typescript
@WorkspaceEntity({
  standardId: STANDARD_OBJECT_IDS.person,
  namePlural: 'people',
  labelSingular: 'Person',
  labelPlural: 'People',
  description: 'A person',
  icon: 'IconUser',
})
@WorkspaceIndex(['phone', 'email'])
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```

Currently composite fields and relation fields are not handled by
@WorkspaceIndex() and you will need to use this notation instead
```typescript
@WorkspaceIndex(['companyId', 'nameFirstName'])
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```
<img width="700" alt="Screenshot 2024-06-21 at 15 15 45"
src="https://github.com/twentyhq/twenty/assets/1834158/ac6da1d9-d315-40a4-9ba6-6ab9ae4709d4">

Next step: We might need to implement more complex index expressions,
this is why we have an expression column in IndexMetadata.
What I had in mind for the decorator, still open to discussion
```typescript
@WorkspaceIndex(['nameFirstName', 'nameLastName'], { expression: "$1 || ' ' || $2"})
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko 2024-06-22 12:39:57 +02:00 committed by GitHub
parent 0b4bfce324
commit e13dc7a1fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 871 additions and 10 deletions

View File

@ -0,0 +1,28 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddIndexMetadataTable1718985664968 implements MigrationInterface {
name = 'AddIndexMetadataTable1718985664968';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "metadata"."indexMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "workspaceId" character varying, "objectMetadataId" uuid NOT NULL, CONSTRAINT "PK_f73bb3c3678aee204e341f0ca4e" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE TABLE "metadata"."indexFieldMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "indexMetadataId" uuid NOT NULL, "fieldMetadataId" uuid NOT NULL, "order" integer NOT NULL, CONSTRAINT "PK_5928f67e43eff7d95aa79fd96fd" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "FK_051487e9b745cb175950130b63f" FOREIGN KEY ("objectMetadataId") REFERENCES "metadata"."objectMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_b20192c432612eb710801dd5664" FOREIGN KEY ("indexMetadataId") REFERENCES "metadata"."indexMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_be0950612a54b58c72bd62d629e" FOREIGN KEY ("fieldMetadataId") REFERENCES "metadata"."fieldMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "metadata"."indexFieldMetadata"`);
await queryRunner.query(`DROP TABLE "metadata"."indexMetadata"`);
}
}

View File

@ -9,6 +9,7 @@ import {
CreateDateColumn,
UpdateDateColumn,
Relation,
OneToMany,
} from 'typeorm';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
@ -18,6 +19,7 @@ import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadat
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
export enum FieldMetadataType {
UUID = 'UUID',
@ -119,6 +121,16 @@ export class FieldMetadataEntity<
)
toRelationMetadata: Relation<RelationMetadataEntity>;
@OneToMany(
() => IndexFieldMetadataEntity,
(indexFieldMetadata: IndexFieldMetadataEntity) =>
indexFieldMetadata.fieldMetadata,
{
cascade: true,
},
)
indexFieldMetadatas: Relation<IndexFieldMetadataEntity>;
@CreateDateColumn({ type: 'timestamptz' })
createdAt: Date;

View File

@ -0,0 +1,46 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
JoinColumn,
ManyToOne,
Relation,
} from 'typeorm';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
@Entity('indexFieldMetadata')
export class IndexFieldMetadataEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false })
indexMetadataId: string;
@ManyToOne(
() => IndexMetadataEntity,
(indexMetadata) => indexMetadata.indexFieldMetadatas,
{
onDelete: 'CASCADE',
},
)
@JoinColumn()
indexMetadata: Relation<IndexMetadataEntity>;
@Column({ nullable: false })
fieldMetadataId: string;
@ManyToOne(
() => FieldMetadataEntity,
(fieldMetadata) => fieldMetadata.indexFieldMetadatas,
{
onDelete: 'CASCADE',
},
)
@JoinColumn()
fieldMetadata: Relation<FieldMetadataEntity>;
@Column({ nullable: false })
order: number;
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
@Module({
imports: [TypeOrmModule.forFeature([IndexFieldMetadataEntity], 'metadata')],
providers: [],
exports: [],
})
export class IndexFieldMetadataModule {}

View File

@ -0,0 +1,43 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
JoinColumn,
ManyToOne,
Relation,
OneToMany,
} from 'typeorm';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
@Entity('indexMetadata')
export class IndexMetadataEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false })
name: string;
@Column({ nullable: true })
workspaceId: string;
@Column({ nullable: false, type: 'uuid' })
objectMetadataId: string;
@ManyToOne(() => ObjectMetadataEntity, (object) => object.indexes, {
onDelete: 'CASCADE',
})
@JoinColumn()
objectMetadata: Relation<ObjectMetadataEntity>;
@OneToMany(
() => IndexFieldMetadataEntity,
(indexFieldMetadata: IndexFieldMetadataEntity) =>
indexFieldMetadata.indexMetadata,
{
cascade: true,
},
)
indexFieldMetadatas: Relation<IndexFieldMetadataEntity[]>;
}

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
@Module({
imports: [TypeOrmModule.forFeature([IndexMetadataEntity], 'metadata')],
providers: [],
exports: [],
})
export class IndexMetadataModule {}

View File

@ -0,0 +1,11 @@
import { createHash } from 'crypto';
export const generateDeterministicIndexName = (columns: string[]): string => {
const hash = createHash('sha256');
columns.forEach((column) => {
hash.update(column);
});
return hash.digest('hex').slice(0, 27);
};

View File

@ -15,6 +15,7 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metad
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
@Entity('objectMetadata')
@Unique('IndexOnNameSingularAndWorkspaceIdUnique', [
@ -82,6 +83,11 @@ export class ObjectMetadataEntity implements ObjectMetadataInterface {
})
fields: Relation<FieldMetadataEntity[]>;
@OneToMany(() => FieldMetadataEntity, (field) => field.object, {
cascade: true,
})
indexes: Relation<IndexMetadataEntity[]>;
@OneToMany(
() => RelationMetadataEntity,
(relation: RelationMetadataEntity) => relation.fromObjectMetadata,

View File

@ -18,6 +18,11 @@ export enum WorkspaceMigrationColumnActionType {
export type WorkspaceMigrationRenamedEnum = { from: string; to: string };
export type WorkspaceMigrationEnum = string | WorkspaceMigrationRenamedEnum;
export enum WorkspaceMigrationIndexActionType {
CREATE = 'CREATE',
DROP = 'DROP',
}
export interface WorkspaceMigrationColumnDefinition {
columnName: string;
columnType: string;
@ -27,6 +32,12 @@ export interface WorkspaceMigrationColumnDefinition {
defaultValue?: any;
}
export interface WorkspaceMigrationIndexAction {
action: WorkspaceMigrationIndexActionType;
name: string;
columns: string[];
}
export interface WorkspaceMigrationColumnCreate
extends WorkspaceMigrationColumnDefinition {
action: WorkspaceMigrationColumnActionType.CREATE;
@ -105,6 +116,7 @@ export enum WorkspaceMigrationTableActionType {
CREATE_FOREIGN_TABLE = 'create_foreign_table',
DROP_FOREIGN_TABLE = 'drop_foreign_table',
ALTER_FOREIGN_TABLE = 'alter_foreign_table',
ALTER_INDEXES = 'alter_indexes',
}
export type WorkspaceMigrationTableAction = {
@ -113,6 +125,7 @@ export type WorkspaceMigrationTableAction = {
action: WorkspaceMigrationTableActionType;
columns?: WorkspaceMigrationColumnAction[];
foreignTable?: WorkspaceMigrationForeignTable;
indexes?: WorkspaceMigrationIndexAction[];
};
@Entity('workspaceMigration')

View File

@ -0,0 +1,37 @@
import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name';
import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage';
export interface WorkspaceIndexOptions {
columns?: string[];
}
export function WorkspaceIndex(): PropertyDecorator;
export function WorkspaceIndex(columns: string[]): ClassDecorator;
export function WorkspaceIndex(
columns?: string[],
): PropertyDecorator | ClassDecorator {
return (target: any, propertyKey: string | symbol) => {
if (propertyKey === undefined && columns === undefined) {
throw new Error('Class level WorkspaceIndex should be used with columns');
}
// TODO: handle composite field metadata types
// TODO: handle relation field metadata types
if (Array.isArray(columns) && columns.length > 0) {
metadataArgsStorage.addIndexes({
name: `IDX_${generateDeterministicIndexName(columns)}`,
columns,
target: target,
});
return;
}
metadataArgsStorage.addIndexes({
name: `IDX_${generateDeterministicIndexName([propertyKey.toString()])}`,
columns: [propertyKey.toString()],
target: target.constructor,
});
};
}

View File

@ -0,0 +1,17 @@
export interface WorkspaceIndexMetadataArgs {
/**
* Class to which index is applied.
*/
// eslint-disable-next-line @typescript-eslint/ban-types
readonly target: Function;
/*
* Index name.
*/
name: string;
/*
* Index columns.
*/
columns: string[];
}

View File

@ -5,6 +5,7 @@ import { WorkspaceFieldMetadataArgs } from 'src/engine/twenty-orm/interfaces/wor
import { WorkspaceEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-entity-metadata-args.interface';
import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface';
import { WorkspaceExtendedEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-extended-entity-metadata-args.interface';
import { WorkspaceIndexMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface';
export class MetadataArgsStorage {
private readonly entities: WorkspaceEntityMetadataArgs[] = [];
@ -13,6 +14,7 @@ export class MetadataArgsStorage {
private readonly relations: WorkspaceRelationMetadataArgs[] = [];
private readonly dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] =
[];
private readonly indexes: WorkspaceIndexMetadataArgs[] = [];
addEntities(...entities: WorkspaceEntityMetadataArgs[]): void {
this.entities.push(...entities);
@ -32,6 +34,10 @@ export class MetadataArgsStorage {
this.relations.push(...relations);
}
addIndexes(...indexes: WorkspaceIndexMetadataArgs[]): void {
this.indexes.push(...indexes);
}
addDynamicRelations(
...dynamicRelations: WorkspaceDynamicRelationMetadataArgs[]
): void {
@ -93,6 +99,16 @@ export class MetadataArgsStorage {
return this.filterByTarget(this.relations, target);
}
filterIndexes(target: Function | string): WorkspaceIndexMetadataArgs[];
filterIndexes(target: (Function | string)[]): WorkspaceIndexMetadataArgs[];
filterIndexes(
target: (Function | string) | (Function | string)[],
): WorkspaceIndexMetadataArgs[] {
return this.filterByTarget(this.indexes, target);
}
filterDynamicRelations(
target: Function | string,
): WorkspaceDynamicRelationMetadataArgs[];

View File

@ -1,3 +1,5 @@
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
import { WorkspaceMigrationObjectFactory } from './workspace-migration-object.factory';
import { WorkspaceMigrationFieldFactory } from './workspace-migration-field.factory';
import { WorkspaceMigrationRelationFactory } from './workspace-migration-relation.factory';
@ -6,4 +8,5 @@ export const workspaceMigrationBuilderFactories = [
WorkspaceMigrationObjectFactory,
WorkspaceMigrationFieldFactory,
WorkspaceMigrationRelationFactory,
WorkspaceMigrationIndexFactory,
];

View File

@ -0,0 +1,156 @@
import { Injectable } from '@nestjs/common';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import {
WorkspaceMigrationEntity,
WorkspaceMigrationIndexActionType,
WorkspaceMigrationTableActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
@Injectable()
export class WorkspaceMigrationIndexFactory {
constructor() {}
async create(
originalObjectMetadataCollection: ObjectMetadataEntity[],
indexMetadataCollection: IndexMetadataEntity[],
action: WorkspaceMigrationBuilderAction,
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const originalObjectMetadataMap = Object.fromEntries(
originalObjectMetadataCollection.map((obj) => [obj.id, obj]),
);
const indexMetadataByObjectMetadataMap = new Map<
ObjectMetadataEntity,
IndexMetadataEntity[]
>();
indexMetadataCollection.forEach((currentIndexMetadata) => {
const objectMetadata =
originalObjectMetadataMap[currentIndexMetadata.objectMetadataId];
if (!objectMetadata) {
throw new Error(
`Object metadata with id ${currentIndexMetadata.objectMetadataId} not found`,
);
}
if (!indexMetadataByObjectMetadataMap.has(objectMetadata)) {
indexMetadataByObjectMetadataMap.set(objectMetadata, []);
}
indexMetadataByObjectMetadataMap
?.get(objectMetadata)
?.push(currentIndexMetadata);
});
switch (action) {
case WorkspaceMigrationBuilderAction.CREATE:
return this.createIndexMigration(indexMetadataByObjectMetadataMap);
case WorkspaceMigrationBuilderAction.DELETE:
return this.deleteIndexMigration(indexMetadataByObjectMetadataMap);
default:
return [];
}
}
private async createIndexMigration(
indexMetadataByObjectMetadataMap: Map<
ObjectMetadataEntity,
IndexMetadataEntity[]
>,
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
for (const [
objectMetadata,
indexMetadataCollection,
] of indexMetadataByObjectMetadataMap) {
const targetTable = computeObjectTargetTable(objectMetadata);
const fieldsById = Object.fromEntries(
objectMetadata.fields.map((field) => [field.id, field]),
);
const indexes = indexMetadataCollection.map((indexMetadata) => ({
name: indexMetadata.name,
action: WorkspaceMigrationIndexActionType.CREATE,
columns: indexMetadata.indexFieldMetadatas
.sort((a, b) => a.order - b.order)
.map((indexFieldMetadata) => {
const fieldMetadata =
fieldsById[indexFieldMetadata.fieldMetadataId];
if (!fieldMetadata) {
throw new Error(
`Field metadata with id ${indexFieldMetadata.fieldMetadataId} not found in object metadata with id ${objectMetadata.id}`,
);
}
return fieldMetadata.name;
}),
}));
workspaceMigrations.push({
workspaceId: objectMetadata.workspaceId,
name: generateMigrationName(
`create-${objectMetadata.nameSingular}-indexes`,
),
isCustom: false,
migrations: [
{
name: targetTable,
action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
indexes,
},
],
});
}
return workspaceMigrations;
}
private async deleteIndexMigration(
indexMetadataByObjectMetadataMap: Map<
ObjectMetadataEntity,
IndexMetadataEntity[]
>,
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
for (const [
objectMetadata,
indexMetadataCollection,
] of indexMetadataByObjectMetadataMap) {
const targetTable = computeObjectTargetTable(objectMetadata);
const indexes = indexMetadataCollection.map((indexMetadata) => ({
name: indexMetadata.name,
action: WorkspaceMigrationIndexActionType.DROP,
columns: [],
}));
workspaceMigrations.push({
workspaceId: objectMetadata.workspaceId,
name: generateMigrationName(
`delete-${objectMetadata.nameSingular}-indexes`,
),
isCustom: false,
migrations: [
{
name: targetTable,
action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
indexes,
},
],
});
}
return workspaceMigrations;
}
}

View File

@ -5,6 +5,7 @@ import {
Table,
TableColumn,
TableForeignKey,
TableIndex,
TableUnique,
} from 'typeorm';
@ -20,6 +21,8 @@ import {
WorkspaceMigrationColumnDropRelation,
WorkspaceMigrationTableActionType,
WorkspaceMigrationForeignTable,
WorkspaceMigrationIndexAction,
WorkspaceMigrationIndexActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
@ -137,6 +140,7 @@ export class WorkspaceMigrationRunnerService {
tableMigration.columns,
);
}
break;
}
case WorkspaceMigrationTableActionType.DROP:
@ -163,6 +167,17 @@ export class WorkspaceMigrationRunnerService {
tableMigration.columns,
);
break;
case WorkspaceMigrationTableActionType.ALTER_INDEXES:
if (tableMigration.indexes && tableMigration.indexes.length > 0) {
await this.handleIndexesChanges(
queryRunner,
schemaName,
tableMigration.newName ?? tableMigration.name,
tableMigration.indexes,
);
}
break;
default:
throw new Error(
`Migration table action ${tableMigration.action} not supported`,
@ -170,6 +185,32 @@ export class WorkspaceMigrationRunnerService {
}
}
private async handleIndexesChanges(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
indexes: WorkspaceMigrationIndexAction[],
) {
for (const index of indexes) {
switch (index.action) {
case WorkspaceMigrationIndexActionType.CREATE:
await queryRunner.createIndex(
`${schemaName}.${tableName}`,
new TableIndex({
name: index.name,
columnNames: index.columns,
}),
);
break;
case WorkspaceMigrationIndexActionType.DROP:
await queryRunner.dropIndex(`${schemaName}.${tableName}`, index.name);
break;
default:
throw new Error(`Migration index action not supported`);
}
}
}
/**
* Creates a table for a given schema and table name
*

View File

@ -5,7 +5,6 @@ import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
@ -28,7 +27,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
private readonly workspaceHealthService: WorkspaceHealthService,
private readonly dataSourceService: DataSourceService,
private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
private readonly workspaceService: WorkspaceService,
) {
super();
}
@ -37,9 +35,8 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
_passedParam: string[],
options: RunWorkspaceMigrationsOptions,
): Promise<void> {
const workspaceIds = options.workspaceId
? [options.workspaceId]
: await this.workspaceService.getWorkspaceIds();
// TODO: re-implement load index from workspaceService, this is breaking the logger
const workspaceIds = options.workspaceId ? [options.workspaceId] : [];
for (const workspaceId of workspaceIds) {
try {
@ -105,7 +102,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: false,
required: true,
})
parseWorkspaceId(value: string): string {
return value;

View File

@ -1,3 +1,5 @@
import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator';
import { WorkspaceFieldComparator } from './workspace-field.comparator';
import { WorkspaceObjectComparator } from './workspace-object.comparator';
import { WorkspaceRelationComparator } from './workspace-relation.comparator';
@ -6,4 +8,5 @@ export const workspaceSyncMetadataComparators = [
WorkspaceFieldComparator,
WorkspaceObjectComparator,
WorkspaceRelationComparator,
WorkspaceIndexComparator,
];

View File

@ -0,0 +1,90 @@
import { Injectable } from '@nestjs/common';
import diff from 'microdiff';
import {
IndexComparatorResult,
ComparatorAction,
} from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { transformMetadataForComparison } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';
const propertiesToIgnore = ['createdAt', 'updatedAt', 'indexFieldMetadatas'];
@Injectable()
export class WorkspaceIndexComparator {
constructor() {}
compare(
originalIndexMetadataCollection: IndexMetadataEntity[],
standardIndexMetadataCollection: Partial<IndexMetadataEntity>[],
): IndexComparatorResult[] {
const results: IndexComparatorResult[] = [];
// Create a map of standard relations
const standardIndexMetadataMap = transformMetadataForComparison(
standardIndexMetadataCollection,
{
keyFactory(indexMetadata) {
return `${indexMetadata.name}`;
},
},
);
const originalIndexMetadataCollectionWithColumns =
originalIndexMetadataCollection.map((indexMetadata) => {
return {
...indexMetadata,
columns: indexMetadata.indexFieldMetadatas.map(
(indexFieldMetadata) => indexFieldMetadata.fieldMetadata.name,
),
indexFieldMetadatas: undefined,
};
});
// Create a filtered map of original relations
// We filter out 'id' later because we need it to remove the relation from DB
const originalIndexMetadataMap = transformMetadataForComparison(
originalIndexMetadataCollectionWithColumns,
{
shouldIgnoreProperty: (property) =>
propertiesToIgnore.includes(property),
keyFactory(indexMetadata) {
return `${indexMetadata.name}`;
},
},
);
// Compare indexes
const indexesDifferences = diff(
originalIndexMetadataMap,
standardIndexMetadataMap,
);
for (const difference of indexesDifferences) {
switch (difference.type) {
case 'CREATE': {
results.push({
action: ComparatorAction.CREATE,
object: difference.value,
});
break;
}
case 'REMOVE': {
if (difference.path[difference.path.length - 1] !== 'id') {
results.push({
action: ComparatorAction.DELETE,
object: difference.oldValue,
});
}
break;
}
default:
break;
}
}
return results;
}
}

View File

@ -1,3 +1,5 @@
import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory';
import { FeatureFlagFactory } from './feature-flags.factory';
import { StandardFieldFactory } from './standard-field.factory';
import { StandardObjectFactory } from './standard-object.factory';
@ -8,4 +10,5 @@ export const workspaceSyncMetadataFactories = [
StandardFieldFactory,
StandardObjectFactory,
StandardRelationFactory,
StandardIndexFactory,
];

View File

@ -0,0 +1,69 @@
import { Injectable } from '@nestjs/common';
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage';
@Injectable()
export class StandardIndexFactory {
create(
standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[],
context: WorkspaceSyncContext,
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Partial<IndexMetadataEntity>[] {
return standardObjectMetadataDefinitions.flatMap((standardObjectMetadata) =>
this.createIndexMetadata(
standardObjectMetadata,
context,
originalObjectMetadataMap,
workspaceFeatureFlagsMap,
),
);
}
private createIndexMetadata(
target: typeof BaseWorkspaceEntity,
context: WorkspaceSyncContext,
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Partial<IndexMetadataEntity>[] {
const workspaceEntity = metadataArgsStorage.filterEntities(target);
if (!workspaceEntity) {
throw new Error(
`Object metadata decorator not found, can't parse ${target.name}`,
);
}
const workspaceIndexMetadataArgsCollection =
metadataArgsStorage.filterIndexes(target);
return workspaceIndexMetadataArgsCollection.map(
(workspaceIndexMetadataArgs) => {
const objectMetadata =
originalObjectMetadataMap[workspaceEntity.nameSingular];
if (!objectMetadata) {
throw new Error(
`Object metadata not found for ${workspaceEntity.nameSingular}`,
);
}
const indexMetadata: PartialIndexMetadata = {
workspaceId: context.workspaceId,
objectMetadataId: objectMetadata.id,
name: workspaceIndexMetadataArgs.name,
columns: workspaceIndexMetadataArgs.columns,
};
return indexMetadata;
},
);
}
}

View File

@ -1,5 +1,6 @@
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { ComputedPartialFieldMetadata } from './partial-field-metadata.interface';
import { ComputedPartialWorkspaceEntity } from './partial-object-metadata.interface';
@ -49,3 +50,8 @@ export type RelationComparatorResult =
| ComparatorCreateResult<Partial<RelationMetadataEntity>>
| ComparatorDeleteResult<RelationMetadataEntity>
| ComparatorUpdateResult<Partial<RelationMetadataEntity>>;
export type IndexComparatorResult =
| ComparatorCreateResult<Partial<IndexMetadataEntity>>
| ComparatorUpdateResult<Partial<IndexMetadataEntity>>
| ComparatorDeleteResult<IndexMetadataEntity>;

View File

@ -0,0 +1,8 @@
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
export type PartialIndexMetadata = Omit<
IndexMetadataEntity,
'id' | 'objectMetadata' | 'indexFieldMetadatas'
> & {
columns: string[];
};

View File

@ -11,6 +11,7 @@ import { v4 as uuidV4 } from 'uuid';
import { DeepPartial } from 'typeorm/common/DeepPartial';
import { PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import {
@ -22,6 +23,7 @@ import { FieldMetadataComplexOption } from 'src/engine/metadata-modules/field-me
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
import { FieldMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import { ObjectMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-object.factory';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
@Injectable()
export class WorkspaceMetadataUpdaterService {
@ -230,6 +232,69 @@ export class WorkspaceMetadataUpdaterService {
};
}
async updateIndexMetadata(
manager: EntityManager,
storage: WorkspaceSyncStorage,
originalObjectMetadataCollection: ObjectMetadataEntity[],
): Promise<{
createdIndexMetadataCollection: IndexMetadataEntity[];
}> {
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
const convertIndexMetadataForSaving = (
indexMetadata: PartialIndexMetadata,
) => {
const convertIndexFieldMetadataForSaving = (
column: string,
order: number,
) => {
const fieldMetadata = originalObjectMetadataCollection
.find((object) => object.id === indexMetadata.objectMetadataId)
?.fields.find((field) => column === field.name);
if (!fieldMetadata) {
throw new Error(`
Field metadata not found for column ${column} in object ${indexMetadata.objectMetadataId}
`);
}
return {
fieldMetadataId: fieldMetadata.id,
order,
};
};
return {
...indexMetadata,
indexFieldMetadatas: indexMetadata.columns.map((column, index) =>
convertIndexFieldMetadataForSaving(column, index),
),
};
};
/**
* Create index metadata
*/
const createdIndexMetadataCollection = await indexMetadataRepository.save(
storage.indexMetadataCreateCollection.map(convertIndexMetadataForSaving),
);
/**
* Delete index metadata
*/
if (storage.indexMetadataDeleteCollection.length > 0) {
await indexMetadataRepository.delete(
storage.indexMetadataDeleteCollection.map(
(indexMetadata) => indexMetadata.id,
),
);
}
return {
createdIndexMetadataCollection,
};
}
/**
* Update entities in the database
* @param manager EntityManager

View File

@ -48,7 +48,7 @@ export class WorkspaceSyncFieldMetadataService {
relations: ['dataSource', 'fields'],
});
// Filter out custom objects
// Filter out non-custom objects
const customObjectMetadataCollection =
originalObjectMetadataCollection.filter(
(objectMetadata) => objectMetadata.isCustom,
@ -61,7 +61,7 @@ export class WorkspaceSyncFieldMetadataService {
workspaceFeatureFlagsMap,
);
// Loop over all standard objects and compare them with the objects in DB
// Loop over all custom objects from the DB and compare their fields with standard fields
for (const customObjectMetadata of customObjectMetadataCollection) {
// Also, maybe it's better to refactor a bit and move generation part into a separate module ?
const standardObjectMetadata = computeStandardObject(

View File

@ -0,0 +1,119 @@
import { Injectable, Logger } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator';
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
@Injectable()
export class WorkspaceSyncIndexMetadataService {
private readonly logger = new Logger(WorkspaceSyncIndexMetadataService.name);
constructor(
private readonly standardIndexFactory: StandardIndexFactory,
private readonly workspaceIndexComparator: WorkspaceIndexComparator,
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
private readonly workspaceMigrationIndexFactory: WorkspaceMigrationIndexFactory,
) {}
async synchronize(
context: WorkspaceSyncContext,
manager: EntityManager,
storage: WorkspaceSyncStorage,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Promise<Partial<WorkspaceMigrationEntity>[]> {
this.logger.log('Syncing index metadata');
const objectMetadataRepository =
manager.getRepository(ObjectMetadataEntity);
// Retrieve object metadata collection from DB
const originalObjectMetadataCollection =
await objectMetadataRepository.find({
where: {
workspaceId: context.workspaceId,
// We're only interested in standard fields
fields: { isCustom: false },
isCustom: false,
},
relations: ['dataSource', 'fields', 'indexes'],
});
// Create map of object metadata & field metadata by unique identifier
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
originalObjectMetadataCollection,
// Relation are based on the singular name
(objectMetadata) => objectMetadata.nameSingular,
);
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
const originalIndexMetadataCollection = await indexMetadataRepository.find({
where: {
workspaceId: context.workspaceId,
},
relations: ['indexFieldMetadatas.fieldMetadata'],
});
// Generate index metadata from models
const standardIndexMetadataCollection = this.standardIndexFactory.create(
standardObjectMetadataDefinitions,
context,
originalObjectMetadataMap,
workspaceFeatureFlagsMap,
);
const indexComparatorResults = this.workspaceIndexComparator.compare(
originalIndexMetadataCollection,
standardIndexMetadataCollection,
);
for (const indexComparatorResult of indexComparatorResults) {
if (indexComparatorResult.action === ComparatorAction.CREATE) {
storage.addCreateIndexMetadata(indexComparatorResult.object);
} else if (indexComparatorResult.action === ComparatorAction.DELETE) {
storage.addDeleteIndexMetadata(indexComparatorResult.object);
}
}
const metadataIndexUpdaterResult =
await this.workspaceMetadataUpdaterService.updateIndexMetadata(
manager,
storage,
originalObjectMetadataCollection,
);
// Create migrations
const createIndexWorkspaceMigrations =
await this.workspaceMigrationIndexFactory.create(
originalObjectMetadataCollection,
metadataIndexUpdaterResult.createdIndexMetadataCollection,
WorkspaceMigrationBuilderAction.CREATE,
);
const deleteIndexWorkspaceMigrations =
await this.workspaceMigrationIndexFactory.create(
originalObjectMetadataCollection,
storage.indexMetadataDeleteCollection,
WorkspaceMigrationBuilderAction.DELETE,
);
return [
...createIndexWorkspaceMigrations,
...deleteIndexWorkspaceMigrations,
];
}
}

View File

@ -112,6 +112,10 @@ export class WorkspaceSyncObjectMetadataService {
/**
* COMPARE FIELD METADATA
* NOTE: This should be moved to WorkspaceSyncFieldMetadataService for more clarity since
* this code only adds field metadata to the storage but it's actually used in the other service.
* NOTE2: WorkspaceSyncFieldMetadataService has been added for custom fields sync, it should be refactored to handle
* both custom and non-custom fields.
*/
const fieldComparatorResults = this.workspaceFieldComparator.compare(
originalObjectMetadata,

View File

@ -4,6 +4,7 @@ import { ComputedPartialFieldMetadata } from 'src/engine/workspace-manager/works
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
export class WorkspaceSyncStorage {
// Object metadata
@ -25,10 +26,17 @@ export class WorkspaceSyncStorage {
// Relation metadata
private readonly _relationMetadataCreateCollection: Partial<RelationMetadataEntity>[] =
[];
private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] =
[];
private readonly _relationMetadataUpdateCollection: Partial<RelationMetadataEntity>[] =
[];
private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] =
[];
// Index metadata
private readonly _indexMetadataCreateCollection: Partial<IndexMetadataEntity>[] =
[];
private readonly _indexMetadataUpdateCollection: Partial<IndexMetadataEntity>[] =
[];
private readonly _indexMetadataDeleteCollection: IndexMetadataEntity[] = [];
constructor() {}
@ -68,6 +76,18 @@ export class WorkspaceSyncStorage {
return this._relationMetadataDeleteCollection;
}
get indexMetadataCreateCollection() {
return this._indexMetadataCreateCollection;
}
get indexMetadataUpdateCollection() {
return this._indexMetadataUpdateCollection;
}
get indexMetadataDeleteCollection() {
return this._indexMetadataDeleteCollection;
}
addCreateObjectMetadata(object: ComputedPartialWorkspaceEntity) {
this._objectMetadataCreateCollection.push(object);
}
@ -107,4 +127,16 @@ export class WorkspaceSyncStorage {
addDeleteRelationMetadata(relation: RelationMetadataEntity) {
this._relationMetadataDeleteCollection.push(relation);
}
addCreateIndexMetadata(index: Partial<IndexMetadataEntity>) {
this._indexMetadataCreateCollection.push(index);
}
addUpdateIndexMetadata(index: Partial<IndexMetadataEntity>) {
this._indexMetadataUpdateCollection.push(index);
}
addDeleteIndexMetadata(index: IndexMetadataEntity) {
this._indexMetadataDeleteCollection.push(index);
}
}

View File

@ -16,6 +16,7 @@ import { WorkspaceSyncRelationMetadataService } from 'src/engine/workspace-manag
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
@Module({
imports: [
@ -41,6 +42,7 @@ import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspa
WorkspaceSyncRelationMetadataService,
WorkspaceSyncFieldMetadataService,
WorkspaceSyncMetadataService,
WorkspaceSyncIndexMetadataService,
],
exports: [...workspaceSyncMetadataFactories, WorkspaceSyncMetadataService],
})

View File

@ -13,6 +13,7 @@ import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
interface SynchronizeOptions {
applyChanges?: boolean;
@ -31,6 +32,7 @@ export class WorkspaceSyncMetadataService {
private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService,
private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService,
) {}
/**
@ -97,11 +99,21 @@ export class WorkspaceSyncMetadataService {
workspaceFeatureFlagsMap,
);
// 4 - Sync standard indexes on standard objects
const workspaceIndexMigrations =
await this.workspaceSyncIndexMetadataService.synchronize(
context,
manager,
storage,
workspaceFeatureFlagsMap,
);
// Save workspace migrations into the database
workspaceMigrations = await workspaceMigrationRepository.save([
...workspaceObjectMigrations,
...workspaceFieldMigrations,
...workspaceRelationMigrations,
...workspaceIndexMigrations,
]);
// If we're running a dry run, rollback the transaction and do not execute migrations