mirror of
https://github.com/twentyhq/twenty.git
synced 2024-09-11 15:35:36 +03:00
[FlexibleSchema] Add IndexMetadata decorator (#5981)
## Context Our Flexible Schema engine dynamically generates entities/tables/APIs for us but was not flexible enough to build indexes in the DB. With more and more features involving heavy queries such as Messaging, we are now adding a new WorkspaceIndex() decorator for our standard objects (will come later for custom objects). This decorator will give enough information to the workspace sync metadata manager to generate the proper migrations that will create or drop indexes on demand. To be aligned with the rest of the engine, we are adding 2 new tables: IndexMetadata and IndexFieldMetadata, that will store the info of our indexes. ## Implementation ```typescript @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', labelSingular: 'Person', labelPlural: 'People', description: 'A person', icon: 'IconUser', }) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceField({ standardId: PERSON_STANDARD_FIELD_IDS.email, type: FieldMetadataType.EMAIL, label: 'Email', description: 'Contact’s Email', icon: 'IconMail', }) @WorkspaceIndex() email: string; ``` By simply adding the WorkspaceIndex decorator, sync-metadata command will create a new index for that column. We can also add composite indexes, note that the order is important for PSQL. ```typescript @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', labelSingular: 'Person', labelPlural: 'People', description: 'A person', icon: 'IconUser', }) @WorkspaceIndex(['phone', 'email']) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` Currently composite fields and relation fields are not handled by @WorkspaceIndex() and you will need to use this notation instead ```typescript @WorkspaceIndex(['companyId', 'nameFirstName']) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` <img width="700" alt="Screenshot 2024-06-21 at 15 15 45" src="https://github.com/twentyhq/twenty/assets/1834158/ac6da1d9-d315-40a4-9ba6-6ab9ae4709d4"> Next step: We might need to implement more complex index expressions, this is why we have an expression column in IndexMetadata. What I had in mind for the decorator, still open to discussion ```typescript @WorkspaceIndex(['nameFirstName', 'nameLastName'], { expression: "$1 || ' ' || $2"}) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
parent
0b4bfce324
commit
e13dc7a1fc
@ -0,0 +1,28 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddIndexMetadataTable1718985664968 implements MigrationInterface {
|
||||
name = 'AddIndexMetadataTable1718985664968';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE "metadata"."indexMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "workspaceId" character varying, "objectMetadataId" uuid NOT NULL, CONSTRAINT "PK_f73bb3c3678aee204e341f0ca4e" PRIMARY KEY ("id"))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE "metadata"."indexFieldMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "indexMetadataId" uuid NOT NULL, "fieldMetadataId" uuid NOT NULL, "order" integer NOT NULL, CONSTRAINT "PK_5928f67e43eff7d95aa79fd96fd" PRIMARY KEY ("id"))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "FK_051487e9b745cb175950130b63f" FOREIGN KEY ("objectMetadataId") REFERENCES "metadata"."objectMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_b20192c432612eb710801dd5664" FOREIGN KEY ("indexMetadataId") REFERENCES "metadata"."indexMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_be0950612a54b58c72bd62d629e" FOREIGN KEY ("fieldMetadataId") REFERENCES "metadata"."fieldMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP TABLE "metadata"."indexFieldMetadata"`);
|
||||
await queryRunner.query(`DROP TABLE "metadata"."indexMetadata"`);
|
||||
}
|
||||
}
|
@ -9,6 +9,7 @@ import {
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
Relation,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
|
||||
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
|
||||
@ -18,6 +19,7 @@ import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadat
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
|
||||
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
|
||||
|
||||
export enum FieldMetadataType {
|
||||
UUID = 'UUID',
|
||||
@ -119,6 +121,16 @@ export class FieldMetadataEntity<
|
||||
)
|
||||
toRelationMetadata: Relation<RelationMetadataEntity>;
|
||||
|
||||
@OneToMany(
|
||||
() => IndexFieldMetadataEntity,
|
||||
(indexFieldMetadata: IndexFieldMetadataEntity) =>
|
||||
indexFieldMetadata.fieldMetadata,
|
||||
{
|
||||
cascade: true,
|
||||
},
|
||||
)
|
||||
indexFieldMetadatas: Relation<IndexFieldMetadataEntity>;
|
||||
|
||||
@CreateDateColumn({ type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
|
@ -0,0 +1,46 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
Relation,
|
||||
} from 'typeorm';
|
||||
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
|
||||
@Entity('indexFieldMetadata')
|
||||
export class IndexFieldMetadataEntity {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ nullable: false })
|
||||
indexMetadataId: string;
|
||||
|
||||
@ManyToOne(
|
||||
() => IndexMetadataEntity,
|
||||
(indexMetadata) => indexMetadata.indexFieldMetadatas,
|
||||
{
|
||||
onDelete: 'CASCADE',
|
||||
},
|
||||
)
|
||||
@JoinColumn()
|
||||
indexMetadata: Relation<IndexMetadataEntity>;
|
||||
|
||||
@Column({ nullable: false })
|
||||
fieldMetadataId: string;
|
||||
|
||||
@ManyToOne(
|
||||
() => FieldMetadataEntity,
|
||||
(fieldMetadata) => fieldMetadata.indexFieldMetadatas,
|
||||
{
|
||||
onDelete: 'CASCADE',
|
||||
},
|
||||
)
|
||||
@JoinColumn()
|
||||
fieldMetadata: Relation<FieldMetadataEntity>;
|
||||
|
||||
@Column({ nullable: false })
|
||||
order: number;
|
||||
}
|
@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([IndexFieldMetadataEntity], 'metadata')],
|
||||
providers: [],
|
||||
exports: [],
|
||||
})
|
||||
export class IndexFieldMetadataModule {}
|
@ -0,0 +1,43 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
Relation,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity';
|
||||
|
||||
@Entity('indexMetadata')
|
||||
export class IndexMetadataEntity {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ nullable: false })
|
||||
name: string;
|
||||
|
||||
@Column({ nullable: true })
|
||||
workspaceId: string;
|
||||
|
||||
@Column({ nullable: false, type: 'uuid' })
|
||||
objectMetadataId: string;
|
||||
|
||||
@ManyToOne(() => ObjectMetadataEntity, (object) => object.indexes, {
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
@JoinColumn()
|
||||
objectMetadata: Relation<ObjectMetadataEntity>;
|
||||
|
||||
@OneToMany(
|
||||
() => IndexFieldMetadataEntity,
|
||||
(indexFieldMetadata: IndexFieldMetadataEntity) =>
|
||||
indexFieldMetadata.indexMetadata,
|
||||
{
|
||||
cascade: true,
|
||||
},
|
||||
)
|
||||
indexFieldMetadatas: Relation<IndexFieldMetadataEntity[]>;
|
||||
}
|
@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([IndexMetadataEntity], 'metadata')],
|
||||
providers: [],
|
||||
exports: [],
|
||||
})
|
||||
export class IndexMetadataModule {}
|
@ -0,0 +1,11 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
export const generateDeterministicIndexName = (columns: string[]): string => {
|
||||
const hash = createHash('sha256');
|
||||
|
||||
columns.forEach((column) => {
|
||||
hash.update(column);
|
||||
});
|
||||
|
||||
return hash.digest('hex').slice(0, 27);
|
||||
};
|
@ -15,6 +15,7 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metad
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
@Entity('objectMetadata')
|
||||
@Unique('IndexOnNameSingularAndWorkspaceIdUnique', [
|
||||
@ -82,6 +83,11 @@ export class ObjectMetadataEntity implements ObjectMetadataInterface {
|
||||
})
|
||||
fields: Relation<FieldMetadataEntity[]>;
|
||||
|
||||
@OneToMany(() => FieldMetadataEntity, (field) => field.object, {
|
||||
cascade: true,
|
||||
})
|
||||
indexes: Relation<IndexMetadataEntity[]>;
|
||||
|
||||
@OneToMany(
|
||||
() => RelationMetadataEntity,
|
||||
(relation: RelationMetadataEntity) => relation.fromObjectMetadata,
|
||||
|
@ -18,6 +18,11 @@ export enum WorkspaceMigrationColumnActionType {
|
||||
export type WorkspaceMigrationRenamedEnum = { from: string; to: string };
|
||||
export type WorkspaceMigrationEnum = string | WorkspaceMigrationRenamedEnum;
|
||||
|
||||
export enum WorkspaceMigrationIndexActionType {
|
||||
CREATE = 'CREATE',
|
||||
DROP = 'DROP',
|
||||
}
|
||||
|
||||
export interface WorkspaceMigrationColumnDefinition {
|
||||
columnName: string;
|
||||
columnType: string;
|
||||
@ -27,6 +32,12 @@ export interface WorkspaceMigrationColumnDefinition {
|
||||
defaultValue?: any;
|
||||
}
|
||||
|
||||
export interface WorkspaceMigrationIndexAction {
|
||||
action: WorkspaceMigrationIndexActionType;
|
||||
name: string;
|
||||
columns: string[];
|
||||
}
|
||||
|
||||
export interface WorkspaceMigrationColumnCreate
|
||||
extends WorkspaceMigrationColumnDefinition {
|
||||
action: WorkspaceMigrationColumnActionType.CREATE;
|
||||
@ -105,6 +116,7 @@ export enum WorkspaceMigrationTableActionType {
|
||||
CREATE_FOREIGN_TABLE = 'create_foreign_table',
|
||||
DROP_FOREIGN_TABLE = 'drop_foreign_table',
|
||||
ALTER_FOREIGN_TABLE = 'alter_foreign_table',
|
||||
ALTER_INDEXES = 'alter_indexes',
|
||||
}
|
||||
|
||||
export type WorkspaceMigrationTableAction = {
|
||||
@ -113,6 +125,7 @@ export type WorkspaceMigrationTableAction = {
|
||||
action: WorkspaceMigrationTableActionType;
|
||||
columns?: WorkspaceMigrationColumnAction[];
|
||||
foreignTable?: WorkspaceMigrationForeignTable;
|
||||
indexes?: WorkspaceMigrationIndexAction[];
|
||||
};
|
||||
|
||||
@Entity('workspaceMigration')
|
||||
|
@ -0,0 +1,37 @@
|
||||
import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name';
|
||||
import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage';
|
||||
|
||||
export interface WorkspaceIndexOptions {
|
||||
columns?: string[];
|
||||
}
|
||||
|
||||
export function WorkspaceIndex(): PropertyDecorator;
|
||||
export function WorkspaceIndex(columns: string[]): ClassDecorator;
|
||||
export function WorkspaceIndex(
|
||||
columns?: string[],
|
||||
): PropertyDecorator | ClassDecorator {
|
||||
return (target: any, propertyKey: string | symbol) => {
|
||||
if (propertyKey === undefined && columns === undefined) {
|
||||
throw new Error('Class level WorkspaceIndex should be used with columns');
|
||||
}
|
||||
|
||||
// TODO: handle composite field metadata types
|
||||
// TODO: handle relation field metadata types
|
||||
|
||||
if (Array.isArray(columns) && columns.length > 0) {
|
||||
metadataArgsStorage.addIndexes({
|
||||
name: `IDX_${generateDeterministicIndexName(columns)}`,
|
||||
columns,
|
||||
target: target,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
metadataArgsStorage.addIndexes({
|
||||
name: `IDX_${generateDeterministicIndexName([propertyKey.toString()])}`,
|
||||
columns: [propertyKey.toString()],
|
||||
target: target.constructor,
|
||||
});
|
||||
};
|
||||
}
|
@ -0,0 +1,17 @@
|
||||
export interface WorkspaceIndexMetadataArgs {
|
||||
/**
|
||||
* Class to which index is applied.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
readonly target: Function;
|
||||
|
||||
/*
|
||||
* Index name.
|
||||
*/
|
||||
name: string;
|
||||
|
||||
/*
|
||||
* Index columns.
|
||||
*/
|
||||
columns: string[];
|
||||
}
|
@ -5,6 +5,7 @@ import { WorkspaceFieldMetadataArgs } from 'src/engine/twenty-orm/interfaces/wor
|
||||
import { WorkspaceEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-entity-metadata-args.interface';
|
||||
import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface';
|
||||
import { WorkspaceExtendedEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-extended-entity-metadata-args.interface';
|
||||
import { WorkspaceIndexMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface';
|
||||
|
||||
export class MetadataArgsStorage {
|
||||
private readonly entities: WorkspaceEntityMetadataArgs[] = [];
|
||||
@ -13,6 +14,7 @@ export class MetadataArgsStorage {
|
||||
private readonly relations: WorkspaceRelationMetadataArgs[] = [];
|
||||
private readonly dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] =
|
||||
[];
|
||||
private readonly indexes: WorkspaceIndexMetadataArgs[] = [];
|
||||
|
||||
addEntities(...entities: WorkspaceEntityMetadataArgs[]): void {
|
||||
this.entities.push(...entities);
|
||||
@ -32,6 +34,10 @@ export class MetadataArgsStorage {
|
||||
this.relations.push(...relations);
|
||||
}
|
||||
|
||||
addIndexes(...indexes: WorkspaceIndexMetadataArgs[]): void {
|
||||
this.indexes.push(...indexes);
|
||||
}
|
||||
|
||||
addDynamicRelations(
|
||||
...dynamicRelations: WorkspaceDynamicRelationMetadataArgs[]
|
||||
): void {
|
||||
@ -93,6 +99,16 @@ export class MetadataArgsStorage {
|
||||
return this.filterByTarget(this.relations, target);
|
||||
}
|
||||
|
||||
filterIndexes(target: Function | string): WorkspaceIndexMetadataArgs[];
|
||||
|
||||
filterIndexes(target: (Function | string)[]): WorkspaceIndexMetadataArgs[];
|
||||
|
||||
filterIndexes(
|
||||
target: (Function | string) | (Function | string)[],
|
||||
): WorkspaceIndexMetadataArgs[] {
|
||||
return this.filterByTarget(this.indexes, target);
|
||||
}
|
||||
|
||||
filterDynamicRelations(
|
||||
target: Function | string,
|
||||
): WorkspaceDynamicRelationMetadataArgs[];
|
||||
|
@ -1,3 +1,5 @@
|
||||
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
|
||||
|
||||
import { WorkspaceMigrationObjectFactory } from './workspace-migration-object.factory';
|
||||
import { WorkspaceMigrationFieldFactory } from './workspace-migration-field.factory';
|
||||
import { WorkspaceMigrationRelationFactory } from './workspace-migration-relation.factory';
|
||||
@ -6,4 +8,5 @@ export const workspaceMigrationBuilderFactories = [
|
||||
WorkspaceMigrationObjectFactory,
|
||||
WorkspaceMigrationFieldFactory,
|
||||
WorkspaceMigrationRelationFactory,
|
||||
WorkspaceMigrationIndexFactory,
|
||||
];
|
||||
|
@ -0,0 +1,156 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import {
|
||||
WorkspaceMigrationEntity,
|
||||
WorkspaceMigrationIndexActionType,
|
||||
WorkspaceMigrationTableActionType,
|
||||
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
|
||||
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMigrationIndexFactory {
|
||||
constructor() {}
|
||||
|
||||
async create(
|
||||
originalObjectMetadataCollection: ObjectMetadataEntity[],
|
||||
indexMetadataCollection: IndexMetadataEntity[],
|
||||
action: WorkspaceMigrationBuilderAction,
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
const originalObjectMetadataMap = Object.fromEntries(
|
||||
originalObjectMetadataCollection.map((obj) => [obj.id, obj]),
|
||||
);
|
||||
|
||||
const indexMetadataByObjectMetadataMap = new Map<
|
||||
ObjectMetadataEntity,
|
||||
IndexMetadataEntity[]
|
||||
>();
|
||||
|
||||
indexMetadataCollection.forEach((currentIndexMetadata) => {
|
||||
const objectMetadata =
|
||||
originalObjectMetadataMap[currentIndexMetadata.objectMetadataId];
|
||||
|
||||
if (!objectMetadata) {
|
||||
throw new Error(
|
||||
`Object metadata with id ${currentIndexMetadata.objectMetadataId} not found`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!indexMetadataByObjectMetadataMap.has(objectMetadata)) {
|
||||
indexMetadataByObjectMetadataMap.set(objectMetadata, []);
|
||||
}
|
||||
|
||||
indexMetadataByObjectMetadataMap
|
||||
?.get(objectMetadata)
|
||||
?.push(currentIndexMetadata);
|
||||
});
|
||||
|
||||
switch (action) {
|
||||
case WorkspaceMigrationBuilderAction.CREATE:
|
||||
return this.createIndexMigration(indexMetadataByObjectMetadataMap);
|
||||
case WorkspaceMigrationBuilderAction.DELETE:
|
||||
return this.deleteIndexMigration(indexMetadataByObjectMetadataMap);
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async createIndexMigration(
|
||||
indexMetadataByObjectMetadataMap: Map<
|
||||
ObjectMetadataEntity,
|
||||
IndexMetadataEntity[]
|
||||
>,
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
|
||||
|
||||
for (const [
|
||||
objectMetadata,
|
||||
indexMetadataCollection,
|
||||
] of indexMetadataByObjectMetadataMap) {
|
||||
const targetTable = computeObjectTargetTable(objectMetadata);
|
||||
|
||||
const fieldsById = Object.fromEntries(
|
||||
objectMetadata.fields.map((field) => [field.id, field]),
|
||||
);
|
||||
|
||||
const indexes = indexMetadataCollection.map((indexMetadata) => ({
|
||||
name: indexMetadata.name,
|
||||
action: WorkspaceMigrationIndexActionType.CREATE,
|
||||
columns: indexMetadata.indexFieldMetadatas
|
||||
.sort((a, b) => a.order - b.order)
|
||||
.map((indexFieldMetadata) => {
|
||||
const fieldMetadata =
|
||||
fieldsById[indexFieldMetadata.fieldMetadataId];
|
||||
|
||||
if (!fieldMetadata) {
|
||||
throw new Error(
|
||||
`Field metadata with id ${indexFieldMetadata.fieldMetadataId} not found in object metadata with id ${objectMetadata.id}`,
|
||||
);
|
||||
}
|
||||
|
||||
return fieldMetadata.name;
|
||||
}),
|
||||
}));
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: objectMetadata.workspaceId,
|
||||
name: generateMigrationName(
|
||||
`create-${objectMetadata.nameSingular}-indexes`,
|
||||
),
|
||||
isCustom: false,
|
||||
migrations: [
|
||||
{
|
||||
name: targetTable,
|
||||
action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
|
||||
indexes,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
return workspaceMigrations;
|
||||
}
|
||||
|
||||
private async deleteIndexMigration(
|
||||
indexMetadataByObjectMetadataMap: Map<
|
||||
ObjectMetadataEntity,
|
||||
IndexMetadataEntity[]
|
||||
>,
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
|
||||
|
||||
for (const [
|
||||
objectMetadata,
|
||||
indexMetadataCollection,
|
||||
] of indexMetadataByObjectMetadataMap) {
|
||||
const targetTable = computeObjectTargetTable(objectMetadata);
|
||||
|
||||
const indexes = indexMetadataCollection.map((indexMetadata) => ({
|
||||
name: indexMetadata.name,
|
||||
action: WorkspaceMigrationIndexActionType.DROP,
|
||||
columns: [],
|
||||
}));
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: objectMetadata.workspaceId,
|
||||
name: generateMigrationName(
|
||||
`delete-${objectMetadata.nameSingular}-indexes`,
|
||||
),
|
||||
isCustom: false,
|
||||
migrations: [
|
||||
{
|
||||
name: targetTable,
|
||||
action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
|
||||
indexes,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
return workspaceMigrations;
|
||||
}
|
||||
}
|
@ -5,6 +5,7 @@ import {
|
||||
Table,
|
||||
TableColumn,
|
||||
TableForeignKey,
|
||||
TableIndex,
|
||||
TableUnique,
|
||||
} from 'typeorm';
|
||||
|
||||
@ -20,6 +21,8 @@ import {
|
||||
WorkspaceMigrationColumnDropRelation,
|
||||
WorkspaceMigrationTableActionType,
|
||||
WorkspaceMigrationForeignTable,
|
||||
WorkspaceMigrationIndexAction,
|
||||
WorkspaceMigrationIndexActionType,
|
||||
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
|
||||
import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service';
|
||||
@ -137,6 +140,7 @@ export class WorkspaceMigrationRunnerService {
|
||||
tableMigration.columns,
|
||||
);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case WorkspaceMigrationTableActionType.DROP:
|
||||
@ -163,6 +167,17 @@ export class WorkspaceMigrationRunnerService {
|
||||
tableMigration.columns,
|
||||
);
|
||||
break;
|
||||
|
||||
case WorkspaceMigrationTableActionType.ALTER_INDEXES:
|
||||
if (tableMigration.indexes && tableMigration.indexes.length > 0) {
|
||||
await this.handleIndexesChanges(
|
||||
queryRunner,
|
||||
schemaName,
|
||||
tableMigration.newName ?? tableMigration.name,
|
||||
tableMigration.indexes,
|
||||
);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw new Error(
|
||||
`Migration table action ${tableMigration.action} not supported`,
|
||||
@ -170,6 +185,32 @@ export class WorkspaceMigrationRunnerService {
|
||||
}
|
||||
}
|
||||
|
||||
private async handleIndexesChanges(
|
||||
queryRunner: QueryRunner,
|
||||
schemaName: string,
|
||||
tableName: string,
|
||||
indexes: WorkspaceMigrationIndexAction[],
|
||||
) {
|
||||
for (const index of indexes) {
|
||||
switch (index.action) {
|
||||
case WorkspaceMigrationIndexActionType.CREATE:
|
||||
await queryRunner.createIndex(
|
||||
`${schemaName}.${tableName}`,
|
||||
new TableIndex({
|
||||
name: index.name,
|
||||
columnNames: index.columns,
|
||||
}),
|
||||
);
|
||||
break;
|
||||
case WorkspaceMigrationIndexActionType.DROP:
|
||||
await queryRunner.dropIndex(`${schemaName}.${tableName}`, index.name);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Migration index action not supported`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a table for a given schema and table name
|
||||
*
|
||||
|
@ -5,7 +5,6 @@ import { Command, CommandRunner, Option } from 'nest-commander';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service';
|
||||
import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service';
|
||||
import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service';
|
||||
|
||||
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
|
||||
|
||||
@ -28,7 +27,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
|
||||
private readonly workspaceHealthService: WorkspaceHealthService,
|
||||
private readonly dataSourceService: DataSourceService,
|
||||
private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
|
||||
private readonly workspaceService: WorkspaceService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@ -37,9 +35,8 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
|
||||
_passedParam: string[],
|
||||
options: RunWorkspaceMigrationsOptions,
|
||||
): Promise<void> {
|
||||
const workspaceIds = options.workspaceId
|
||||
? [options.workspaceId]
|
||||
: await this.workspaceService.getWorkspaceIds();
|
||||
// TODO: re-implement load index from workspaceService, this is breaking the logger
|
||||
const workspaceIds = options.workspaceId ? [options.workspaceId] : [];
|
||||
|
||||
for (const workspaceId of workspaceIds) {
|
||||
try {
|
||||
@ -105,7 +102,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
|
||||
@Option({
|
||||
flags: '-w, --workspace-id [workspace_id]',
|
||||
description: 'workspace id',
|
||||
required: false,
|
||||
required: true,
|
||||
})
|
||||
parseWorkspaceId(value: string): string {
|
||||
return value;
|
||||
|
@ -1,3 +1,5 @@
|
||||
import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator';
|
||||
|
||||
import { WorkspaceFieldComparator } from './workspace-field.comparator';
|
||||
import { WorkspaceObjectComparator } from './workspace-object.comparator';
|
||||
import { WorkspaceRelationComparator } from './workspace-relation.comparator';
|
||||
@ -6,4 +8,5 @@ export const workspaceSyncMetadataComparators = [
|
||||
WorkspaceFieldComparator,
|
||||
WorkspaceObjectComparator,
|
||||
WorkspaceRelationComparator,
|
||||
WorkspaceIndexComparator,
|
||||
];
|
||||
|
@ -0,0 +1,90 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import diff from 'microdiff';
|
||||
|
||||
import {
|
||||
IndexComparatorResult,
|
||||
ComparatorAction,
|
||||
} from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
import { transformMetadataForComparison } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util';
|
||||
|
||||
const propertiesToIgnore = ['createdAt', 'updatedAt', 'indexFieldMetadatas'];
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceIndexComparator {
|
||||
constructor() {}
|
||||
|
||||
compare(
|
||||
originalIndexMetadataCollection: IndexMetadataEntity[],
|
||||
standardIndexMetadataCollection: Partial<IndexMetadataEntity>[],
|
||||
): IndexComparatorResult[] {
|
||||
const results: IndexComparatorResult[] = [];
|
||||
|
||||
// Create a map of standard relations
|
||||
const standardIndexMetadataMap = transformMetadataForComparison(
|
||||
standardIndexMetadataCollection,
|
||||
{
|
||||
keyFactory(indexMetadata) {
|
||||
return `${indexMetadata.name}`;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const originalIndexMetadataCollectionWithColumns =
|
||||
originalIndexMetadataCollection.map((indexMetadata) => {
|
||||
return {
|
||||
...indexMetadata,
|
||||
columns: indexMetadata.indexFieldMetadatas.map(
|
||||
(indexFieldMetadata) => indexFieldMetadata.fieldMetadata.name,
|
||||
),
|
||||
indexFieldMetadatas: undefined,
|
||||
};
|
||||
});
|
||||
|
||||
// Create a filtered map of original relations
|
||||
// We filter out 'id' later because we need it to remove the relation from DB
|
||||
const originalIndexMetadataMap = transformMetadataForComparison(
|
||||
originalIndexMetadataCollectionWithColumns,
|
||||
{
|
||||
shouldIgnoreProperty: (property) =>
|
||||
propertiesToIgnore.includes(property),
|
||||
keyFactory(indexMetadata) {
|
||||
return `${indexMetadata.name}`;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Compare indexes
|
||||
const indexesDifferences = diff(
|
||||
originalIndexMetadataMap,
|
||||
standardIndexMetadataMap,
|
||||
);
|
||||
|
||||
for (const difference of indexesDifferences) {
|
||||
switch (difference.type) {
|
||||
case 'CREATE': {
|
||||
results.push({
|
||||
action: ComparatorAction.CREATE,
|
||||
object: difference.value,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'REMOVE': {
|
||||
if (difference.path[difference.path.length - 1] !== 'id') {
|
||||
results.push({
|
||||
action: ComparatorAction.DELETE,
|
||||
object: difference.oldValue,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
@ -1,3 +1,5 @@
|
||||
import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory';
|
||||
|
||||
import { FeatureFlagFactory } from './feature-flags.factory';
|
||||
import { StandardFieldFactory } from './standard-field.factory';
|
||||
import { StandardObjectFactory } from './standard-object.factory';
|
||||
@ -8,4 +10,5 @@ export const workspaceSyncMetadataFactories = [
|
||||
StandardFieldFactory,
|
||||
StandardObjectFactory,
|
||||
StandardRelationFactory,
|
||||
StandardIndexFactory,
|
||||
];
|
||||
|
@ -0,0 +1,69 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
|
||||
|
||||
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage';
|
||||
|
||||
@Injectable()
|
||||
export class StandardIndexFactory {
|
||||
create(
|
||||
standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[],
|
||||
context: WorkspaceSyncContext,
|
||||
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Partial<IndexMetadataEntity>[] {
|
||||
return standardObjectMetadataDefinitions.flatMap((standardObjectMetadata) =>
|
||||
this.createIndexMetadata(
|
||||
standardObjectMetadata,
|
||||
context,
|
||||
originalObjectMetadataMap,
|
||||
workspaceFeatureFlagsMap,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private createIndexMetadata(
|
||||
target: typeof BaseWorkspaceEntity,
|
||||
context: WorkspaceSyncContext,
|
||||
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Partial<IndexMetadataEntity>[] {
|
||||
const workspaceEntity = metadataArgsStorage.filterEntities(target);
|
||||
|
||||
if (!workspaceEntity) {
|
||||
throw new Error(
|
||||
`Object metadata decorator not found, can't parse ${target.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
const workspaceIndexMetadataArgsCollection =
|
||||
metadataArgsStorage.filterIndexes(target);
|
||||
|
||||
return workspaceIndexMetadataArgsCollection.map(
|
||||
(workspaceIndexMetadataArgs) => {
|
||||
const objectMetadata =
|
||||
originalObjectMetadataMap[workspaceEntity.nameSingular];
|
||||
|
||||
if (!objectMetadata) {
|
||||
throw new Error(
|
||||
`Object metadata not found for ${workspaceEntity.nameSingular}`,
|
||||
);
|
||||
}
|
||||
|
||||
const indexMetadata: PartialIndexMetadata = {
|
||||
workspaceId: context.workspaceId,
|
||||
objectMetadataId: objectMetadata.id,
|
||||
name: workspaceIndexMetadataArgs.name,
|
||||
columns: workspaceIndexMetadataArgs.columns,
|
||||
};
|
||||
|
||||
return indexMetadata;
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
@ -1,5 +1,6 @@
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
import { ComputedPartialFieldMetadata } from './partial-field-metadata.interface';
|
||||
import { ComputedPartialWorkspaceEntity } from './partial-object-metadata.interface';
|
||||
@ -49,3 +50,8 @@ export type RelationComparatorResult =
|
||||
| ComparatorCreateResult<Partial<RelationMetadataEntity>>
|
||||
| ComparatorDeleteResult<RelationMetadataEntity>
|
||||
| ComparatorUpdateResult<Partial<RelationMetadataEntity>>;
|
||||
|
||||
export type IndexComparatorResult =
|
||||
| ComparatorCreateResult<Partial<IndexMetadataEntity>>
|
||||
| ComparatorUpdateResult<Partial<IndexMetadataEntity>>
|
||||
| ComparatorDeleteResult<IndexMetadataEntity>;
|
||||
|
@ -0,0 +1,8 @@
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
export type PartialIndexMetadata = Omit<
|
||||
IndexMetadataEntity,
|
||||
'id' | 'objectMetadata' | 'indexFieldMetadatas'
|
||||
> & {
|
||||
columns: string[];
|
||||
};
|
@ -11,6 +11,7 @@ import { v4 as uuidV4 } from 'uuid';
|
||||
import { DeepPartial } from 'typeorm/common/DeepPartial';
|
||||
|
||||
import { PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
|
||||
import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import {
|
||||
@ -22,6 +23,7 @@ import { FieldMetadataComplexOption } from 'src/engine/metadata-modules/field-me
|
||||
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
import { FieldMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
|
||||
import { ObjectMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-object.factory';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMetadataUpdaterService {
|
||||
@ -230,6 +232,69 @@ export class WorkspaceMetadataUpdaterService {
|
||||
};
|
||||
}
|
||||
|
||||
async updateIndexMetadata(
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
originalObjectMetadataCollection: ObjectMetadataEntity[],
|
||||
): Promise<{
|
||||
createdIndexMetadataCollection: IndexMetadataEntity[];
|
||||
}> {
|
||||
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
|
||||
|
||||
const convertIndexMetadataForSaving = (
|
||||
indexMetadata: PartialIndexMetadata,
|
||||
) => {
|
||||
const convertIndexFieldMetadataForSaving = (
|
||||
column: string,
|
||||
order: number,
|
||||
) => {
|
||||
const fieldMetadata = originalObjectMetadataCollection
|
||||
.find((object) => object.id === indexMetadata.objectMetadataId)
|
||||
?.fields.find((field) => column === field.name);
|
||||
|
||||
if (!fieldMetadata) {
|
||||
throw new Error(`
|
||||
Field metadata not found for column ${column} in object ${indexMetadata.objectMetadataId}
|
||||
`);
|
||||
}
|
||||
|
||||
return {
|
||||
fieldMetadataId: fieldMetadata.id,
|
||||
order,
|
||||
};
|
||||
};
|
||||
|
||||
return {
|
||||
...indexMetadata,
|
||||
indexFieldMetadatas: indexMetadata.columns.map((column, index) =>
|
||||
convertIndexFieldMetadataForSaving(column, index),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Create index metadata
|
||||
*/
|
||||
const createdIndexMetadataCollection = await indexMetadataRepository.save(
|
||||
storage.indexMetadataCreateCollection.map(convertIndexMetadataForSaving),
|
||||
);
|
||||
|
||||
/**
|
||||
* Delete index metadata
|
||||
*/
|
||||
if (storage.indexMetadataDeleteCollection.length > 0) {
|
||||
await indexMetadataRepository.delete(
|
||||
storage.indexMetadataDeleteCollection.map(
|
||||
(indexMetadata) => indexMetadata.id,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
createdIndexMetadataCollection,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update entities in the database
|
||||
* @param manager EntityManager
|
||||
|
@ -48,7 +48,7 @@ export class WorkspaceSyncFieldMetadataService {
|
||||
relations: ['dataSource', 'fields'],
|
||||
});
|
||||
|
||||
// Filter out custom objects
|
||||
// Filter out non-custom objects
|
||||
const customObjectMetadataCollection =
|
||||
originalObjectMetadataCollection.filter(
|
||||
(objectMetadata) => objectMetadata.isCustom,
|
||||
@ -61,7 +61,7 @@ export class WorkspaceSyncFieldMetadataService {
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
// Loop over all standard objects and compare them with the objects in DB
|
||||
// Loop over all custom objects from the DB and compare their fields with standard fields
|
||||
for (const customObjectMetadata of customObjectMetadataCollection) {
|
||||
// Also, maybe it's better to refactor a bit and move generation part into a separate module ?
|
||||
const standardObjectMetadata = computeStandardObject(
|
||||
|
@ -0,0 +1,119 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
|
||||
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
|
||||
|
||||
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
|
||||
import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator';
|
||||
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
|
||||
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceSyncIndexMetadataService {
|
||||
private readonly logger = new Logger(WorkspaceSyncIndexMetadataService.name);
|
||||
|
||||
constructor(
|
||||
private readonly standardIndexFactory: StandardIndexFactory,
|
||||
private readonly workspaceIndexComparator: WorkspaceIndexComparator,
|
||||
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
|
||||
private readonly workspaceMigrationIndexFactory: WorkspaceMigrationIndexFactory,
|
||||
) {}
|
||||
|
||||
async synchronize(
|
||||
context: WorkspaceSyncContext,
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
this.logger.log('Syncing index metadata');
|
||||
|
||||
const objectMetadataRepository =
|
||||
manager.getRepository(ObjectMetadataEntity);
|
||||
|
||||
// Retrieve object metadata collection from DB
|
||||
const originalObjectMetadataCollection =
|
||||
await objectMetadataRepository.find({
|
||||
where: {
|
||||
workspaceId: context.workspaceId,
|
||||
// We're only interested in standard fields
|
||||
fields: { isCustom: false },
|
||||
isCustom: false,
|
||||
},
|
||||
relations: ['dataSource', 'fields', 'indexes'],
|
||||
});
|
||||
|
||||
// Create map of object metadata & field metadata by unique identifier
|
||||
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
|
||||
originalObjectMetadataCollection,
|
||||
// Relation are based on the singular name
|
||||
(objectMetadata) => objectMetadata.nameSingular,
|
||||
);
|
||||
|
||||
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
|
||||
|
||||
const originalIndexMetadataCollection = await indexMetadataRepository.find({
|
||||
where: {
|
||||
workspaceId: context.workspaceId,
|
||||
},
|
||||
relations: ['indexFieldMetadatas.fieldMetadata'],
|
||||
});
|
||||
|
||||
// Generate index metadata from models
|
||||
const standardIndexMetadataCollection = this.standardIndexFactory.create(
|
||||
standardObjectMetadataDefinitions,
|
||||
context,
|
||||
originalObjectMetadataMap,
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
const indexComparatorResults = this.workspaceIndexComparator.compare(
|
||||
originalIndexMetadataCollection,
|
||||
standardIndexMetadataCollection,
|
||||
);
|
||||
|
||||
for (const indexComparatorResult of indexComparatorResults) {
|
||||
if (indexComparatorResult.action === ComparatorAction.CREATE) {
|
||||
storage.addCreateIndexMetadata(indexComparatorResult.object);
|
||||
} else if (indexComparatorResult.action === ComparatorAction.DELETE) {
|
||||
storage.addDeleteIndexMetadata(indexComparatorResult.object);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataIndexUpdaterResult =
|
||||
await this.workspaceMetadataUpdaterService.updateIndexMetadata(
|
||||
manager,
|
||||
storage,
|
||||
originalObjectMetadataCollection,
|
||||
);
|
||||
|
||||
// Create migrations
|
||||
const createIndexWorkspaceMigrations =
|
||||
await this.workspaceMigrationIndexFactory.create(
|
||||
originalObjectMetadataCollection,
|
||||
metadataIndexUpdaterResult.createdIndexMetadataCollection,
|
||||
WorkspaceMigrationBuilderAction.CREATE,
|
||||
);
|
||||
|
||||
const deleteIndexWorkspaceMigrations =
|
||||
await this.workspaceMigrationIndexFactory.create(
|
||||
originalObjectMetadataCollection,
|
||||
storage.indexMetadataDeleteCollection,
|
||||
WorkspaceMigrationBuilderAction.DELETE,
|
||||
);
|
||||
|
||||
return [
|
||||
...createIndexWorkspaceMigrations,
|
||||
...deleteIndexWorkspaceMigrations,
|
||||
];
|
||||
}
|
||||
}
|
@ -112,6 +112,10 @@ export class WorkspaceSyncObjectMetadataService {
|
||||
|
||||
/**
|
||||
* COMPARE FIELD METADATA
|
||||
* NOTE: This should be moved to WorkspaceSyncFieldMetadataService for more clarity since
|
||||
* this code only adds field metadata to the storage but it's actually used in the other service.
|
||||
* NOTE2: WorkspaceSyncFieldMetadataService has been added for custom fields sync, it should be refactored to handle
|
||||
* both custom and non-custom fields.
|
||||
*/
|
||||
const fieldComparatorResults = this.workspaceFieldComparator.compare(
|
||||
originalObjectMetadata,
|
||||
|
@ -4,6 +4,7 @@ import { ComputedPartialFieldMetadata } from 'src/engine/workspace-manager/works
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
|
||||
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
|
||||
|
||||
export class WorkspaceSyncStorage {
|
||||
// Object metadata
|
||||
@ -25,10 +26,17 @@ export class WorkspaceSyncStorage {
|
||||
// Relation metadata
|
||||
private readonly _relationMetadataCreateCollection: Partial<RelationMetadataEntity>[] =
|
||||
[];
|
||||
private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] =
|
||||
[];
|
||||
private readonly _relationMetadataUpdateCollection: Partial<RelationMetadataEntity>[] =
|
||||
[];
|
||||
private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] =
|
||||
[];
|
||||
|
||||
// Index metadata
|
||||
private readonly _indexMetadataCreateCollection: Partial<IndexMetadataEntity>[] =
|
||||
[];
|
||||
private readonly _indexMetadataUpdateCollection: Partial<IndexMetadataEntity>[] =
|
||||
[];
|
||||
private readonly _indexMetadataDeleteCollection: IndexMetadataEntity[] = [];
|
||||
|
||||
constructor() {}
|
||||
|
||||
@ -68,6 +76,18 @@ export class WorkspaceSyncStorage {
|
||||
return this._relationMetadataDeleteCollection;
|
||||
}
|
||||
|
||||
get indexMetadataCreateCollection() {
|
||||
return this._indexMetadataCreateCollection;
|
||||
}
|
||||
|
||||
get indexMetadataUpdateCollection() {
|
||||
return this._indexMetadataUpdateCollection;
|
||||
}
|
||||
|
||||
get indexMetadataDeleteCollection() {
|
||||
return this._indexMetadataDeleteCollection;
|
||||
}
|
||||
|
||||
addCreateObjectMetadata(object: ComputedPartialWorkspaceEntity) {
|
||||
this._objectMetadataCreateCollection.push(object);
|
||||
}
|
||||
@ -107,4 +127,16 @@ export class WorkspaceSyncStorage {
|
||||
addDeleteRelationMetadata(relation: RelationMetadataEntity) {
|
||||
this._relationMetadataDeleteCollection.push(relation);
|
||||
}
|
||||
|
||||
addCreateIndexMetadata(index: Partial<IndexMetadataEntity>) {
|
||||
this._indexMetadataCreateCollection.push(index);
|
||||
}
|
||||
|
||||
addUpdateIndexMetadata(index: Partial<IndexMetadataEntity>) {
|
||||
this._indexMetadataUpdateCollection.push(index);
|
||||
}
|
||||
|
||||
addDeleteIndexMetadata(index: IndexMetadataEntity) {
|
||||
this._indexMetadataDeleteCollection.push(index);
|
||||
}
|
||||
}
|
||||
|
@ -16,6 +16,7 @@ import { WorkspaceSyncRelationMetadataService } from 'src/engine/workspace-manag
|
||||
import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service';
|
||||
import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module';
|
||||
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
|
||||
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@ -41,6 +42,7 @@ import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspa
|
||||
WorkspaceSyncRelationMetadataService,
|
||||
WorkspaceSyncFieldMetadataService,
|
||||
WorkspaceSyncMetadataService,
|
||||
WorkspaceSyncIndexMetadataService,
|
||||
],
|
||||
exports: [...workspaceSyncMetadataFactories, WorkspaceSyncMetadataService],
|
||||
})
|
||||
|
@ -13,6 +13,7 @@ import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/
|
||||
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
|
||||
import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service';
|
||||
|
||||
interface SynchronizeOptions {
|
||||
applyChanges?: boolean;
|
||||
@ -31,6 +32,7 @@ export class WorkspaceSyncMetadataService {
|
||||
private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService,
|
||||
private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService,
|
||||
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
|
||||
private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
@ -97,11 +99,21 @@ export class WorkspaceSyncMetadataService {
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
// 4 - Sync standard indexes on standard objects
|
||||
const workspaceIndexMigrations =
|
||||
await this.workspaceSyncIndexMetadataService.synchronize(
|
||||
context,
|
||||
manager,
|
||||
storage,
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
// Save workspace migrations into the database
|
||||
workspaceMigrations = await workspaceMigrationRepository.save([
|
||||
...workspaceObjectMigrations,
|
||||
...workspaceFieldMigrations,
|
||||
...workspaceRelationMigrations,
|
||||
...workspaceIndexMigrations,
|
||||
]);
|
||||
|
||||
// If we're running a dry run, rollback the transaction and do not execute migrations
|
||||
|
Loading…
Reference in New Issue
Block a user