diff --git a/server/package.json b/server/package.json index 8d72437050..88d60b8ad4 100644 --- a/server/package.json +++ b/server/package.json @@ -26,7 +26,8 @@ "prisma:migrate": "npx prisma migrate deploy && yarn typeorm migration:run -- -d ./src/tenant/metadata/metadata.datasource.ts", "prisma:seed": "npx prisma db seed", "prisma:reset": "npx prisma migrate reset && yarn prisma:generate", - "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js" + "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js", + "typeorm:migrate": "yarn typeorm migration:run -- -d ./src/tenant/metadata/metadata.datasource.ts" }, "dependencies": { "@apollo/server": "^4.7.3", diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts index 49c9ad7b4f..d8ce822b14 100644 --- a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts @@ -23,8 +23,8 @@ export class DataSourceMetadata { @Column({ type: 'enum', enum: ['postgres'], default: 'postgres' }) type: DataSourceType; - @Column({ nullable: true }) - name: string; + @Column({ nullable: true, name: 'display_name' }) + displayName: string; @Column({ default: false, name: 'is_remote' }) isRemote: boolean; diff --git a/server/src/tenant/metadata/data-source/data-source.service.ts b/server/src/tenant/metadata/data-source/data-source.service.ts index f89f77ea36..78efaebcf0 100644 --- a/server/src/tenant/metadata/data-source/data-source.service.ts +++ b/server/src/tenant/metadata/data-source/data-source.service.ts @@ -5,17 +5,19 @@ import { OnModuleInit, } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, QueryRunner, Table } from 'typeorm'; import { EnvironmentService } from 'src/integrations/environment/environment.service'; import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service'; import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service'; +import { TenantMigration } from 'src/tenant/metadata/migration-generator/tenant-migration.entity'; import { uuidToBase36 } from './data-source.util'; @Injectable() export class DataSourceService implements OnModuleInit, OnModuleDestroy { private mainDataSource: DataSource; + private dataSources: Map = new Map(); constructor( private readonly environmentService: EnvironmentService, @@ -39,7 +41,15 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { const schemaName = this.getSchemaName(workspaceId); const queryRunner = this.mainDataSource.createQueryRunner(); + const schemaAlreadyExists = await queryRunner.hasSchema(schemaName); + if (schemaAlreadyExists) { + throw new Error( + `Schema ${schemaName} already exists for workspace ${workspaceId}`, + ); + } + await queryRunner.createSchema(schemaName, true); + await this.createMigrationTable(queryRunner, schemaName); await queryRunner.release(); await this.dataSourceMetadataService.createDataSourceMetadata( @@ -48,6 +58,40 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { ); } + private async createMigrationTable( + queryRunner: QueryRunner, + schemaName: string, + ) { + await queryRunner.createTable( + new Table({ + name: 'tenant_migrations', + schema: schemaName, + columns: [ + { + name: 'id', + type: 'uuid', + isPrimary: true, + default: 'uuid_generate_v4()', + }, + { + name: 'migrations', + type: 'jsonb', + }, + { + name: 'applied_at', + type: 'timestamp', + isNullable: true, + }, + { + name: 'created_at', + type: 'timestamp', + default: 'now()', + }, + ], + }), + ); + } + /** * Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists. * @param workspaceId @@ -56,10 +100,10 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { public async connectToWorkspaceDataSource( workspaceId: string, ): Promise { - // if (this.dataSources.has(workspaceId)) { - // const cachedDataSource = this.dataSources.get(workspaceId); - // return cachedDataSource; - // } + if (this.dataSources.has(workspaceId)) { + const cachedDataSource = this.dataSources.get(workspaceId); + return cachedDataSource; + } const dataSourcesMetadata = await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId( @@ -93,16 +137,35 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { type: 'postgres', logging: ['query'], schema, - entities: entities, - synchronize: true, // TODO: remove this in production + entities: { + TenantMigration, + ...entities, + }, }); await workspaceDataSource.initialize(); - return workspaceDataSource; - // this.dataSources.set(workspaceId, workspaceDataSource); + this.dataSources.set(workspaceId, workspaceDataSource); - // return this.dataSources.get(workspaceId); + return this.dataSources.get(workspaceId); + } + + /** + * Disconnects from a workspace data source. + * @param workspaceId + * @returns Promise + * + */ + public async disconnectFromWorkspaceDataSource(workspaceId: string) { + if (!this.dataSources.has(workspaceId)) { + return; + } + + const dataSource = this.dataSources.get(workspaceId); + + await dataSource?.destroy(); + + this.dataSources.delete(workspaceId); } /** @@ -111,7 +174,7 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { * @param workspaceId * @returns string */ - private getSchemaName(workspaceId: string): string { + public getSchemaName(workspaceId: string): string { return `workspace_${uuidToBase36(workspaceId)}`; } diff --git a/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts index f7e629b290..a0a3ccfce0 100644 --- a/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts +++ b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts @@ -22,13 +22,13 @@ export class EntitySchemaGeneratorService { const entities = objectMetadata.map((object) => { return new EntitySchema({ - name: object.name, + name: object.targetTableName, columns: { ...baseColumns, ...object.fields.reduce((columns, field) => { return { ...columns, - [sanitizeColumnName(field.name)]: { + [sanitizeColumnName(field.targetColumnName)]: { type: convertFieldTypeToPostgresType(field.type), nullable: true, }, diff --git a/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts b/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts index 3dacaa8965..e025e61438 100644 --- a/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts +++ b/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts @@ -21,8 +21,11 @@ export class FieldMetadata { @Column({ nullable: false }) type: string; - @Column({ nullable: false }) - name: string; + @Column({ nullable: false, name: 'display_name' }) + displayName: string; + + @Column({ nullable: false, name: 'target_column_name' }) + targetColumnName: string; @Column({ default: false, name: 'is_custom' }) isCustom: boolean; diff --git a/server/src/tenant/metadata/metadata.controller.spec.ts b/server/src/tenant/metadata/metadata.controller.spec.ts index 5b0503d764..68e9e0917e 100644 --- a/server/src/tenant/metadata/metadata.controller.spec.ts +++ b/server/src/tenant/metadata/metadata.controller.spec.ts @@ -5,6 +5,7 @@ import { MetadataController } from './metadata.controller'; import { DataSourceService } from './data-source/data-source.service'; import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service'; import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service'; +import { MigrationGeneratorService } from './migration-generator/migration-generator.service'; describe('MetadataController', () => { let controller: MetadataController; @@ -25,6 +26,10 @@ describe('MetadataController', () => { provide: EntitySchemaGeneratorService, useValue: {}, }, + { + provide: MigrationGeneratorService, + useValue: {}, + }, ], }).compile(); diff --git a/server/src/tenant/metadata/metadata.controller.ts b/server/src/tenant/metadata/metadata.controller.ts index 48f6234e8e..57c3323a64 100644 --- a/server/src/tenant/metadata/metadata.controller.ts +++ b/server/src/tenant/metadata/metadata.controller.ts @@ -9,6 +9,7 @@ import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service'; import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service'; import { DataSourceService } from './data-source/data-source.service'; +import { MigrationGeneratorService } from './migration-generator/migration-generator.service'; import { uuidToBase36 } from './data-source/data-source.util'; @UseGuards(JwtAuthGuard) @@ -18,6 +19,7 @@ export class MetadataController { private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService, private readonly dataSourceMetadataService: DataSourceMetadataService, private readonly dataSourceService: DataSourceService, + private readonly migrationGenerator: MigrationGeneratorService, ) {} @Get() @@ -40,6 +42,10 @@ export class MetadataController { entities.push(...dataSourceEntities); } + return await this.migrationGenerator.executeMigrationFromPendingMigrations( + workspace.id, + ); + this.dataSourceService.createWorkspaceSchema(workspace.id); console.log('entities', uuidToBase36(workspace.id), workspace.id); diff --git a/server/src/tenant/metadata/metadata.module.ts b/server/src/tenant/metadata/metadata.module.ts index 5cb1f9a125..ebc0e93650 100644 --- a/server/src/tenant/metadata/metadata.module.ts +++ b/server/src/tenant/metadata/metadata.module.ts @@ -10,6 +10,7 @@ import { DataSourceMetadataModule } from './data-source-metadata/data-source-met import { FieldMetadataModule } from './field-metadata/field-metadata.module'; import { ObjectMetadataModule } from './object-metadata/object-metadata.module'; import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module'; +import { MigrationGeneratorModule } from './migration-generator/migration-generator.module'; const typeORMFactory = async (): Promise => ({ ...typeORMMetadataModuleOptions, @@ -26,6 +27,7 @@ const typeORMFactory = async (): Promise => ({ FieldMetadataModule, ObjectMetadataModule, EntitySchemaGeneratorModule, + MigrationGeneratorModule, ], providers: [MetadataService], exports: [MetadataService], diff --git a/server/src/tenant/metadata/migration-generator/migration-generator.module.ts b/server/src/tenant/metadata/migration-generator/migration-generator.module.ts new file mode 100644 index 0000000000..4b68e8d206 --- /dev/null +++ b/server/src/tenant/metadata/migration-generator/migration-generator.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; + +import { DataSourceModule } from 'src/tenant/metadata/data-source/data-source.module'; + +import { MigrationGeneratorService } from './migration-generator.service'; + +@Module({ + imports: [DataSourceModule], + exports: [MigrationGeneratorService], + providers: [MigrationGeneratorService], +}) +export class MigrationGeneratorModule {} diff --git a/server/src/tenant/metadata/migration-generator/migration-generator.service.spec.ts b/server/src/tenant/metadata/migration-generator/migration-generator.service.spec.ts new file mode 100644 index 0000000000..27c83d2dfb --- /dev/null +++ b/server/src/tenant/metadata/migration-generator/migration-generator.service.spec.ts @@ -0,0 +1,27 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service'; + +import { MigrationGeneratorService } from './migration-generator.service'; + +describe('MigrationGeneratorService', () => { + let service: MigrationGeneratorService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + MigrationGeneratorService, + { + provide: DataSourceService, + useValue: {}, + }, + ], + }).compile(); + + service = module.get(MigrationGeneratorService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/migration-generator/migration-generator.service.ts b/server/src/tenant/metadata/migration-generator/migration-generator.service.ts new file mode 100644 index 0000000000..ef5967537b --- /dev/null +++ b/server/src/tenant/metadata/migration-generator/migration-generator.service.ts @@ -0,0 +1,187 @@ +import { Injectable } from '@nestjs/common'; + +import { IsNull, QueryRunner, Table, TableColumn } from 'typeorm'; + +import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service'; + +import { + Migration, + MigrationColumn, + TenantMigration, +} from './tenant-migration.entity'; + +@Injectable() +export class MigrationGeneratorService { + constructor(private readonly dataSourceService: DataSourceService) {} + + private async getPendingMigrations(workspaceId: string) { + const workspaceDataSource = + await this.dataSourceService.connectToWorkspaceDataSource(workspaceId); + + if (!workspaceDataSource) { + throw new Error('Workspace data source not found'); + } + + const tenantMigrationRepository = + workspaceDataSource.getRepository(TenantMigration); + + return tenantMigrationRepository.find({ + order: { createdAt: 'ASC' }, + where: { appliedAt: IsNull() }, + }); + } + + private async setAppliedAtForMigration( + workspaceId: string, + migration: TenantMigration, + ) { + const workspaceDataSource = + await this.dataSourceService.connectToWorkspaceDataSource(workspaceId); + + if (!workspaceDataSource) { + throw new Error('Workspace data source not found'); + } + + const tenantMigrationRepository = + workspaceDataSource.getRepository(TenantMigration); + + await tenantMigrationRepository.save({ + id: migration.id, + appliedAt: new Date(), + }); + } + + public async executeMigrationFromPendingMigrations(workspaceId: string) { + const workspaceDataSource = + await this.dataSourceService.connectToWorkspaceDataSource(workspaceId); + + if (!workspaceDataSource) { + throw new Error('Workspace data source not found'); + } + + const pendingMigrations = await this.getPendingMigrations(workspaceId); + + const flattenedPendingMigrations: Migration[] = pendingMigrations.reduce( + (acc, pendingMigration) => { + return [...acc, ...pendingMigration.migrations]; + }, + [], + ); + + const queryRunner = workspaceDataSource?.createQueryRunner(); + const schemaName = this.dataSourceService.getSchemaName(workspaceId); + + // Loop over each migration and create or update the table + // TODO: Should be done in a transaction + flattenedPendingMigrations.forEach(async (migration) => { + await this.handleTableChanges(queryRunner, schemaName, migration); + }); + + // Update appliedAt date for each migration + // TODO: Should be done after the migration is successful + pendingMigrations.forEach(async (pendingMigration) => { + await this.setAppliedAtForMigration(workspaceId, pendingMigration); + }); + + return flattenedPendingMigrations; + } + + private async handleTableChanges( + queryRunner: QueryRunner, + schemaName: string, + tableMigration: Migration, + ) { + switch (tableMigration.change) { + case 'create': + await this.createTable(queryRunner, schemaName, tableMigration.name); + break; + case 'alter': + await this.handleColumnChanges( + queryRunner, + schemaName, + tableMigration.name, + tableMigration?.columns, + ); + break; + default: + throw new Error( + `Migration table change ${tableMigration.change} not supported`, + ); + } + } + + private async createTable( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + ) { + await queryRunner.createTable( + new Table({ + name: tableName, + schema: schemaName, + columns: [ + { + name: 'id', + type: 'uuid', + isPrimary: true, + default: 'uuid_generate_v4()', + }, + { + name: 'created_at', + type: 'timestamp', + default: 'now()', + }, + ], + }), + true, + ); + } + + private async handleColumnChanges( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + columnMigrations?: MigrationColumn[], + ) { + if (!columnMigrations || columnMigrations.length === 0) { + return; + } + + for (const columnMigration of columnMigrations) { + switch (columnMigration.change) { + case 'create': + await this.createColumn( + queryRunner, + schemaName, + tableName, + columnMigration, + ); + break; + case 'alter': + throw new Error( + `Migration column change ${columnMigration.change} not supported`, + ); + default: + throw new Error( + `Migration column change ${columnMigration.change} not supported`, + ); + } + } + } + + private async createColumn( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + migrationColumn: MigrationColumn, + ) { + await queryRunner.addColumn( + `${schemaName}.${tableName}`, + new TableColumn({ + name: migrationColumn.name, + type: migrationColumn.type, + isNullable: true, + }), + ); + } +} diff --git a/server/src/tenant/metadata/migration-generator/tenant-migration.entity.ts b/server/src/tenant/metadata/migration-generator/tenant-migration.entity.ts new file mode 100644 index 0000000000..e7836a6b71 --- /dev/null +++ b/server/src/tenant/metadata/migration-generator/tenant-migration.entity.ts @@ -0,0 +1,33 @@ +import { + Column, + CreateDateColumn, + Entity, + PrimaryGeneratedColumn, +} from 'typeorm'; + +export type MigrationColumn = { + name: string; + type: string; + change: 'create' | 'alter'; +}; + +export type Migration = { + name: string; + change: 'create' | 'alter'; + columns?: MigrationColumn[]; +}; + +@Entity('tenant_migrations') +export class TenantMigration { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: true, type: 'jsonb' }) + migrations: Migration[]; + + @Column({ nullable: true, name: 'applied_at' }) + appliedAt: Date; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; +} diff --git a/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts b/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts new file mode 100644 index 0000000000..1bf7c08b4f --- /dev/null +++ b/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class InitMetadataTables1695214465080 implements MigrationInterface { + name = 'InitMetadataTables1695214465080'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "metadata"."data_source_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "url" character varying, "schema" character varying, "type" "metadata"."data_source_metadata_type_enum" NOT NULL DEFAULT 'postgres', "display_name" character varying, "is_remote" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_923752b7e62a300a4969bd0e038" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TABLE "metadata"."field_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "object_id" uuid NOT NULL, "type" character varying NOT NULL, "display_name" character varying NOT NULL, "target_column_name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c75db587904cad6af109b5c65f1" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TABLE "metadata"."object_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "data_source_id" character varying NOT NULL, "display_name" character varying NOT NULL, "target_table_name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c8c5f885767b356949c18c201c1" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."field_metadata" ADD CONSTRAINT "FK_38179b299795e48887fc99f937a" FOREIGN KEY ("object_id") REFERENCES "metadata"."object_metadata"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."field_metadata" DROP CONSTRAINT "FK_38179b299795e48887fc99f937a"`, + ); + await queryRunner.query(`DROP TABLE "metadata"."object_metadata"`); + await queryRunner.query(`DROP TABLE "metadata"."field_metadata"`); + await queryRunner.query(`DROP TABLE "metadata"."data_source_metadata"`); + } +} diff --git a/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts b/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts index 22351630fa..73732ae318 100644 --- a/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts +++ b/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts @@ -17,8 +17,11 @@ export class ObjectMetadata { @Column({ nullable: false, name: 'data_source_id' }) dataSourceId: string; - @Column({ nullable: false }) - name: string; + @Column({ nullable: false, name: 'display_name' }) + displayName: string; + + @Column({ nullable: false, name: 'target_table_name' }) + targetTableName: string; @Column({ default: false, name: 'is_custom' }) isCustom: boolean;