twentyhq/twenty
1
1
Fork 0
mirror of https://github.com/twentyhq/twenty.git synced 2024-11-30 23:23:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add metadata migration setup (#1674)

Browse Source 
* Add metadata migration setup

* add migration generator

* fix missing 'mocks'

---------

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>
This commit is contained in:
Weiko 2023-09-21 02:27:07 +02:00 committed by GitHub
parent fc820f47b2
commit 19365f6639
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 391 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/package.json
View File

@ -26,7 +26,8 @@
"prisma:migrate": "npx prisma migrate deploy && yarn typeorm migration:run -- -d ./src/tenant/metadata/metadata.datasource.ts",
"prisma:seed": "npx prisma db seed",
"prisma:reset": "npx prisma migrate reset && yarn prisma:generate",
"typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js"
"typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js",
"typeorm:migrate": "yarn typeorm migration:run -- -d ./src/tenant/metadata/metadata.datasource.ts"
},
"dependencies": {
"@apollo/server": "^4.7.3",

4
server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts
View File

@ -23,8 +23,8 @@ export class DataSourceMetadata {
@Column({ type: 'enum', enum: ['postgres'], default: 'postgres' })
type: DataSourceType;
@Column({ nullable: true })
name: string;
@Column({ nullable: true, name: 'display_name' })
displayName: string;
@Column({ default: false, name: 'is_remote' })
isRemote: boolean;

85
server/src/tenant/metadata/data-source/data-source.service.ts
View File

@ -5,17 +5,19 @@ import {
OnModuleInit,
} from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, QueryRunner, Table } from 'typeorm';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service';
import { TenantMigration } from 'src/tenant/metadata/migration-generator/tenant-migration.entity';
import { uuidToBase36 } from './data-source.util';
@Injectable()
export class DataSourceService implements OnModuleInit, OnModuleDestroy {
private mainDataSource: DataSource;
private dataSources: Map<string, DataSource> = new Map();
constructor(
private readonly environmentService: EnvironmentService,
@ -39,7 +41,15 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
const schemaName = this.getSchemaName(workspaceId);
const queryRunner = this.mainDataSource.createQueryRunner();
const schemaAlreadyExists = await queryRunner.hasSchema(schemaName);
if (schemaAlreadyExists) {
throw new Error(
`Schema ${schemaName} already exists for workspace ${workspaceId}`,
);
}
await queryRunner.createSchema(schemaName, true);
await this.createMigrationTable(queryRunner, schemaName);
await queryRunner.release();
await this.dataSourceMetadataService.createDataSourceMetadata(
@ -48,6 +58,40 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
);
}
private async createMigrationTable(
queryRunner: QueryRunner,
schemaName: string,
) {
await queryRunner.createTable(
new Table({
name: 'tenant_migrations',
schema: schemaName,
columns: [
{
name: 'id',
type: 'uuid',
isPrimary: true,
default: 'uuid_generate_v4()',
},
{
name: 'migrations',
type: 'jsonb',
},
{
name: 'applied_at',
type: 'timestamp',
isNullable: true,
},
{
name: 'created_at',
type: 'timestamp',
default: 'now()',
},
],
}),
);
}
/**
* Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists.
* @param workspaceId
@ -56,10 +100,10 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
public async connectToWorkspaceDataSource(
workspaceId: string,
): Promise<DataSource | undefined> {
// if (this.dataSources.has(workspaceId)) {
// const cachedDataSource = this.dataSources.get(workspaceId);
// return cachedDataSource;
// }
if (this.dataSources.has(workspaceId)) {
const cachedDataSource = this.dataSources.get(workspaceId);
return cachedDataSource;
}
const dataSourcesMetadata =
await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId(
@ -93,16 +137,35 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
type: 'postgres',
logging: ['query'],
schema,
entities: entities,
synchronize: true, // TODO: remove this in production
entities: {
TenantMigration,
...entities,
},
});
await workspaceDataSource.initialize();
return workspaceDataSource;
// this.dataSources.set(workspaceId, workspaceDataSource);
this.dataSources.set(workspaceId, workspaceDataSource);
// return this.dataSources.get(workspaceId);
return this.dataSources.get(workspaceId);
}
/**
* Disconnects from a workspace data source.
* @param workspaceId
* @returns Promise<void>
*
*/
public async disconnectFromWorkspaceDataSource(workspaceId: string) {
if (!this.dataSources.has(workspaceId)) {
return;
}
const dataSource = this.dataSources.get(workspaceId);
await dataSource?.destroy();
this.dataSources.delete(workspaceId);
}
/**
@ -111,7 +174,7 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
* @param workspaceId
* @returns string
*/
private getSchemaName(workspaceId: string): string {
public getSchemaName(workspaceId: string): string {
return `workspace_${uuidToBase36(workspaceId)}`;
}

4
server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts
View File

@ -22,13 +22,13 @@ export class EntitySchemaGeneratorService {
const entities = objectMetadata.map((object) => {
return new EntitySchema({
name: object.name,
name: object.targetTableName,
columns: {
...baseColumns,
...object.fields.reduce((columns, field) => {
return {
...columns,
[sanitizeColumnName(field.name)]: {
[sanitizeColumnName(field.targetColumnName)]: {
type: convertFieldTypeToPostgresType(field.type),
nullable: true,
},

7
server/src/tenant/metadata/field-metadata/field-metadata.entity.ts
View File

@ -21,8 +21,11 @@ export class FieldMetadata {
@Column({ nullable: false })
type: string;
@Column({ nullable: false })
name: string;
@Column({ nullable: false, name: 'display_name' })
displayName: string;
@Column({ nullable: false, name: 'target_column_name' })
targetColumnName: string;
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;

5
server/src/tenant/metadata/metadata.controller.spec.ts
View File

@ -5,6 +5,7 @@ import { MetadataController } from './metadata.controller';
import { DataSourceService } from './data-source/data-source.service';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
describe('MetadataController', () => {
let controller: MetadataController;
@ -25,6 +26,10 @@ describe('MetadataController', () => {
provide: EntitySchemaGeneratorService,
useValue: {},
},
{
provide: MigrationGeneratorService,
useValue: {},
},
],
}).compile();

6
server/src/tenant/metadata/metadata.controller.ts
View File

@ -9,6 +9,7 @@ import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
import { DataSourceService } from './data-source/data-source.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
import { uuidToBase36 } from './data-source/data-source.util';
@UseGuards(JwtAuthGuard)
@ -18,6 +19,7 @@ export class MetadataController {
private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
private readonly dataSourceService: DataSourceService,
private readonly migrationGenerator: MigrationGeneratorService,
) {}
@Get()
@ -40,6 +42,10 @@ export class MetadataController {
entities.push(...dataSourceEntities);
}
return await this.migrationGenerator.executeMigrationFromPendingMigrations(
workspace.id,
);
this.dataSourceService.createWorkspaceSchema(workspace.id);
console.log('entities', uuidToBase36(workspace.id), workspace.id);

2
server/src/tenant/metadata/metadata.module.ts
View File

@ -10,6 +10,7 @@ import { DataSourceMetadataModule } from './data-source-metadata/data-source-met
import { FieldMetadataModule } from './field-metadata/field-metadata.module';
import { ObjectMetadataModule } from './object-metadata/object-metadata.module';
import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module';
import { MigrationGeneratorModule } from './migration-generator/migration-generator.module';
const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
...typeORMMetadataModuleOptions,
@ -26,6 +27,7 @@ const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
FieldMetadataModule,
ObjectMetadataModule,
EntitySchemaGeneratorModule,
MigrationGeneratorModule,
],
providers: [MetadataService],
exports: [MetadataService],

12
server/src/tenant/metadata/migration-generator/migration-generator.module.ts Normal file
View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { DataSourceModule } from 'src/tenant/metadata/data-source/data-source.module';
import { MigrationGeneratorService } from './migration-generator.service';
@Module({
imports: [DataSourceModule],
exports: [MigrationGeneratorService],
providers: [MigrationGeneratorService],
})
export class MigrationGeneratorModule {}

27
server/src/tenant/metadata/migration-generator/migration-generator.service.spec.ts Normal file
View File

@ -0,0 +1,27 @@
import { Test, TestingModule } from '@nestjs/testing';
import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
import { MigrationGeneratorService } from './migration-generator.service';
describe('MigrationGeneratorService', () => {
let service: MigrationGeneratorService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
MigrationGeneratorService,
{
provide: DataSourceService,
useValue: {},
},
],
}).compile();
service = module.get<MigrationGeneratorService>(MigrationGeneratorService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

187
server/src/tenant/metadata/migration-generator/migration-generator.service.ts Normal file
View File

@ -0,0 +1,187 @@
import { Injectable } from '@nestjs/common';
import { IsNull, QueryRunner, Table, TableColumn } from 'typeorm';
import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
import {
Migration,
MigrationColumn,
TenantMigration,
} from './tenant-migration.entity';
@Injectable()
export class MigrationGeneratorService {
constructor(private readonly dataSourceService: DataSourceService) {}
private async getPendingMigrations(workspaceId: string) {
const workspaceDataSource =
await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const tenantMigrationRepository =
workspaceDataSource.getRepository(TenantMigration);
return tenantMigrationRepository.find({
order: { createdAt: 'ASC' },
where: { appliedAt: IsNull() },
});
}
private async setAppliedAtForMigration(
workspaceId: string,
migration: TenantMigration,
) {
const workspaceDataSource =
await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const tenantMigrationRepository =
workspaceDataSource.getRepository(TenantMigration);
await tenantMigrationRepository.save({
id: migration.id,
appliedAt: new Date(),
});
}
public async executeMigrationFromPendingMigrations(workspaceId: string) {
const workspaceDataSource =
await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const pendingMigrations = await this.getPendingMigrations(workspaceId);
const flattenedPendingMigrations: Migration[] = pendingMigrations.reduce(
(acc, pendingMigration) => {
return [...acc, ...pendingMigration.migrations];
},
[],
);
const queryRunner = workspaceDataSource?.createQueryRunner();
const schemaName = this.dataSourceService.getSchemaName(workspaceId);
// Loop over each migration and create or update the table
// TODO: Should be done in a transaction
flattenedPendingMigrations.forEach(async (migration) => {
await this.handleTableChanges(queryRunner, schemaName, migration);
});
// Update appliedAt date for each migration
// TODO: Should be done after the migration is successful
pendingMigrations.forEach(async (pendingMigration) => {
await this.setAppliedAtForMigration(workspaceId, pendingMigration);
});
return flattenedPendingMigrations;
}
private async handleTableChanges(
queryRunner: QueryRunner,
schemaName: string,
tableMigration: Migration,
) {
switch (tableMigration.change) {
case 'create':
await this.createTable(queryRunner, schemaName, tableMigration.name);
break;
case 'alter':
await this.handleColumnChanges(
queryRunner,
schemaName,
tableMigration.name,
tableMigration?.columns,
);
break;
default:
throw new Error(
`Migration table change ${tableMigration.change} not supported`,
);
}
}
private async createTable(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
) {
await queryRunner.createTable(
new Table({
name: tableName,
schema: schemaName,
columns: [
{
name: 'id',
type: 'uuid',
isPrimary: true,
default: 'uuid_generate_v4()',
},
{
name: 'created_at',
type: 'timestamp',
default: 'now()',
},
],
}),
true,
);
}
private async handleColumnChanges(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
columnMigrations?: MigrationColumn[],
) {
if (!columnMigrations || columnMigrations.length === 0) {
return;
}
for (const columnMigration of columnMigrations) {
switch (columnMigration.change) {
case 'create':
await this.createColumn(
queryRunner,
schemaName,
tableName,
columnMigration,
);
break;
case 'alter':
throw new Error(
`Migration column change ${columnMigration.change} not supported`,
);
default:
throw new Error(
`Migration column change ${columnMigration.change} not supported`,
);
}
}
}
private async createColumn(
queryRunner: QueryRunner,
schemaName: string,
tableName: string,
migrationColumn: MigrationColumn,
) {
await queryRunner.addColumn(
`${schemaName}.${tableName}`,
new TableColumn({
name: migrationColumn.name,
type: migrationColumn.type,
isNullable: true,
}),
);
}
}

33
server/src/tenant/metadata/migration-generator/tenant-migration.entity.ts Normal file
View File

@ -0,0 +1,33 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
} from 'typeorm';
export type MigrationColumn = {
name: string;
type: string;
change: 'create' | 'alter';
};
export type Migration = {
name: string;
change: 'create' | 'alter';
columns?: MigrationColumn[];
};
@Entity('tenant_migrations')
export class TenantMigration {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: true, type: 'jsonb' })
migrations: Migration[];
@Column({ nullable: true, name: 'applied_at' })
appliedAt: Date;
@CreateDateColumn({ name: 'created_at' })
createdAt: Date;
}

29
server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts Normal file
View File

@ -0,0 +1,29 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class InitMetadataTables1695214465080 implements MigrationInterface {
name = 'InitMetadataTables1695214465080';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "metadata"."data_source_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "url" character varying, "schema" character varying, "type" "metadata"."data_source_metadata_type_enum" NOT NULL DEFAULT 'postgres', "display_name" character varying, "is_remote" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_923752b7e62a300a4969bd0e038" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE TABLE "metadata"."field_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "object_id" uuid NOT NULL, "type" character varying NOT NULL, "display_name" character varying NOT NULL, "target_column_name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c75db587904cad6af109b5c65f1" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE TABLE "metadata"."object_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "data_source_id" character varying NOT NULL, "display_name" character varying NOT NULL, "target_table_name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c8c5f885767b356949c18c201c1" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD CONSTRAINT "FK_38179b299795e48887fc99f937a" FOREIGN KEY ("object_id") REFERENCES "metadata"."object_metadata"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP CONSTRAINT "FK_38179b299795e48887fc99f937a"`,
);
await queryRunner.query(`DROP TABLE "metadata"."object_metadata"`);
await queryRunner.query(`DROP TABLE "metadata"."field_metadata"`);
await queryRunner.query(`DROP TABLE "metadata"."data_source_metadata"`);
}
}

7
server/src/tenant/metadata/object-metadata/object-metadata.entity.ts
View File

@ -17,8 +17,11 @@ export class ObjectMetadata {
@Column({ nullable: false, name: 'data_source_id' })
dataSourceId: string;
@Column({ nullable: false })
name: string;
@Column({ nullable: false, name: 'display_name' })
displayName: string;
@Column({ nullable: false, name: 'target_table_name' })
targetTableName: string;
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;