From 2d758c990bdcc0a060efcc6b0100c03abeba2ee5 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 20 Sep 2023 19:11:21 -0700 Subject: [PATCH] Fix merge issue flexible backend (#1685) * Fix merge issue flexible backend * Fix tests * Try fix tests * Try fix tests --- .github/workflows/ci-server.yaml | 6 +- docs/docs/developer/local-setup.mdx | 11 ++++ infra/dev/postgres/init.sql | 10 ++++ server/scripts/run-integration.sh | 2 + .../data-source-metadata.service.ts | 2 +- .../data-source/data-source.service.ts | 2 +- .../tenant/metadata/metadata.controller.ts | 9 +-- .../tenant/metadata/metadata.datasource.ts | 2 +- server/src/tenant/metadata/metadata.module.ts | 1 + .../migrations/1695198840363-migrations.ts | 29 ---------- .../1695214465080-InitMetadataTables.ts | 3 + .../universal/universal.resolver.spec.ts | 6 ++ .../tenant/universal/universal.resolver.ts | 5 +- server/test/utils/setup-db.ts | 56 +++++++++++++++++++ 14 files changed, 98 insertions(+), 46 deletions(-) delete mode 100644 server/src/tenant/metadata/migrations/1695198840363-migrations.ts create mode 100644 server/test/utils/setup-db.ts diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index 2d1758ada1..5056b07b82 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -34,8 +34,6 @@ jobs: - name: Server / Run linter run: cd server && yarn lint - name: Server / Run jest tests - run: | - cd server && yarn test + run: cd server && yarn test - name: Server / Run e2e tests - run: | - cd server && yarn test:e2e + run: cd server && yarn test:e2e diff --git a/docs/docs/developer/local-setup.mdx b/docs/docs/developer/local-setup.mdx index 0db0e7f437..a9479f129e 100644 --- a/docs/docs/developer/local-setup.mdx +++ b/docs/docs/developer/local-setup.mdx @@ -85,6 +85,17 @@ Create a `twenty` user with password `twenty`: ```sql CREATE USER twenty PASSWORD 'twenty'; ALTER USER twenty CREATEDB; +``` + +Create `metadata` schema: +```sql +CREATE SCHEMA IF NOT EXISTS "metadata"; +GRANT ALL ON SCHEMA metadata TO twenty; +``` + +Activate `uuid-ossp` extension: +```sql +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; ``` diff --git a/infra/dev/postgres/init.sql b/infra/dev/postgres/init.sql index ea9e4f485d..a0ffb4e2fc 100644 --- a/infra/dev/postgres/init.sql +++ b/infra/dev/postgres/init.sql @@ -13,3 +13,13 @@ ALTER USER twenty CREATEDB; -- Create the metadata schema if it doesn't exist CREATE SCHEMA IF NOT EXISTS "metadata"; +GRANT ALL ON SCHEMA metadata TO twenty; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Connect to the "default" database +\c "test"; + +-- Create the metadata schema if it doesn't exist +CREATE SCHEMA IF NOT EXISTS "metadata"; +GRANT ALL ON SCHEMA metadata TO twenty; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; diff --git a/server/scripts/run-integration.sh b/server/scripts/run-integration.sh index 7d84441cb0..257326888a 100755 --- a/server/scripts/run-integration.sh +++ b/server/scripts/run-integration.sh @@ -9,7 +9,9 @@ EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo '🟡 - Database is not initialized. Running migrations...' + npx ts-node ./test/utils/setup-db.ts npx prisma migrate reset --force && yarn prisma:generate + yarn typeorm:migrate else echo "🟢 - Database is already initialized." fi diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts index 6c094656e5..865c68fcc8 100644 --- a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts @@ -28,7 +28,7 @@ export class DataSourceMetadataService { }); } - getDataSourcesMedataFromWorkspaceId(workspaceId: string) { + getDataSourcesMetadataFromWorkspaceId(workspaceId: string) { return this.dataSourceMetadataRepository.find({ where: { workspaceId }, order: { createdAt: 'DESC' }, diff --git a/server/src/tenant/metadata/data-source/data-source.service.ts b/server/src/tenant/metadata/data-source/data-source.service.ts index 78efaebcf0..575581aa3f 100644 --- a/server/src/tenant/metadata/data-source/data-source.service.ts +++ b/server/src/tenant/metadata/data-source/data-source.service.ts @@ -106,7 +106,7 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { } const dataSourcesMetadata = - await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId( + await this.dataSourceMetadataService.getDataSourcesMetadataFromWorkspaceId( workspaceId, ); diff --git a/server/src/tenant/metadata/metadata.controller.ts b/server/src/tenant/metadata/metadata.controller.ts index 57c3323a64..c2da837a71 100644 --- a/server/src/tenant/metadata/metadata.controller.ts +++ b/server/src/tenant/metadata/metadata.controller.ts @@ -10,7 +10,6 @@ import { DataSourceMetadataService } from './data-source-metadata/data-source-me import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service'; import { DataSourceService } from './data-source/data-source.service'; import { MigrationGeneratorService } from './migration-generator/migration-generator.service'; -import { uuidToBase36 } from './data-source/data-source.util'; @UseGuards(JwtAuthGuard) @Controller('metadata') @@ -25,7 +24,7 @@ export class MetadataController { @Get() async getMetadata(@AuthWorkspace() workspace: Workspace) { const dataSourcesMetadata = - await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId( + await this.dataSourceMetadataService.getDataSourcesMetadataFromWorkspaceId( workspace.id, ); @@ -42,14 +41,10 @@ export class MetadataController { entities.push(...dataSourceEntities); } - return await this.migrationGenerator.executeMigrationFromPendingMigrations( + await this.migrationGenerator.executeMigrationFromPendingMigrations( workspace.id, ); - this.dataSourceService.createWorkspaceSchema(workspace.id); - - console.log('entities', uuidToBase36(workspace.id), workspace.id); - this.dataSourceService.connectToWorkspaceDataSource(workspace.id); return entities; diff --git a/server/src/tenant/metadata/metadata.datasource.ts b/server/src/tenant/metadata/metadata.datasource.ts index adfee49fa2..01cc074538 100644 --- a/server/src/tenant/metadata/metadata.datasource.ts +++ b/server/src/tenant/metadata/metadata.datasource.ts @@ -9,7 +9,7 @@ config(); const configService = new ConfigService(); export const typeORMMetadataModuleOptions: TypeOrmModuleOptions = { - url: configService.get('PG_DATABASE_URL')!, + url: configService.get('PG_DATABASE_URL'), type: 'postgres', logging: false, schema: 'metadata', diff --git a/server/src/tenant/metadata/metadata.module.ts b/server/src/tenant/metadata/metadata.module.ts index ebc0e93650..336c0968a9 100644 --- a/server/src/tenant/metadata/metadata.module.ts +++ b/server/src/tenant/metadata/metadata.module.ts @@ -14,6 +14,7 @@ import { MigrationGeneratorModule } from './migration-generator/migration-genera const typeORMFactory = async (): Promise => ({ ...typeORMMetadataModuleOptions, + name: 'metadata', }); @Module({ diff --git a/server/src/tenant/metadata/migrations/1695198840363-migrations.ts b/server/src/tenant/metadata/migrations/1695198840363-migrations.ts deleted file mode 100644 index d6674dbda3..0000000000 --- a/server/src/tenant/metadata/migrations/1695198840363-migrations.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class Migrations1695198840363 implements MigrationInterface { - name = 'Migrations1695198840363'; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `CREATE TABLE "metadata"."data_source_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "url" character varying, "schema" character varying, "type" "metadata"."data_source_metadata_type_enum" NOT NULL DEFAULT 'postgres', "name" character varying, "is_remote" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_923752b7e62a300a4969bd0e038" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `CREATE TABLE "metadata"."field_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "object_id" uuid NOT NULL, "type" character varying NOT NULL, "name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c75db587904cad6af109b5c65f1" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `CREATE TABLE "metadata"."object_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "data_source_id" character varying NOT NULL, "name" character varying NOT NULL, "is_custom" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_c8c5f885767b356949c18c201c1" PRIMARY KEY ("id"))`, - ); - await queryRunner.query( - `ALTER TABLE "metadata"."field_metadata" ADD CONSTRAINT "FK_38179b299795e48887fc99f937a" FOREIGN KEY ("object_id") REFERENCES "metadata"."object_metadata"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`, - ); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "metadata"."field_metadata" DROP CONSTRAINT "FK_38179b299795e48887fc99f937a"`, - ); - await queryRunner.query(`DROP TABLE "metadata"."object_metadata"`); - await queryRunner.query(`DROP TABLE "metadata"."field_metadata"`); - await queryRunner.query(`DROP TABLE "metadata"."data_source_metadata"`); - } -} diff --git a/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts b/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts index 1bf7c08b4f..13647df58d 100644 --- a/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts +++ b/server/src/tenant/metadata/migrations/1695214465080-InitMetadataTables.ts @@ -4,6 +4,9 @@ export class InitMetadataTables1695214465080 implements MigrationInterface { name = 'InitMetadataTables1695214465080'; public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TYPE "metadata"."data_source_metadata_type_enum" AS ENUM ('postgres', 'mysql');`, + ); await queryRunner.query( `CREATE TABLE "metadata"."data_source_metadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "url" character varying, "schema" character varying, "type" "metadata"."data_source_metadata_type_enum" NOT NULL DEFAULT 'postgres', "display_name" character varying, "is_remote" boolean NOT NULL DEFAULT false, "workspace_id" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), "updated_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_923752b7e62a300a4969bd0e038" PRIMARY KEY ("id"))`, ); diff --git a/server/src/tenant/universal/universal.resolver.spec.ts b/server/src/tenant/universal/universal.resolver.spec.ts index 772e08e889..a5b6515fa0 100644 --- a/server/src/tenant/universal/universal.resolver.spec.ts +++ b/server/src/tenant/universal/universal.resolver.spec.ts @@ -1,5 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { EnvironmentService } from 'src/integrations/environment/environment.service'; + import { UniversalResolver } from './universal.resolver'; import { UniversalService } from './universal.service'; @@ -14,6 +16,10 @@ describe('UniversalResolver', () => { provide: UniversalService, useValue: {}, }, + { + provide: EnvironmentService, + useValue: {}, + }, ], }).compile(); diff --git a/server/src/tenant/universal/universal.resolver.ts b/server/src/tenant/universal/universal.resolver.ts index d511f8b1a5..0cd484fe36 100644 --- a/server/src/tenant/universal/universal.resolver.ts +++ b/server/src/tenant/universal/universal.resolver.ts @@ -12,7 +12,6 @@ import { UniversalService } from './universal.service'; import { FindManyUniversalArgs } from './args/find-many-universal.args'; import { FindUniqueUniversalArgs } from './args/find-unique-universal.args'; -import { UpdateOneCustomArgs } from './args/update-one-custom.args'; @UseGuards(JwtAuthGuard) @Resolver(() => UniversalEntity) @@ -47,7 +46,7 @@ export class UniversalResolver { } @Query(() => UniversalEntity) - updateOneCustom(@Args() args: UpdateOneCustomArgs): UniversalEntity { + updateOneCustom(): UniversalEntity { if (!this.environmentService.isFlexibleBackendEnabled()) { throw new ForbiddenException(); } @@ -61,7 +60,7 @@ export class UniversalResolver { } @Query(() => UniversalEntity) - deleteOneCustom(@Args() args: UpdateOneCustomArgs): UniversalEntity { + deleteOneCustom(): UniversalEntity { if (!this.environmentService.isFlexibleBackendEnabled()) { throw new ForbiddenException(); } diff --git a/server/test/utils/setup-db.ts b/server/test/utils/setup-db.ts new file mode 100644 index 0000000000..b14f9d33b8 --- /dev/null +++ b/server/test/utils/setup-db.ts @@ -0,0 +1,56 @@ +// check-db.ts + +import { PrismaClient } from '@prisma/client'; + +const prisma = new PrismaClient(); + +const createMetadataSchema = async () => { + try { + await prisma.$queryRawUnsafe<[any]>( + `CREATE SCHEMA IF NOT EXISTS "metadata";`, + ); + await prisma.$queryRawUnsafe<[any]>( + `GRANT ALL ON SCHEMA metadata TO twenty;`, + ); + + return true; + } catch { + return false; + } +}; + +const activateUUIDExtension = async () => { + try { + const result = await prisma.$queryRawUnsafe<[any]>( + `CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`, + ); + + return result; + } catch { + return false; + } +}; + +const main = async () => { + const metadataSchemaCreationSuccess = await createMetadataSchema(); + const uuidExtensionActivationSuccess = await activateUUIDExtension(); + + if (!metadataSchemaCreationSuccess) { + throw new Error(`Failed to create metadata schema`); + } + + if (!uuidExtensionActivationSuccess) { + throw new Error(`Failed to activate uuid extension`); + } +}; + +main() + .then(() => { + process.exit(0); + }) + .catch(() => { + process.exit(1); + }) + .finally(async () => { + await prisma.$disconnect(); + });