Standard migration command (#2236)

* Add Standard Object migration commands

* rebase

* add sync-tenant-metadata command

* fix naming

* renaming command class names

* remove field deletion and use object cascade instead

---------

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>

server/package.json

@@ -35,7 +35,8 @@
     "database:migrate": "yarn typeorm:migrate && yarn prisma:migrate",
     "database:generate": "yarn prisma:generate",
     "database:seed": "yarn prisma:seed",
-    "database:reset": "yarn database:truncate && yarn database:init"
+    "database:reset": "yarn database:truncate && yarn database:init",
+    "command": "node dist/src/command"
   },
   "dependencies": {
     "@apollo/server": "^4.7.3",
@@ -91,6 +92,7 @@
     "lodash.snakecase": "^4.1.1",
     "lodash.upperfirst": "^4.3.1",
     "ms": "^2.1.3",
+    "nest-commander": "^3.12.0",
     "passport": "^0.6.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
@@ -157,4 +159,4 @@
     "schema": "src/database/schema.prisma",
     "seed": "ts-node src/database/seeds/index.ts"
   }
-}
+}

server/src/command.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { AppModule } from './app.module';
+
+import { MetadataCommandModule } from './metadata/commands/metadata-command.module';
+
+@Module({
+  imports: [AppModule, MetadataCommandModule],
+})
+export class CommandModule {}

server/src/command.ts

@@ -0,0 +1,9 @@
+import { CommandFactory } from 'nest-commander';
+
+import { CommandModule } from './command.module';
+
+async function bootstrap() {
+  // TODO: inject our own logger service to handle the output (Sentry, etc.)
+  await CommandFactory.run(CommandModule, ['warn', 'error']);
+}
+bootstrap();

server/src/metadata/commands/metadata-command.module.ts

@@ -0,0 +1,24 @@
+import { Module } from '@nestjs/common';
+
+import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
+import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-runner.module';
+import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
+import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
+import { TenantInitialisationModule } from 'src/metadata/tenant-initialisation/tenant-initialisation.module';
+import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
+
+import { SyncTenantMetadataCommand } from './sync-tenant-metadata.command';
+import { RunTenantMigrationsCommand } from './run-tenant-migrations.command';
+
+@Module({
+  imports: [
+    TenantMigrationModule,
+    MigrationRunnerModule,
+    ObjectMetadataModule,
+    FieldMetadataModule,
+    DataSourceMetadataModule,
+    TenantInitialisationModule,
+  ],
+  providers: [RunTenantMigrationsCommand, SyncTenantMetadataCommand],
+})
+export class MetadataCommandModule {}

server/src/metadata/commands/run-tenant-migrations.command.ts

@@ -0,0 +1,45 @@
+import { Command, CommandRunner, Option } from 'nest-commander';
+
+import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service';
+import { MigrationRunnerService } from 'src/metadata/migration-runner/migration-runner.service';
+
+// TODO: implement dry-run
+interface RunTenantMigrationsOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'tenant:migrate',
+  description: 'Run tenant migrations',
+})
+export class RunTenantMigrationsCommand extends CommandRunner {
+  constructor(
+    private readonly tenantMigrationService: TenantMigrationService,
+    private readonly migrationRunnerService: MigrationRunnerService,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: RunTenantMigrationsOptions,
+  ): Promise<void> {
+    // TODO: run in a dedicated job + run queries in a transaction.
+    await this.tenantMigrationService.insertStandardMigrations(
+      options.workspaceId,
+    );
+    await this.migrationRunnerService.executeMigrationFromPendingMigrations(
+      options.workspaceId,
+    );
+  }
+
+  // TODO: workspaceId should be optional and we should run migrations for all workspaces
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+}

server/src/metadata/commands/sync-tenant-metadata.command.ts

@@ -0,0 +1,57 @@
+import { Command, CommandRunner, Option } from 'nest-commander';
+
+import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service';
+import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service';
+import { TenantInitialisationService } from 'src/metadata/tenant-initialisation/tenant-initialisation.service';
+import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
+
+// TODO: implement dry-run
+interface RunTenantMigrationsOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'tenant:sync-metadata',
+  description: 'Sync metadata',
+})
+export class SyncTenantMetadataCommand extends CommandRunner {
+  constructor(
+    private readonly objectMetadataService: ObjectMetadataService,
+    private readonly fieldMetadataService: FieldMetadataService,
+    private readonly dataSourceMetadataService: DataSourceMetadataService,
+    private readonly tenantInitialisationService: TenantInitialisationService,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: RunTenantMigrationsOptions,
+  ): Promise<void> {
+    // TODO: run in a dedicated job + run queries in a transaction.
+    const dataSourceMetadata =
+      await this.dataSourceMetadataService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
+        options.workspaceId,
+      );
+
+    // TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them.
+    await this.objectMetadataService.deleteMany({
+      workspaceId: { eq: options.workspaceId },
+    });
+
+    // TODO: this should not be the responsibility of tenantInitialisationService.
+    await this.tenantInitialisationService.createObjectsAndFieldsMetadata(
+      dataSourceMetadata.id,
+      options.workspaceId,
+    );
+  }
+
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+}

server/src/metadata/data-source/data-source.service.ts

@@ -132,5 +132,10 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
   async onModuleDestroy() {
     // Destroy main data source "default" schema
     await this.mainDataSource.destroy();
+
+    // Destroy all workspace data sources
+    for (const [, dataSource] of this.dataSources) {
+      await dataSource.destroy();
+    }
   }
 }

server/src/metadata/migration-runner/migration-runner.service.ts

@@ -35,6 +35,10 @@ export class MigrationRunnerService {
     const pendingMigrations =
       await this.tenantMigrationService.getPendingMigrations(workspaceId);
 
+    if (pendingMigrations.length === 0) {
+      return [];
+    }
+
     const flattenedPendingMigrations: TenantMigrationTableAction[] =
       pendingMigrations.reduce((acc, pendingMigration) => {
         return [...acc, ...pendingMigration.migrations];

server/src/metadata/tenant-initialisation/standard-objects/people/people.metadata.json

@@ -0,0 +1,113 @@
+{
+  "nameSingular": "personV2",
+  "namePlural": "peopleV2",
+  "labelSingular": "Person",
+  "labelPlural": "People",
+  "targetTableName": "person",
+  "description": "A person",
+  "icon": "people",
+  "fields": [
+    {
+      "type": "text",
+      "name": "firstName",
+      "label": "First Name",
+      "targetColumnMap": {
+        "value": "firstName"
+      },
+      "description": "First Name of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "lastName",
+      "label": "Last Name",
+      "targetColumnMap": {
+        "value": "lastName"
+      },
+      "description": "Last Name of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "email",
+      "label": "Email",
+      "targetColumnMap": {
+        "value": "email"
+      },
+      "description": "Email of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "phone",
+      "name": "phone",
+      "label": "Phone",
+      "targetColumnMap": {
+        "value": "phone"
+      },
+      "description": "phone of the company",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "city",
+      "label": "City",
+      "targetColumnMap": {
+        "value": "city"
+      },
+      "description": "City of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "text",
+      "name": "jobTitle",
+      "label": "Job Title",
+      "targetColumnMap": {
+        "value": "jobTitle"
+      },
+      "description": "Job title of the person",
+      "icon": null,
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "linkedinUrl",
+      "label": "Linkedin URL",
+      "targetColumnMap": {
+        "text": "Linkedin URL",
+        "link": "linkedinUrl"
+      },
+      "description": "Linkedin URL of the person",
+      "icon": "url",
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "xUrl",
+      "label": "X URL",
+      "targetColumnMap": {
+        "text": "X URL",
+        "link": "xUrl"
+      },
+      "description": "X URL of the person",
+      "icon": "url",
+      "isNullable": true
+    },
+    {
+      "type": "url",
+      "name": "avatarUrl",
+      "label": "Avatar URL",
+      "targetColumnMap": {
+        "text": "Avatar URL",
+        "link": "avatarUrl"
+      },
+      "description": "Avatar URL of the person",
+      "icon": "url",
+      "isNullable": true
+    }
+  ]
+}

server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts

@@ -1,5 +1,7 @@
-import companyObject from './companies.metadata.json';
+import companyObject from './companies/companies.metadata.json';
+import personObject from './people/people.metadata.json';
 
 export const standardObjectsMetadata = {
   companyV2: companyObject,
+  personV2: personObject,
 };

server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts

@@ -1,4 +1,4 @@
-import companySeeds from './companies.seeds.json';
+import companySeeds from './companies/companies.seeds.json';
 
 export const standardObjectsSeeds = {
   companyV2: companySeeds,

server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts

@@ -67,7 +67,7 @@ export class TenantInitialisationService {
    * @param dataSourceMetadataId
    * @param workspaceId
    */
-  private async createObjectsAndFieldsMetadata(
+  public async createObjectsAndFieldsMetadata(
     dataSourceMetadataId: string,
     workspaceId: string,
   ) {

server/src/metadata/tenant-migration/migrations/1697618010-addPeopleTable.ts

@@ -0,0 +1,59 @@
+import { TenantMigrationTableAction } from 'src/metadata/tenant-migration/tenant-migration.entity';
+
+export const addPeopleTable: TenantMigrationTableAction[] = [
+  {
+    name: 'people',
+    action: 'create',
+  },
+  {
+    name: 'people',
+    action: 'alter',
+    columns: [
+      {
+        name: 'firstName',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'lastName',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'email',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'phone',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'city',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'jobTitle',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'linkedinUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'xUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+      {
+        name: 'avatarUrl',
+        type: 'varchar',
+        action: 'create',
+      },
+    ],
+  },
+];

server/src/metadata/tenant-migration/standard-migrations.ts

@@ -1,6 +1,8 @@
 import { addCompanyTable } from './migrations/1697618009-addCompanyTable';
+import { addPeopleTable } from './migrations/1697618010-addPeopleTable';
 
 // TODO: read the folder and return all migrations
 export const standardMigrations = {
   '1697618009-addCompanyTable': addCompanyTable,
+  '1697618010-addPeopleTable': addPeopleTable,
 };

server/src/metadata/tenant-migration/tenant-migration.service.ts

@@ -21,7 +21,7 @@ export class TenantMigrationService {
    *
    * @param workspaceId
    */
-  public async insertStandardMigrations(workspaceId: string) {
+  public async insertStandardMigrations(workspaceId: string): Promise<void> {
     // TODO: we actually don't need to fetch all of them, to improve later so it scales well.
     const insertedStandardMigrations =
       await this.tenantMigrationRepository.find({
@@ -34,20 +34,21 @@ export class TenantMigrationService {
         return acc;
       }, {});
 
-    const standardMigrationsList = standardMigrations;
-
     const standardMigrationsListThatNeedToBeInserted = Object.entries(
-      standardMigrationsList,
+      standardMigrations,
     )
       .filter(([name]) => !insertedStandardMigrationsMapByName[name])
       .map(([name, migrations]) => ({ name, migrations }));
 
-    await this.tenantMigrationRepository.save(
+    const standardMigrationsThatNeedToBeInserted =
       standardMigrationsListThatNeedToBeInserted.map((migration) => ({
         ...migration,
         workspaceId,
         isCustom: false,
-      })),
+      }));
+
+    await this.tenantMigrationRepository.save(
+      standardMigrationsThatNeedToBeInserted,
     );
   }
 
@@ -60,7 +61,7 @@ export class TenantMigrationService {
   public async getPendingMigrations(
     workspaceId: string,
   ): Promise<TenantMigration[]> {
-    return this.tenantMigrationRepository.find({
+    return await this.tenantMigrationRepository.find({
       order: { createdAt: 'ASC' },
       where: {
         appliedAt: IsNull(),

server/yarn.lock

@@ -1329,6 +1329,20 @@
   resolved "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz"
   integrity sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw==
 
+"@fig/complete-commander@^2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@fig/complete-commander/-/complete-commander-2.0.1.tgz#6dd84f8812389107529aaedebd1bb67ac8bc16c6"
+  integrity sha512-AbGETely7iwD4F7XHe4g7pW6icWYYqJMdQog8CdEi9syU/av5L0O24BvCfgEeGO6TRPMpC+rFL7ZDJsqRtckOA==
+  dependencies:
+    prettier "^2.3.2"
+
+"@golevelup/nestjs-discovery@4.0.0":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/@golevelup/nestjs-discovery/-/nestjs-discovery-4.0.0.tgz#3428f0b620b51e4d425bc9e41cc8f2f338472dc1"
+  integrity sha512-iyZLYip9rhVMR0C93vo860xmboRrD5g5F5iEOfpeblGvYSz8ymQrL9RAST7x/Fp3n+TAXSeOLzDIASt+rak68g==
+  dependencies:
+    lodash "^4.17.21"
+
 "@graphql-tools/executor@^1.0.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@graphql-tools/executor/-/executor-1.2.0.tgz#6c45f4add765769d9820c4c4405b76957ba39c79"
@@ -4507,6 +4521,11 @@ combined-stream@^1.0.8:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-11.0.0.tgz#43e19c25dbedc8256203538e8d7e9346877a6f67"
+  integrity sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==
+
 commander@4.1.1:
   version "4.1.1"
   resolved "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz"
@@ -4612,6 +4631,16 @@ cors@2.8.5, cors@^2.8.5:
     object-assign "^4"
     vary "^1"
 
+cosmiconfig@8.2.0:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.2.0.tgz#f7d17c56a590856cd1e7cee98734dca272b0d8fd"
+  integrity sha512-3rTMnFJA1tCOPwRxtgF4wd7Ab2qvDbL8jX+3smjIbS4HlZBagTlpERbdN7iAbWlrfxE3M8c27kTwTawQ7st+OQ==
+  dependencies:
+    import-fresh "^3.2.1"
+    js-yaml "^4.1.0"
+    parse-json "^5.0.0"
+    path-type "^4.0.0"
+
 cosmiconfig@^7.0.1:
   version "7.1.0"
   resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz"
@@ -7389,6 +7418,17 @@ neo-async@^2.6.2:
   resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz"
   integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
 
+nest-commander@^3.12.0:
+  version "3.12.0"
+  resolved "https://registry.yarnpkg.com/nest-commander/-/nest-commander-3.12.0.tgz#9d1d7df7c9fa129d899c1e85c49eeb749bd11376"
+  integrity sha512-6ncAT13l7lH9Hya3GKKOIG+ltRD7b4idTlbuNXaCsm2IJIuuVxnx35UxiogJPz+GarE437H3I+GJXzehBnDQqg==
+  dependencies:
+    "@fig/complete-commander" "^2.0.1"
+    "@golevelup/nestjs-discovery" "4.0.0"
+    commander "11.0.0"
+    cosmiconfig "8.2.0"
+    inquirer "8.2.5"
+
 new-github-issue-url@0.2.1:
   version "0.2.1"
   resolved "https://registry.npmjs.org/new-github-issue-url/-/new-github-issue-url-0.2.1.tgz"