feat: workspace:health nullable fix (#3882)

This commit is contained in:
Jérémy M 2024-02-08 18:22:29 +01:00 committed by GitHub
parent 2ba9a209e8
commit d3fe1b9e31
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 265 additions and 131 deletions

View File

@ -1,2 +1,3 @@
dist/*
.local-storage
.local-storage
logs/**/*

View File

@ -0,0 +1,38 @@
import { Injectable } from '@nestjs/common';
import { existsSync } from 'fs';
import fs from 'fs/promises';
import { kebabCase } from 'src/utils/kebab-case';
@Injectable()
export class CommandLogger {
constructor(private readonly className: string) {}
async writeLog(
fileName: string,
data: unknown,
append: boolean = false,
): Promise<void> {
const path = `./logs/${kebabCase(this.className)}`;
if (existsSync(path) === false) {
await fs.mkdir(path, { recursive: true });
}
try {
await fs.writeFile(
`${path}/${fileName}.json`,
JSON.stringify(data, null, 2),
{
flag: append ? 'a' : 'w',
},
);
} catch (err) {
console.error(
`Error writing to file ${path}/${fileName}.json: ${err?.message}`,
);
throw err;
}
}
}

View File

@ -5,12 +5,14 @@ import { WorkspaceHealthMode } from 'src/workspace/workspace-health/interfaces/w
import { WorkspaceHealthFixKind } from 'src/workspace/workspace-health/interfaces/workspace-health-fix-kind.interface';
import { WorkspaceHealthService } from 'src/workspace/workspace-health/workspace-health.service';
import { CommandLogger } from 'src/commands/command-logger';
interface WorkspaceHealthCommandOptions {
workspaceId: string;
verbose?: boolean;
mode?: WorkspaceHealthMode;
fix?: WorkspaceHealthFixKind;
dryRun?: boolean;
}
@Command({
@ -18,6 +20,10 @@ interface WorkspaceHealthCommandOptions {
description: 'Check health of the given workspace.',
})
export class WorkspaceHealthCommand extends CommandRunner {
private readonly commandLogger = new CommandLogger(
WorkspaceHealthCommand.name,
);
constructor(private readonly workspaceHealthService: WorkspaceHealthService) {
super();
}
@ -48,11 +54,29 @@ export class WorkspaceHealthCommand extends CommandRunner {
}
if (options.fix) {
await this.workspaceHealthService.fixIssues(
console.log(chalk.yellow('Fixing issues'));
const workspaceMigrations = await this.workspaceHealthService.fixIssues(
options.workspaceId,
issues,
options.fix,
{
type: options.fix,
applyChanges: !options.dryRun,
},
);
if (options.dryRun) {
await this.commandLogger.writeLog(
`workspace-health-${options.fix}-migrations`,
workspaceMigrations,
);
} else {
console.log(
chalk.green(
`Fixed ${workspaceMigrations.length}/${issues.length} issues`,
),
);
}
}
}
@ -100,4 +124,13 @@ export class WorkspaceHealthCommand extends CommandRunner {
return value as WorkspaceHealthMode;
}
@Option({
flags: '-d, --dry-run',
description: 'Dry run without applying changes',
required: false,
})
dryRun(): boolean {
return true;
}
}

View File

@ -26,7 +26,6 @@ export enum WorkspaceHealthIssueType {
COLUMN_OPTIONS_NOT_VALID = 'COLUMN_OPTIONS_NOT_VALID',
RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID = 'RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID',
RELATION_FOREIGN_KEY_NOT_VALID = 'RELATION_FOREIGN_KEY_NOT_VALID',
RELATION_NULLABILITY_CONFLICT = 'RELATION_NULLABILITY_CONFLICT',
RELATION_FOREIGN_KEY_CONFLICT = 'RELATION_FOREIGN_KEY_CONFLICT',
RELATION_TYPE_NOT_VALID = 'RELATION_TYPE_NOT_VALID',
}
@ -83,7 +82,6 @@ export interface WorkspaceHealthRelationIssue<
T,
| WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID
| WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT
| WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID
>;

View File

@ -188,16 +188,6 @@ export class RelationMetadataHealthService {
});
}
if (relationColumn.isNullable !== relationFieldMetadata.isNullable) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} foreign key is not properly set`,
});
}
if (
relationMetadata.relationType === RelationMetadataType.ONE_TO_ONE &&
!relationColumn.isUnique

View File

@ -5,29 +5,69 @@ import { EntityManager } from 'typeorm';
import {
WorkspaceHealthColumnIssue,
WorkspaceHealthIssueType,
WorkspaceHealthRelationIssue,
} from 'src/workspace/workspace-health/interfaces/workspace-health-issue.interface';
import { WorkspaceMigrationBuilderAction } from 'src/workspace/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory';
type WorkspaceHealthNullableIssue =
| WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>
| WorkspaceHealthRelationIssue<WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT>;
WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>;
@Injectable()
export class WorkspaceFixNullableService {
constructor() {}
constructor(
private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory,
) {}
async fix(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
manager: EntityManager,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
objectMetadataCollection: ObjectMetadataEntity[],
// eslint-disable-next-line @typescript-eslint/no-unused-vars
issues: WorkspaceHealthNullableIssue[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
// TODO: Implement nullable fix
return [];
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
for (const issue of issues) {
switch (issue.type) {
case WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT: {
const columnNullabilityWorkspaceMigrations =
await this.fixColumnNullabilityIssues(
objectMetadataCollection,
issues.filter(
(issue) =>
issue.type ===
WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT,
) as WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>[],
);
workspaceMigrations.push(...columnNullabilityWorkspaceMigrations);
break;
}
}
}
return workspaceMigrations;
}
private async fixColumnNullabilityIssues(
objectMetadataCollection: ObjectMetadataEntity[],
issues: WorkspaceHealthColumnIssue<WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT>[],
): Promise<Partial<WorkspaceMigrationEntity>[]> {
const fieldMetadataUpdateCollection = issues.map((issue) => {
return {
current: {
...issue.fieldMetadata,
isNullable: issue.columnStructure?.isNullable ?? false,
},
altered: issue.fieldMetadata,
};
});
return this.workspaceMigrationFieldFactory.create(
objectMetadataCollection,
fieldMetadataUpdateCollection,
WorkspaceMigrationBuilderAction.UPDATE,
);
}
}

View File

@ -2,11 +2,6 @@ import { WorkspaceHealthIssueType } from 'src/workspace/workspace-health/interfa
export const isWorkspaceHealthNullableIssue = (
type: WorkspaceHealthIssueType,
): type is
| WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT
| WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT => {
return type === WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT ||
type === WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT
? true
: false;
): type is WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT => {
return type === WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT;
};

View File

@ -121,8 +121,16 @@ export class WorkspaceHealthService {
async fixIssues(
workspaceId: string,
issues: WorkspaceHealthIssue[],
type: WorkspaceHealthFixKind,
): Promise<void> {
options: {
type: WorkspaceHealthFixKind;
applyChanges?: boolean;
},
): Promise<Partial<WorkspaceMigrationEntity>[]> {
let workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
// Set default options
options.applyChanges ??= true;
const queryRunner = this.metadataDataSource.createQueryRunner();
await queryRunner.connect();
@ -137,16 +145,25 @@ export class WorkspaceHealthService {
const objectMetadataCollection =
await this.objectMetadataService.findManyWithinWorkspace(workspaceId);
const workspaceMigrations = await this.workspaceFixService.fix(
workspaceMigrations = await this.workspaceFixService.fix(
manager,
objectMetadataCollection,
type,
options.type,
issues,
);
// Save workspace migrations into the database
await workspaceMigrationRepository.save(workspaceMigrations);
if (!options.applyChanges) {
// Rollback transactions
await queryRunner.rollbackTransaction();
await queryRunner.release();
return workspaceMigrations;
}
// Commit the transaction
await queryRunner.commitTransaction();
@ -160,5 +177,7 @@ export class WorkspaceHealthService {
} finally {
await queryRunner.release();
}
return workspaceMigrations;
}
}

View File

@ -78,6 +78,7 @@ export class WorkspaceMigrationRunnerService {
await queryRunner.commitTransaction();
} catch (error) {
console.error('Error executing migration', error);
await queryRunner.rollbackTransaction();
throw error;
} finally {

View File

@ -0,0 +1,73 @@
import { Injectable } from '@nestjs/common';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
import { CommandLogger } from 'src/commands/command-logger';
@Injectable()
export class SyncWorkspaceLoggerService {
private readonly commandLogger = new CommandLogger(
SyncWorkspaceLoggerService.name,
);
constructor() {}
async saveLogs(
storage: WorkspaceSyncStorage,
workspaceMigrations: WorkspaceMigrationEntity[],
) {
// Save workspace migrations
await this.commandLogger.writeLog(
'workspace-migrations',
workspaceMigrations,
);
// Save object metadata create collection
await this.commandLogger.writeLog(
'object-metadata-create-collection',
storage.objectMetadataCreateCollection,
);
// Save object metadata update collection
await this.commandLogger.writeLog(
'object-metadata-update-collection',
storage.objectMetadataUpdateCollection,
);
// Save object metadata delete collection
await this.commandLogger.writeLog(
'object-metadata-delete-collection',
storage.objectMetadataDeleteCollection,
);
// Save field metadata create collection
await this.commandLogger.writeLog(
'field-metadata-create-collection',
storage.fieldMetadataCreateCollection,
);
// Save field metadata update collection
await this.commandLogger.writeLog(
'field-metadata-update-collection',
storage.fieldMetadataUpdateCollection,
);
// Save field metadata delete collection
await this.commandLogger.writeLog(
'field-metadata-delete-collection',
storage.fieldMetadataDeleteCollection,
);
// Save relation metadata create collection
await this.commandLogger.writeLog(
'relation-metadata-create-collection',
storage.relationMetadataCreateCollection,
);
// Save relation metadata delete collection
await this.commandLogger.writeLog(
'relation-metadata-delete-collection',
storage.relationMetadataDeleteCollection,
);
}
}

View File

@ -3,6 +3,8 @@ import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSourceService } from 'src/metadata/data-source/data-source.service';
import { WorkspaceSyncMetadataService } from 'src/workspace/workspace-sync-metadata/workspace-sync-metadata.service';
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
// TODO: implement dry-run
interface RunWorkspaceMigrationsOptions {
workspaceId: string;
@ -17,6 +19,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
constructor(
private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService,
private readonly dataSourceService: DataSourceService,
private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService,
) {
super();
}
@ -31,13 +34,21 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner {
options.workspaceId,
);
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
{
workspaceId: options.workspaceId,
dataSourceId: dataSourceMetadata.id,
},
{ dryRun: options.dryRun },
);
const { storage, workspaceMigrations } =
await this.workspaceSyncMetadataService.syncStandardObjectsAndFieldsMetadata(
{
workspaceId: options.workspaceId,
dataSourceId: dataSourceMetadata.id,
},
{ applyChanges: !options.dryRun },
);
if (options.dryRun) {
await this.syncWorkspaceLoggerService.saveLogs(
storage,
workspaceMigrations,
);
}
}
@Option({

View File

@ -5,8 +5,10 @@ import { WorkspaceSyncMetadataModule } from 'src/workspace/workspace-sync-metada
import { SyncWorkspaceMetadataCommand } from './sync-workspace-metadata.command';
import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service';
@Module({
imports: [WorkspaceSyncMetadataModule, DataSourceModule],
providers: [SyncWorkspaceMetadataCommand],
providers: [SyncWorkspaceMetadataCommand, SyncWorkspaceLoggerService],
})
export class WorkspaceSyncMetadataCommandsModule {}

View File

@ -1,76 +0,0 @@
import { Injectable } from '@nestjs/common';
import { existsSync } from 'fs';
import fs from 'fs/promises';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
@Injectable()
export class WorkspaceLogsService {
constructor() {}
async saveLogs(
storage: WorkspaceSyncStorage,
workspaceMigrations: WorkspaceMigrationEntity[],
) {
// Check if `logs` folder exists
if (existsSync('./logs') === false) {
await fs.mkdir('./logs', { recursive: true });
}
// Save workspace migrations
await fs.writeFile(
'./logs/workspace-migrations.json',
JSON.stringify(workspaceMigrations, null, 2),
);
// Save object metadata create collection
await fs.writeFile(
'./logs/object-metadata-create-collection.json',
JSON.stringify(storage.objectMetadataCreateCollection, null, 2),
);
// Save object metadata update collection
await fs.writeFile(
'./logs/object-metadata-update-collection.json',
JSON.stringify(storage.objectMetadataUpdateCollection, null, 2),
);
// Save object metadata delete collection
await fs.writeFile(
'./logs/object-metadata-delete-collection.json',
JSON.stringify(storage.objectMetadataDeleteCollection, null, 2),
);
// Save field metadata create collection
await fs.writeFile(
'./logs/field-metadata-create-collection.json',
JSON.stringify(storage.fieldMetadataCreateCollection, null, 2),
);
// Save field metadata update collection
await fs.writeFile(
'./logs/field-metadata-update-collection.json',
JSON.stringify(storage.fieldMetadataUpdateCollection, null, 2),
);
// Save field metadata delete collection
await fs.writeFile(
'./logs/field-metadata-delete-collection.json',
JSON.stringify(storage.fieldMetadataDeleteCollection, null, 2),
);
// Save relation metadata create collection
await fs.writeFile(
'./logs/relation-metadata-create-collection.json',
JSON.stringify(storage.relationMetadataCreateCollection, null, 2),
);
// Save relation metadata delete collection
await fs.writeFile(
'./logs/relation-metadata-delete-collection.json',
JSON.stringify(storage.relationMetadataDeleteCollection, null, 2),
);
}
}

View File

@ -13,7 +13,6 @@ import { workspaceSyncMetadataComparators } from 'src/workspace/workspace-sync-m
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
import { WorkspaceLogsService } from 'src/workspace/workspace-sync-metadata/services/workspace-logs.service';
import { WorkspaceMigrationBuilderModule } from 'src/workspace/workspace-migration-builder/workspace-migration-builder.module';
@Module({
@ -38,7 +37,6 @@ import { WorkspaceMigrationBuilderModule } from 'src/workspace/workspace-migrati
WorkspaceSyncObjectMetadataService,
WorkspaceSyncRelationMetadataService,
WorkspaceSyncMetadataService,
WorkspaceLogsService,
],
exports: [WorkspaceSyncMetadataService],
})

View File

@ -10,7 +10,6 @@ import { FeatureFlagFactory } from 'src/workspace/workspace-sync-metadata/factor
import { WorkspaceSyncObjectMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-object-metadata.service';
import { WorkspaceSyncRelationMetadataService } from 'src/workspace/workspace-sync-metadata/services/workspace-sync-relation-metadata.service';
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
import { WorkspaceLogsService } from 'src/workspace/workspace-sync-metadata/services/workspace-logs.service';
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
@Injectable()
@ -24,7 +23,6 @@ export class WorkspaceSyncMetadataService {
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService,
private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService,
private readonly workspaceLogsService: WorkspaceLogsService,
) {}
/**
@ -37,18 +35,23 @@ export class WorkspaceSyncMetadataService {
*/
public async syncStandardObjectsAndFieldsMetadata(
context: WorkspaceSyncContext,
options?: { dryRun?: boolean },
) {
this.logger.log('Syncing standard objects and fields metadata');
options: { applyChanges?: boolean } = { applyChanges: true },
): Promise<{
workspaceMigrations: WorkspaceMigrationEntity[];
storage: WorkspaceSyncStorage;
}> {
let workspaceMigrations: WorkspaceMigrationEntity[] = [];
const storage = new WorkspaceSyncStorage();
const queryRunner = this.metadataDataSource.createQueryRunner();
this.logger.log('Syncing standard objects and fields metadata');
await queryRunner.connect();
await queryRunner.startTransaction();
const manager = queryRunner.manager;
try {
const storage = new WorkspaceSyncStorage();
const workspaceMigrationRepository = manager.getRepository(
WorkspaceMigrationEntity,
);
@ -76,20 +79,23 @@ export class WorkspaceSyncMetadataService {
);
// Save workspace migrations into the database
const workspaceMigrations = await workspaceMigrationRepository.save([
workspaceMigrations = await workspaceMigrationRepository.save([
...workspaceObjectMigrations,
...workspaceRelationMigrations,
]);
// If we're running a dry run, rollback the transaction and do not execute migrations
if (options?.dryRun) {
if (!options.applyChanges) {
this.logger.log('Running in dry run mode, rolling back transaction');
await queryRunner.rollbackTransaction();
await this.workspaceLogsService.saveLogs(storage, workspaceMigrations);
await queryRunner.release();
return;
return {
workspaceMigrations,
storage,
};
}
await queryRunner.commitTransaction();
@ -104,5 +110,10 @@ export class WorkspaceSyncMetadataService {
} finally {
await queryRunner.release();
}
return {
workspaceMigrations,
storage,
};
}
}