From 78f4916b6d6cf847f82e704d68714b2d82dba6f2 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 15 Dec 2021 01:34:18 +0700 Subject: [PATCH] Show model diff (#562) Signed-off-by: Andrey Sobolev --- common/config/rush/pnpm-lock.yaml | 70 +++++-------- dev/tool/package.json | 3 +- dev/tool/src/index.ts | 9 +- dev/tool/src/mdiff.ts | 157 ++++++++++++++++++++++++++++++ dev/tool/src/workspace.ts | 38 ++++++++ 5 files changed, 227 insertions(+), 50 deletions(-) create mode 100644 dev/tool/src/mdiff.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index e4a6899466..1da7b771ed 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -110,7 +110,6 @@ specifiers: '@types/express-fileupload': ^1.1.7 '@types/faker': ~5.5.9 '@types/heft-jest': ^1.0.2 - '@types/jpeg-js': ~0.3.7 '@types/koa': ^2.13.4 '@types/koa-bodyparser': ^4.3.3 '@types/koa-router': ^7.4.4 @@ -136,6 +135,7 @@ specifiers: express: ^4.17.1 express-fileupload: ^1.2.1 faker: ~5.5.3 + fast-equals: ^2.0.3 file-loader: ^6.2.0 filesize: ^8.0.3 intl-messageformat: ^9.7.1 @@ -279,7 +279,6 @@ dependencies: '@types/express-fileupload': 1.1.7 '@types/faker': 5.5.9 '@types/heft-jest': 1.0.2 - '@types/jpeg-js': 0.3.7 '@types/koa': 2.13.4 '@types/koa-bodyparser': 4.3.3 '@types/koa-router': 7.4.4 @@ -305,6 +304,7 @@ dependencies: express: 4.17.1 express-fileupload: 1.2.1 faker: 5.5.3 + fast-equals: 2.0.4 file-loader: 6.2.0_webpack@5.57.1 filesize: 8.0.3 intl-messageformat: 9.7.1 @@ -4502,6 +4502,10 @@ packages: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false + /fast-equals/2.0.4: + resolution: {integrity: sha512-caj/ZmjHljPrZtbzJ3kfH5ia/k4mTJe/qSiXAGzxZWRZgsgDV0cvNaQULqUX8t0/JVlzzEdYOwCN5DmzTxoD4w==} + dev: false + /fast-glob/3.2.7: resolution: {integrity: sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==} engines: {node: '>=8'} @@ -9134,37 +9138,6 @@ packages: webpack: 5.57.1_webpack-cli@4.8.0 dev: false - /ts-node/10.2.1_8304ecd715830f7c190b4d1dea90b100: - resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==} - engines: {node: '>=12.0.0'} - hasBin: true - peerDependencies: - '@swc/core': '>=1.2.50' - '@swc/wasm': '>=1.2.50' - '@types/node': '*' - typescript: '>=2.7' - peerDependenciesMeta: - '@swc/core': - optional: true - '@swc/wasm': - optional: true - dependencies: - '@cspotcode/source-map-support': 0.6.1 - '@tsconfig/node10': 1.0.8 - '@tsconfig/node12': 1.0.9 - '@tsconfig/node14': 1.0.1 - '@tsconfig/node16': 1.0.2 - '@types/node': 16.10.3 - acorn: 8.5.0 - acorn-walk: 8.2.0 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 4.4.3 - yn: 3.1.1 - dev: false - /ts-node/10.2.1_c2efd757c6d07d33b05f123839e1b1a4: resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==} engines: {node: '>=12.0.0'} @@ -10052,7 +10025,7 @@ packages: dependencies: '@rushstack/heft': 0.41.1 '@types/heft-jest': 1.0.2 - '@types/node': 16.10.3 + '@types/node': 16.11.12 '@typescript-eslint/eslint-plugin': 5.4.0_87dbf04088b125598d0271706532eaf3 '@typescript-eslint/parser': 5.4.0_eslint@7.32.0+typescript@4.4.3 eslint: 7.32.0 @@ -10292,7 +10265,7 @@ packages: dev: false file:projects/contact-resources.tgz_476f694f64637160ae71e12ff57815b9: - resolution: {integrity: sha512-5aEH1fHFn/BbM8mJcS2gPNW9koEkNtX8P3uC+iEID5xQPUuJV5rsfba17/Iq2bYj5DeLkyvNO5wAwoECwVCvRA==, tarball: file:projects/contact-resources.tgz} + resolution: {integrity: sha512-4SUPVPcAFE83qvwymkwI7i+z0ioEgAmM+H08vsjXRkEeD0vOlRLD3tEBX+0W49mL93EZG0TUEPP8zPcl7wiiAg==, tarball: file:projects/contact-resources.tgz} id: file:projects/contact-resources.tgz name: '@rush-temp/contact-resources' version: 0.0.0 @@ -10353,7 +10326,7 @@ packages: dependencies: '@rushstack/heft': 0.41.1 '@types/heft-jest': 1.0.2 - '@types/node': 16.10.3 + '@types/node': 16.11.12 '@types/ws': 7.4.7 '@typescript-eslint/eslint-plugin': 5.4.0_87dbf04088b125598d0271706532eaf3 '@typescript-eslint/parser': 5.4.0_eslint@7.32.0+typescript@4.4.3 @@ -10592,7 +10565,7 @@ packages: '@types/express-fileupload': 1.1.7 '@types/heft-jest': 1.0.2 '@types/minio': 7.0.10 - '@types/node': 16.10.3 + '@types/node': 16.11.12 '@types/uuid': 8.3.1 '@typescript-eslint/eslint-plugin': 5.4.0_87dbf04088b125598d0271706532eaf3 '@typescript-eslint/parser': 5.4.0_eslint@7.32.0+typescript@4.4.3 @@ -10615,7 +10588,7 @@ packages: dev: false file:projects/generator.tgz: - resolution: {integrity: sha512-n46hjl25xFASqC9A+jFGBgQNshHS/86Q4a07NdpR7jn7rdcpHWn+3mpndvyplwyOVfdhbbP/AGmasHJhWtEsxg==, tarball: file:projects/generator.tgz} + resolution: {integrity: sha512-vufLNo3Nd5EZMfZnpDyCgzmtiOxuuOCfdcez1VUXR8EJV+07YGGLKpYcUsJ+hp7vrOMe5/iW0gGun4p42SEL9Q==, tarball: file:projects/generator.tgz} name: '@rush-temp/generator' version: 0.0.0 dependencies: @@ -10804,7 +10777,7 @@ packages: dependencies: '@rushstack/heft': 0.41.1 '@types/heft-jest': 1.0.2 - '@types/node': 16.10.3 + '@types/node': 16.11.12 '@typescript-eslint/eslint-plugin': 5.4.0_87dbf04088b125598d0271706532eaf3 '@typescript-eslint/parser': 5.4.0_eslint@7.32.0+typescript@4.4.3 eslint: 7.32.0 @@ -10813,7 +10786,7 @@ packages: eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 5.1.1_eslint@7.32.0 prettier: 2.4.1 - ts-node: 10.2.1_8304ecd715830f7c190b4d1dea90b100 + ts-node: 10.2.1_c2efd757c6d07d33b05f123839e1b1a4 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' @@ -11093,7 +11066,7 @@ packages: dev: false file:projects/model-task.tgz_typescript@4.4.3: - resolution: {integrity: sha512-WzEdSzLqN2Fj+TXMdy9pLEsBPMjqlz6eT2nQLwwQgRyPcT6S6qAbEgqTiPZmk10vW2/qA5X6DDAf82yC/8zVJA==, tarball: file:projects/model-task.tgz} + resolution: {integrity: sha512-8CPB0TE05gwOb4WKefT6zaL/jIgiwfqBM46riENkiR6TJgy1oiACqoS1p7YA+XzZJSy0eAaR3308lGohMNNGzQ==, tarball: file:projects/model-task.tgz} id: file:projects/model-task.tgz name: '@rush-temp/model-task' version: 0.0.0 @@ -11156,7 +11129,7 @@ packages: dev: false file:projects/model-workbench.tgz_typescript@4.4.3: - resolution: {integrity: sha512-BBgFzmrL/3h+HI8NnMkFXjL84t27f8yOF87D4LdeSHlF/sUC20IVdLzwk/6wn/6ayGlPWbQF/54Y5V2X4NtsJQ==, tarball: file:projects/model-workbench.tgz} + resolution: {integrity: sha512-iQ3Un4SlIh2PhEenOxWjXpRfWE3TrSRTtnBI3j2ar5kllG2vB58LLny9urWYpxVI2yRqyrquowtQBj/ZO+vTAA==, tarball: file:projects/model-workbench.tgz} id: file:projects/model-workbench.tgz name: '@rush-temp/model-workbench' version: 0.0.0 @@ -11440,7 +11413,7 @@ packages: dev: false file:projects/recruit-resources.tgz_476f694f64637160ae71e12ff57815b9: - resolution: {integrity: sha512-V7iAy6fX/2McQ0gZjvzHL4qA4ETElxXHEBh6ZnXBuYE8ht0sqLIXSGdgJstxyR++MWTe5ZM7l4GS/re3zYGdnw==, tarball: file:projects/recruit-resources.tgz} + resolution: {integrity: sha512-+AEmp4tNJEHvQatbY5fN1niGYpaIyWMPXbo0QRhUDELdEkm0nhevUfw2Cf44Yj0DroXL+S0YgnXcEI7AtV2QOQ==, tarball: file:projects/recruit-resources.tgz} id: file:projects/recruit-resources.tgz name: '@rush-temp/recruit-resources' version: 0.0.0 @@ -11477,7 +11450,7 @@ packages: dev: false file:projects/recruit.tgz: - resolution: {integrity: sha512-tYzfXei8i5vr9puawxtiFBtY1xE6jzFUpEykNwvoGC7VfkrCQ+HvFqfEIF0sD24tGVxSogQ9u+LkEu+ePl32Jw==, tarball: file:projects/recruit.tgz} + resolution: {integrity: sha512-nqrkga8ccMV8sqQ6O4ta9qEdvHkgEChdViELXUXViGldrhz6qi9UAE1aDuVYHgEFRX+eC+OACSME0YpHbibylw==, tarball: file:projects/recruit.tgz} name: '@rush-temp/recruit' version: 0.0.0 dependencies: @@ -11700,7 +11673,7 @@ packages: dev: false file:projects/setting-resources.tgz_476f694f64637160ae71e12ff57815b9: - resolution: {integrity: sha512-I85L1AQz7schJl8/PKyTjw+/svzxhQ9xnlHSQQnMffB1cSQ265P1v7Zt+NERnsqLGD/l3c59nqXep8JrSDf/Tw==, tarball: file:projects/setting-resources.tgz} + resolution: {integrity: sha512-Ybi5GrbbI/5SMBIe5W9Ub8cwA3wIKPCj98lSJ+Z9YCokpjYn2dGhRJ5LeBKBPu8LDWPfxlrU2Yi17YcHu0xufw==, tarball: file:projects/setting-resources.tgz} id: file:projects/setting-resources.tgz name: '@rush-temp/setting-resources' version: 0.0.0 @@ -11822,7 +11795,7 @@ packages: dependencies: '@rushstack/heft': 0.41.1 '@types/heft-jest': 1.0.2 - '@types/node': 16.10.3 + '@types/node': 16.11.12 '@typescript-eslint/eslint-plugin': 5.4.0_87dbf04088b125598d0271706532eaf3 '@typescript-eslint/parser': 5.4.0_eslint@7.32.0+typescript@4.4.3 eslint: 7.32.0 @@ -11968,7 +11941,7 @@ packages: dev: false file:projects/tool.tgz: - resolution: {integrity: sha512-TNUt6NCYHOiiDDPuhdNtvqtdvWb8N2xFRQAu+Ku3QyKgKH4dMpzV2WNSDnURNSUtN3mCE2OUyiGOLej8B9lbXA==, tarball: file:projects/tool.tgz} + resolution: {integrity: sha512-Ymz2K4mDdajWonLpamGYn7fTN7OqKKjW6SWBB0iZzBxb9BtZOpIY3IVaV9Bm3PYDpm7WOgJ82EjwZbO9+nczFQ==, tarball: file:projects/tool.tgz} name: '@rush-temp/tool' version: 0.0.0 dependencies: @@ -11987,6 +11960,7 @@ packages: eslint-plugin-import: 2.25.3_eslint@7.32.0 eslint-plugin-node: 11.1.0_eslint@7.32.0 eslint-plugin-promise: 5.1.1_eslint@7.32.0 + fast-equals: 2.0.4 jwt-simple: 0.5.6 minio: 7.0.19 mongodb: 4.1.3 @@ -12105,7 +12079,7 @@ packages: dev: false file:projects/workbench-resources.tgz_476f694f64637160ae71e12ff57815b9: - resolution: {integrity: sha512-HKzFdAy5VwIlv4gJv9iN8FjsuvYCp98kvZ0uTAQtK0aavp5lve4rKPGehK+WHUmm7H/IiPD8yXQWEPherq3eZQ==, tarball: file:projects/workbench-resources.tgz} + resolution: {integrity: sha512-yWHOAPBVj1y9VbkYHOaANIkCbBfQOkKKvxj2dEFK3It4WUN/jxZSPqrmW1/ZNkENZVaWzRL1VY6uGhJJ5dwgFQ==, tarball: file:projects/workbench-resources.tgz} id: file:projects/workbench-resources.tgz name: '@rush-temp/workbench-resources' version: 0.0.0 diff --git a/dev/tool/package.json b/dev/tool/package.json index 7e890b93f6..6a3c75460a 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -51,6 +51,7 @@ "ws": "^8.2.0", "@anticrm/client": "~0.6.1", "@anticrm/platform": "~0.6.5", - "@anticrm/model": "~0.6.0" + "@anticrm/model": "~0.6.0", + "fast-equals": "^2.0.3" } } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index cb099235df..68b51e335b 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -24,7 +24,7 @@ import { Client } from 'minio' import { Db, MongoClient } from 'mongodb' import { connect } from './connect' import { clearTelegramHistory } from './telegram' -import { dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace' +import { diffWorkspace, dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace' const mongodbUri = process.env.MONGO_URL if (mongodbUri === undefined) { @@ -195,6 +195,13 @@ program return await restoreWorkspace(mongodbUri, workspace, dirName, minio) }) +program + .command('diff-workspace ') + .description('restore workspace transactions and minio resources from previous dump.') + .action(async (workspace, cmd) => { + return await diffWorkspace(mongodbUri, workspace) + }) + program .command('clear-telegram-history') .description('clear telegram history') diff --git a/dev/tool/src/mdiff.ts b/dev/tool/src/mdiff.ts new file mode 100644 index 0000000000..2e40444056 --- /dev/null +++ b/dev/tool/src/mdiff.ts @@ -0,0 +1,157 @@ +import core, { Attribute, Data, Doc, DocumentUpdate, Hierarchy, ModelDb, Ref, Tx, Type } from '@anticrm/core' +import { deepEqual } from 'fast-equals' + +/** + * @public + */ +export async function buildModel (existingTxes: Tx[]): Promise<{ hierarchy: Hierarchy, model: ModelDb, dropTx: Tx[] }> { + existingTxes = existingTxes.filter((tx) => tx.modifiedBy === core.account.System) + const dropTx: Tx[] = [] + const hierarchy = new Hierarchy() + const model = new ModelDb(hierarchy) + // Construct existing model + existingTxes.forEach(hierarchy.tx.bind(hierarchy)) + for (const tx of existingTxes) { + await applyTx(model, tx, dropTx) + } + return { hierarchy, model, dropTx } +} + +async function applyTx (model: ModelDb, tx: Tx, dropTx: Tx[]): Promise { + try { + await model.tx(tx) + } catch (err: any) { + dropTx.push(tx) + console.info('Found issue during processing of tx. Transaction', tx, 'is dropped...') + } +} + +function toUndef (value: any): any { + return value === null ? undefined : value +} + +function diffAttributes (doc: Data, newDoc: Data): DocumentUpdate { + const result: DocumentUpdate = {} + const allDocuments = new Map(Object.entries(doc)) + const newDocuments = new Map(Object.entries(newDoc)) + + for (const [key, value] of allDocuments) { + const newValue = toUndef(newDocuments.get(key)) + if (!deepEqual(newValue, toUndef(value))) { + // update is required, since values are different + result[key] = newValue + } + } + for (const [key, value] of newDocuments) { + const oldValue = toUndef(allDocuments.get(key)) + if (oldValue === undefined && value !== undefined) { + // Update with new value. + result[key] = value + } + } + return result +} + +/** + * Generate a set of transactions to upgrade from one model to another. + * @public + */ +export async function generateModelDiff (existingTxes: Tx[], txes: Tx[]): Promise<{ diffTx: Op[], dropTx: Tx[] }> { + const { model, dropTx } = await buildModel(existingTxes) + const { model: newModel } = await buildModel(txes) + + const diffTx = generateDocumentDiff( + await model.findAll(core.class.Doc, {}), + await newModel.findAll(core.class.Doc, {}) + ) + return { diffTx, dropTx } +} + +export type Op = Record +/** + * @public + */ +export function generateDocumentDiff (oldDocs: Doc[], newDocs: Doc[]): Op[] { + const diffTx: Op[] = [] + + const allDocuments = new Map(oldDocs.map((d) => [getId(d), d])) + const newDocuments = new Map(newDocs.map((d) => [getId(d), d])) + + // Find same documents. + allDocuments.forEach(handleUpdateRemove(newDocuments, diffTx)) + newDocuments.forEach(handleAdd(allDocuments, diffTx)) + return diffTx +} + +function getId (d: Doc): Ref { + // We need to update Attribute IDS + if (d._class === core.class.Attribute) { + const attr = d as Attribute> + return (attr.attributeOf + '.' + attr.name) as Ref + } else if (d._class === 'view:class:Viewlet' as Ref) { + const cr = d as any + return ((cr.attachTo as string) + '.' + (cr.open as string)) as Ref + } else if (d._class === 'workbench:class:Application' as Ref) { + const cr = d as any + return ('workbench.app.' + (cr.label as string)) as Ref + } else if (d._class === 'view:class:ActionTarget' as Ref) { + const cr = d as any + return ((cr.target as string) + '.' + (cr.action as string)) as Ref + } else if (d._class === 'server-core:class:Trigger' as Ref) { + const cr = d as any + return ((cr.trigger as string)) as Ref + } + return d._id +} + +function handleAdd (allDocuments: Map, Doc>, newTxes: Op[]): (value: Doc, key: Ref) => void { + return (doc, key) => { + if (!allDocuments.has(key)) { + // Add is required + const { _id, _class, modifiedBy, modifiedOn, space, ...data } = doc + const tx: Op = { + _class: 'create-doc', + objectId: _id, + objectClass: doc._class, + attributes: data + } + newTxes.push(tx) + } + } +} + +function handleUpdateRemove (newDocuments: Map, Doc>, newTxes: Op[]): (value: Doc, key: Ref) => void { + return (doc, key) => { + const newDoc = newDocuments.get(key) + if (newDoc !== undefined) { + // update is required. + const { _id, _class, modifiedBy, modifiedOn, space, ...data } = newDoc + const { _id: _0, _class: _1, modifiedBy: _2, modifiedOn: _3, space: _4, ...oldData } = doc + const operations = diffAttributes(oldData, data) + if (Object.keys(operations).length > 0) { + const tx: Op = { + _class: 'update-doc', + objectId: _id, + objectClass: _class, + operations + } + newTxes.push(tx) + } + } else { + // Delete is required + const { _id: oldId, _class: _1, modifiedBy: _2, modifiedOn: _3, space: _4, ...oldData } = doc + const tx: Op = { + _class: 'remove-doc', + objectId: oldId, + objectClass: doc._class, + data: oldData + } + newTxes.push(tx) + } + } +} + +export function printDiff (diffTx: Op[]): void { + // Collect Classes. + console.log('Diff Transactions', JSON.stringify(diffTx, undefined, 2)) +} diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index 841d53e00e..7ab58c2890 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -24,6 +24,7 @@ import { Document, MongoClient } from 'mongodb' import { join } from 'path' import { connect } from './connect' import { MigrateClientImpl } from './upgrade' +import { generateModelDiff, printDiff } from './mdiff' const txes = JSON.parse(JSON.stringify(builder.getTxes())) as Tx[] @@ -245,3 +246,40 @@ export async function restoreWorkspace (mongoUrl: string, dbName: string, fileNa await client.close() } } + +export async function diffWorkspace (mongoUrl: string, dbName: string): Promise { + const client = new MongoClient(mongoUrl) + try { + await client.connect() + const db = client.db(dbName) + + console.log('diffing transactions...') + + const currentModel = await db.collection(DOMAIN_TX).find({ + objectSpace: core.space.Model, + modifiedBy: core.account.System, + objectClass: { $ne: contact.class.EmployeeAccount } + }).toArray() + + const txes = builder.getTxes().filter(tx => { + return tx.objectSpace === core.space.Model && + tx.modifiedBy === core.account.System && + (tx as any).objectClass !== contact.class.EmployeeAccount + }) + + const { diffTx, dropTx } = await generateModelDiff(currentModel, txes) + if (diffTx.length > 0) { + console.log('DIFF Transactions:') + + printDiff(diffTx) + } + if (dropTx.length > 0) { + console.log('Broken Transactions:') + for (const tx of dropTx) { + console.log(JSON.stringify(tx, undefined, 2)) + } + } + } finally { + await client.close() + } +}