From 15f6f5aabb6879a612d3744545085d5078118563 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Wed, 8 Dec 2021 01:45:11 +0700 Subject: [PATCH] Allow model upgrades (#561) Signed-off-by: Andrey Sobolev --- dev/docker-compose.yaml | 3 +- dev/tool/package.json | 3 +- dev/tool/src/connect.ts | 12 +-- dev/tool/src/index.ts | 26 ++++-- dev/tool/src/upgrade.ts | 72 ++++++++++++++++ dev/tool/src/workspace.ts | 129 ++++++++++++++++++++++++----- models/all/package.json | 4 +- models/all/src/index.ts | 3 + models/all/src/migration.ts | 22 +++++ models/attachment/src/index.ts | 2 + models/attachment/src/migration.ts | 45 ++++++++++ models/task/src/index.ts | 2 + models/task/src/migration.ts | 55 ++++++++++++ packages/model/src/index.ts | 1 + packages/model/src/migration.ts | 46 ++++++++++ 15 files changed, 383 insertions(+), 42 deletions(-) create mode 100644 dev/tool/src/upgrade.ts create mode 100644 models/all/src/migration.ts create mode 100644 models/attachment/src/migration.ts create mode 100644 models/task/src/migration.ts create mode 100644 packages/model/src/migration.ts diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 26ad2eb6f6..ace5b19e73 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -45,8 +45,7 @@ services: - mongodb - minio - elastic - - server - - upload + - transactor ports: - 8081:8080 environment: diff --git a/dev/tool/package.json b/dev/tool/package.json index b224256e0d..b99b7aea50 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -49,6 +49,7 @@ "@anticrm/client-resources": "~0.6.4", "ws": "^8.2.0", "@anticrm/client": "~0.6.1", - "@anticrm/platform": "~0.6.5" + "@anticrm/platform": "~0.6.5", + "@anticrm/model": "~0.6.0" } } diff --git a/dev/tool/src/connect.ts b/dev/tool/src/connect.ts index ae267fb163..bba42f2067 100644 --- a/dev/tool/src/connect.ts +++ b/dev/tool/src/connect.ts @@ -1,24 +1,18 @@ import client from '@anticrm/client' import clientResources from '@anticrm/client-resources' -import core, { TxOperations } from '@anticrm/core' +import { Client } from '@anticrm/core' import { setMetadata } from '@anticrm/platform' import { encode } from 'jwt-simple' // eslint-disable-next-line const WebSocket = require('ws') -export async function connect (transactorUrl: string, workspace: string): Promise<{ connection: TxOperations, close: () => Promise}> { +export async function connect (transactorUrl: string, workspace: string): Promise { console.log('connecting to transactor...') const token = encode({ email: 'anticrm@hc.engineering', workspace }, 'secret') // We need to override default factory with 'ws' one. setMetadata(client.metadata.ClientSocketFactory, (url) => new WebSocket(url)) - const connection = await (await clientResources()).function.GetClient(token, transactorUrl) - return { - connection: new TxOperations(connection, core.account.System), - close: async () => { - await connection.close() - } - } + return await (await clientResources()).function.GetClient(token, transactorUrl) } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 362e9f28e1..cb099235df 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -18,13 +18,13 @@ import { ACCOUNT_DB, assignWorkspace, createAccount, createWorkspace, dropAccount, dropWorkspace, getAccount, listWorkspaces } from '@anticrm/account' import contact, { combineName } from '@anticrm/contact' -import core from '@anticrm/core' +import core, { TxOperations } from '@anticrm/core' import { program } from 'commander' import { Client } from 'minio' import { Db, MongoClient } from 'mongodb' import { connect } from './connect' import { clearTelegramHistory } from './telegram' -import { dumpWorkspace, initWorkspace, upgradeWorkspace } from './workspace' +import { dumpWorkspace, initWorkspace, restoreWorkspace, upgradeWorkspace } from './workspace' const mongodbUri = process.env.MONGO_URL if (mongodbUri === undefined) { @@ -103,24 +103,25 @@ program await assignWorkspace(db, email, workspace) console.log('connecting to transactor...') - const { connection, close } = await connect(transactorUrl, workspace) + const connection = await connect(transactorUrl, workspace) + const ops = new TxOperations(connection, core.account.System) const name = combineName(account.first, account.last) console.log('create user in target workspace...') - const employee = await connection.createDoc(contact.class.Employee, contact.space.Employee, { + const employee = await ops.createDoc(contact.class.Employee, contact.space.Employee, { name, city: 'Mountain View', channels: [] }) console.log('create account in target workspace...') - await connection.createDoc(contact.class.EmployeeAccount, core.space.Model, { + await ops.createDoc(contact.class.EmployeeAccount, core.space.Model, { email, employee, name }) - await close() + await connection.close() }) }) @@ -181,10 +182,17 @@ program }) program - .command('dump-workspace ') + .command('dump-workspace ') .description('dump workspace transactions and minio resources') - .action(async (workspace, fileName, cmd) => { - return await dumpWorkspace(mongodbUri, workspace, fileName, minio) + .action(async (workspace, dirName, cmd) => { + return await dumpWorkspace(mongodbUri, workspace, dirName, minio) + }) + +program + .command('restore-workspace ') + .description('restore workspace transactions and minio resources from previous dump.') + .action(async (workspace, dirName, cmd) => { + return await restoreWorkspace(mongodbUri, workspace, dirName, minio) }) program diff --git a/dev/tool/src/upgrade.ts b/dev/tool/src/upgrade.ts new file mode 100644 index 0000000000..3e526d991e --- /dev/null +++ b/dev/tool/src/upgrade.ts @@ -0,0 +1,72 @@ +import { + Doc, + DocumentQuery, + Domain, + FindOptions, + isOperator, SortingOrder +} from '@anticrm/core' +import { MigrationClient, MigrateUpdate, MigrationResult } from '@anticrm/model' +import { Db, Document, Filter, Sort, UpdateFilter } from 'mongodb' + +/** + * Upgrade client implementation. + */ +export class MigrateClientImpl implements MigrationClient { + constructor (readonly db: Db) { + } + + private translateQuery(query: DocumentQuery): Filter { + const translated: any = {} + for (const key in query) { + const value = (query as any)[key] + if (value !== null && typeof value === 'object') { + const keys = Object.keys(value) + if (keys[0] === '$like') { + const pattern = value.$like as string + translated[key] = { + $regex: `^${pattern.split('%').join('.*')}$`, + $options: 'i' + } + continue + } + } + translated[key] = value + } + return translated + } + + async find( + domain: Domain, + query: DocumentQuery, + options?: FindOptions | undefined + ): Promise { + let cursor = this.db.collection(domain).find(this.translateQuery(query)) + if (options?.limit !== undefined) { + cursor = cursor.limit(options.limit) + } + if (options !== null && options !== undefined) { + if (options.sort !== undefined) { + const sort: Sort = {} + for (const key in options.sort) { + const order = options.sort[key] === SortingOrder.Ascending ? 1 : -1 + sort[key] = order + } + cursor = cursor.sort(sort) + } + } + return await cursor.toArray() + } + + async update(domain: Domain, query: DocumentQuery, operations: MigrateUpdate): Promise { + if (isOperator(operations)) { + const result = await this.db + .collection(domain) + .updateMany(this.translateQuery(query), { ...operations } as unknown as UpdateFilter) + + return { matched: result.matchedCount, updated: result.modifiedCount } + } else { + const result = await this.db.collection(domain).updateMany(this.translateQuery(query), { $set: operations }) + return { matched: result.matchedCount, updated: result.modifiedCount } + } + } +} diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index e79f07e207..af70bd4c6d 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -16,13 +16,14 @@ import contact from '@anticrm/contact' import core, { DOMAIN_TX, Tx } from '@anticrm/core' -import builder from '@anticrm/model-all' +import builder, { migrateOperations } from '@anticrm/model-all' import { existsSync } from 'fs' -import { mkdir, writeFile } from 'fs/promises' -import { BucketItem, Client } from 'minio' +import { mkdir, readFile, writeFile } from 'fs/promises' +import { BucketItem, Client, ItemBucketMetadata } from 'minio' import { Document, MongoClient } from 'mongodb' import { join } from 'path' import { connect } from './connect' +import { MigrateClientImpl } from './upgrade' const txes = JSON.parse(JSON.stringify(builder.getTxes())) as Tx[] @@ -46,11 +47,11 @@ export async function initWorkspace (mongoUrl: string, dbName: string, transacto console.log('creating data...') const data = txes.filter((tx) => tx.objectSpace !== core.space.Model) - const { connection, close } = await connect(transactorUrl, dbName) + const connection = await connect(transactorUrl, dbName) for (const tx of data) { await connection.tx(tx) } - await close() + await connection.close() console.log('create minio bucket') if (!(await minio.bucketExists(dbName))) { @@ -88,11 +89,38 @@ export async function upgradeWorkspace ( const model = txes.filter((tx) => tx.objectSpace === core.space.Model) const insert = await db.collection(DOMAIN_TX).insertMany(model as Document[]) console.log(`${insert.insertedCount} model transactions inserted.`) + + const migrateClient = new MigrateClientImpl(db) + for (const op of migrateOperations) { + await op.migrate(migrateClient) + } + + console.log('Apply upgrade operations') + + const connection = await connect(transactorUrl, dbName) + for (const op of migrateOperations) { + await op.upgrade(connection) + } + + await connection.close() } finally { await client.close() } } +interface CollectionInfo { + name: string + file: string +} + +type MinioWorkspaceItem = BucketItem & { metaData: ItemBucketMetadata } + +interface WorkspaceInfo { + version: string + collections: CollectionInfo[] + minioData: MinioWorkspaceItem[] +} + /** * @public */ @@ -104,29 +132,34 @@ export async function dumpWorkspace (mongoUrl: string, dbName: string, fileName: console.log('dumping transactions...') - const dbTxes = await db.collection(DOMAIN_TX).find().toArray() - await writeFile(fileName + '.tx.json', JSON.stringify(dbTxes, undefined, 2)) + if (!existsSync(fileName)) { + await mkdir(fileName, { recursive: true }) + } + + const workspaceInfo: WorkspaceInfo = { + version: '0.6.0', + collections: [], + minioData: [] + } + const collections = await db.collections() + for (const c of collections) { + const docs = await c.find().toArray() + workspaceInfo.collections.push({ name: c.collectionName, file: c.collectionName + '.json' }) + await writeFile(fileName + c.collectionName + '.json', JSON.stringify(docs, undefined, 2)) + } console.log('Dump minio objects') if (await minio.bucketExists(dbName)) { - const minioData: BucketItem[] = [] - const list = await minio.listObjects(dbName, undefined, true) - await new Promise((resolve) => { - list.on('data', (data) => { - minioData.push(data) - }) - list.on('end', () => { - resolve(null) - }) - }) + workspaceInfo.minioData.push(...(await listMinioObjects(minio, dbName))) const minioDbLocation = fileName + '.minio' if (!existsSync(minioDbLocation)) { await mkdir(minioDbLocation) } - await writeFile(fileName + '.minio.json', JSON.stringify(minioData, undefined, 2)) - for (const d of minioData) { + for (const d of workspaceInfo.minioData) { + const stat = await minio.statObject(dbName, d.name) const data = await minio.getObject(dbName, d.name) const allData = [] + d.metaData = stat.metaData let chunk while ((chunk = data.read()) !== null) { allData.push(chunk) @@ -134,6 +167,64 @@ export async function dumpWorkspace (mongoUrl: string, dbName: string, fileName: await writeFile(join(minioDbLocation, d.name), allData.join('')) } } + + await writeFile(fileName + '.workspace.json', JSON.stringify(workspaceInfo, undefined, 2)) + } finally { + await client.close() + } +} + +async function listMinioObjects (minio: Client, dbName: string): Promise { + const items: MinioWorkspaceItem[] = [] + const list = await minio.listObjects(dbName, undefined, true) + await new Promise((resolve) => { + list.on('data', (data) => { + items.push({ ...data, metaData: {} }) + }) + list.on('end', () => { + resolve(null) + }) + }) + return items +} + +export async function restoreWorkspace (mongoUrl: string, dbName: string, fileName: string, minio: Client): Promise { + const client = new MongoClient(mongoUrl) + try { + await client.connect() + const db = client.db(dbName) + + console.log('dumping transactions...') + + const workspaceInfo = JSON.parse((await readFile(fileName + '.workspace.json')).toString()) as WorkspaceInfo + + // Drop existing collections + + const cols = await db.collections() + for (const c of cols) { + await db.dropCollection(c.collectionName) + } + // Restore collections. + for (const c of workspaceInfo.collections) { + const collection = db.collection(c.name) + await collection.deleteMany({}) + const data = JSON.parse((await readFile(fileName + c.name + '.json')).toString()) as Document[] + await collection.insertMany(data) + } + + console.log('Restore minio objects') + if (await minio.bucketExists(dbName)) { + const objectNames = (await listMinioObjects(minio, dbName)).map(i => i.name) + await minio.removeObjects(dbName, objectNames) + await minio.removeBucket(dbName) + } + await minio.makeBucket(dbName, 'k8s') + + const minioDbLocation = fileName + '.minio' + for (const d of workspaceInfo.minioData) { + const data = await readFile(join(minioDbLocation, d.name)) + await minio.putObject(dbName, d.name, data, d.size, d.metaData) + } } finally { await client.close() } diff --git a/models/all/package.json b/models/all/package.json index 3408ec9f16..f54a4f7eaa 100644 --- a/models/all/package.json +++ b/models/all/package.json @@ -45,7 +45,7 @@ "@anticrm/model-server-recruit": "~0.6.0", "@anticrm/model-server-view": "~0.6.0", "@anticrm/model-activity": "~0.6.0", - "@anticrm/model-attachment": "~0.6.0" - + "@anticrm/model-attachment": "~0.6.0", + "@anticrm/core": "~0.6.13" } } diff --git a/models/all/src/index.ts b/models/all/src/index.ts index 48b7220025..d8355f56ad 100644 --- a/models/all/src/index.ts +++ b/models/all/src/index.ts @@ -58,3 +58,6 @@ serverViewModel(builder) createDemo(builder) export default builder + +// Export upgrade procedures +export { migrateOperations } from './migration' diff --git a/models/all/src/migration.ts b/models/all/src/migration.ts new file mode 100644 index 0000000000..67316c969f --- /dev/null +++ b/models/all/src/migration.ts @@ -0,0 +1,22 @@ +// +// Copyright © 2020 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MigrateOperation } from '@anticrm/model' + +// Import migrate operations. +import { taskOperation } from '@anticrm/model-task' +import { attachmentOperation } from '@anticrm/model-attachment' + +export const migrateOperations: MigrateOperation[] = [taskOperation, attachmentOperation] diff --git a/models/attachment/src/index.ts b/models/attachment/src/index.ts index bb1050c4a7..1cfa26f751 100644 --- a/models/attachment/src/index.ts +++ b/models/attachment/src/index.ts @@ -23,6 +23,8 @@ import activity from '@anticrm/activity' import view from '@anticrm/model-view' import attachment from './plugin' +export { attachmentOperation } from './migration' + export const DOMAIN_ATTACHMENT = 'attachment' as Domain @Model(attachment.class.Attachment, core.class.AttachedDoc, DOMAIN_ATTACHMENT) diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts new file mode 100644 index 0000000000..53a5b29a8d --- /dev/null +++ b/models/attachment/src/migration.ts @@ -0,0 +1,45 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Class, Doc, DOMAIN_TX, Ref, TxCUD } from '@anticrm/core' +import { MigrateOperation, MigrationClient, MigrationUpgradeClient } from '@anticrm/model' +import { DOMAIN_ATTACHMENT } from './index' +import attachment from './plugin' + +export const attachmentOperation: MigrateOperation = { + async migrate (client: MigrationClient): Promise { + console.log('Attachments: Upgrading attachments.') + // Replace attachment class + const attachResult = await client.update(DOMAIN_ATTACHMENT, { _class: 'chunter:class:Attachment' as Ref> }, { _class: attachment.class.Attachment as Ref> }) + if (attachResult.updated > 0) { + console.log(`Attachments: Update ${attachResult.updated} Attachment objects`) + } + + // Update transactions. + const txResult = await client.update>(DOMAIN_TX, { objectClass: 'chunter:class:Attachment' as Ref> }, { objectClass: attachment.class.Attachment as Ref> }) + + if (txResult.updated > 0) { + console.log(`Attachments: Update ${txResult.updated} Transactions`) + } + + const txResult2 = await client.update>(DOMAIN_TX, { 'tx.objectClass': 'chunter:class:Attachment' as Ref> }, { 'tx.objectClass': attachment.class.Attachment as Ref> }) + + if (txResult2.updated > 0) { + console.log(`Attachments: Update ${txResult.updated} Transactions`) + } + }, + async upgrade (client: MigrationUpgradeClient): Promise { + } +} diff --git a/models/task/src/index.ts b/models/task/src/index.ts index c789aafb7b..831055d3b0 100644 --- a/models/task/src/index.ts +++ b/models/task/src/index.ts @@ -144,3 +144,5 @@ export function createModel (builder: Builder): void { return await Promise.resolve() }).catch((err) => console.error(err)) } + +export { taskOperation } from './migration' diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts new file mode 100644 index 0000000000..06853b1a93 --- /dev/null +++ b/models/task/src/migration.ts @@ -0,0 +1,55 @@ +// +// Copyright © 2020, 2021 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Doc, TxOperations } from '@anticrm/core' +import { MigrateOperation, MigrationClient, MigrationUpgradeClient } from '@anticrm/model' +import core from '@anticrm/model-core' +import view from '@anticrm/model-view' +import { createProjectKanban } from '@anticrm/task' +import task from './plugin' + +export const taskOperation: MigrateOperation = { + async migrate (client: MigrationClient): Promise { + + }, + async upgrade (client: MigrationUpgradeClient): Promise { + console.log('Task: Performing model upgrades') + + const ops = new TxOperations(client, core.account.System) + if (await client.findOne(view.class.Sequence, { attachedTo: task.class.Task }) === undefined) { + console.info('Create sequence for default task project.') + // We need to create sequence + await ops.createDoc(view.class.Sequence, view.space.Sequence, { + attachedTo: task.class.Task, + sequence: 0 + }) + } else { + console.log('Task: => sequence is ok') + } + if (await client.findOne(view.class.Kanban, { attachedTo: task.space.TasksPublic }) === undefined) { + console.info('Create kanban for default task project.') + await createProjectKanban(task.space.TasksPublic, async (_class, space, data, id) => { + const doc = await ops.findOne(_class, { _id: id }) + if (doc === undefined) { + await ops.createDoc(_class, space, data, id) + } else { + await ops.updateDoc(_class, space, id, data) + } + }).catch((err) => console.error(err)) + } else { + console.log('Task: => public project Kanban is ok') + } + } +} diff --git a/packages/model/src/index.ts b/packages/model/src/index.ts index 1058e8d73a..36418d439b 100644 --- a/packages/model/src/index.ts +++ b/packages/model/src/index.ts @@ -14,3 +14,4 @@ // export * from './dsl' +export * from './migration' diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts new file mode 100644 index 0000000000..2ca6b8669e --- /dev/null +++ b/packages/model/src/migration.ts @@ -0,0 +1,46 @@ +import { Client, Doc, DocumentQuery, Domain, FindOptions, IncOptions, PushOptions } from '@anticrm/core' + +/** + * @public + */ +export type MigrateUpdate = Partial & Omit, '$move'> & IncOptions & { + // For any other mongo stuff + [key: string]: any +} + +/** + * @public + */ +export interface MigrationResult { + matched: number + updated: number +} + +/** + * @public + * Client to perform model upgrades + */ +export interface MigrationClient { + // Raw collection operations + + // Raw FIND, allow to find documents inside domain. + find: (domain: Domain, query: DocumentQuery, options?: Omit, 'lookup'>) => Promise + + // Allow to raw update documents inside domain. + update: (domain: Domain, query: DocumentQuery, operations: MigrateUpdate) => Promise +} + +/** + * @public + */ +export type MigrationUpgradeClient = Client + +/** + * @public + */ +export interface MigrateOperation { + // Perform low level migration + migrate: (client: MigrationClient) => Promise + // Perform high level upgrade operations. + upgrade: (client: MigrationUpgradeClient) => Promise +}