From 27e46ef3aeb6abc8459d067e7a6824a67d8a4e5d Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Thu, 14 Mar 2024 12:24:25 +0700 Subject: [PATCH] UBERF-5986: Upgrade fixes (#4957) Signed-off-by: Andrey Sobolev --- common/scripts/docker_tag_push.sh | 4 ++ dev/tool/src/index.ts | 40 ++++++++--- models/server-activity/src/migration.ts | 50 +++++++++++--- models/task/src/migration.ts | 67 ++++++++++++++++++- .../src/components/ClassAttributeRow.svelte | 6 +- server/account/src/index.ts | 23 ++++++- server/tool/src/upgrade.ts | 4 +- 7 files changed, 167 insertions(+), 27 deletions(-) create mode 100755 common/scripts/docker_tag_push.sh diff --git a/common/scripts/docker_tag_push.sh b/common/scripts/docker_tag_push.sh new file mode 100755 index 0000000000..8b7bc94134 --- /dev/null +++ b/common/scripts/docker_tag_push.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "Tagging release $1 with version $2" +docker tag "$1" "$1:$2" +docker push "$1:$2" \ No newline at end of file diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 5383207b19..e2c296a8d4 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -26,10 +26,13 @@ import { getWorkspaceById, listAccounts, listWorkspaces, + listWorkspacesRaw, replacePassword, setAccountAdmin, setRole, + updateWorkspace, upgradeWorkspace, + type Workspace, type WorkspaceInfo } from '@hcengineering/account' import { setMetadata } from '@hcengineering/platform' @@ -43,12 +46,12 @@ import { import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' -import { type Command, program } from 'commander' -import { type Db, MongoClient } from 'mongodb' +import { program, type Command } from 'commander' +import { MongoClient, type Db } from 'mongodb' import { clearTelegramHistory } from './telegram' import { diffWorkspace, updateField } from './workspace' -import { type Data, getWorkspaceId, RateLimiter, type Tx, type Version, type AccountRole } from '@hcengineering/core' +import { RateLimiter, getWorkspaceId, type AccountRole, type Data, type Tx, type Version } from '@hcengineering/core' import { type MinioService } from '@hcengineering/minio' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import { openAIConfigDefaults } from '@hcengineering/openai' @@ -273,10 +276,27 @@ export function devTool ( .action(async (cmd: { parallel: string, logs: string, retry: string, force: boolean, console: boolean }) => { const { mongodbUri, version, txes, migrateOperations } = prepareTools() await withDatabase(mongodbUri, async (db) => { - const workspaces = await listWorkspaces(db, productId) + const workspaces = await listWorkspacesRaw(db, productId) + + // We need to update workspaces with missing workspaceUrl + for (const ws of workspaces) { + if (ws.workspaceUrl == null) { + const upd: Partial = { + workspaceUrl: ws.workspace + } + if (ws.workspaceName == null) { + upd.workspaceName = ws.workspace + } + await updateWorkspace(db, productId, ws, upd) + } + } + const withError: string[] = [] async function _upgradeWorkspace (ws: WorkspaceInfo): Promise { + if (ws.disabled === true) { + return + } const t = Date.now() const logger = cmd.console ? consoleModelLogger @@ -308,11 +328,13 @@ export function devTool ( const parallel = parseInt(cmd.parallel) ?? 1 const rateLimit = new RateLimiter(parallel) console.log('parallel upgrade', parallel, cmd.parallel) - for (const ws of workspaces) { - await rateLimit.exec(() => { - return _upgradeWorkspace(ws) - }) - } + await Promise.all( + workspaces.map((it) => + rateLimit.add(() => { + return _upgradeWorkspace(it) + }) + ) + ) } else { console.log('UPGRADE write logs at:', cmd.logs) for (const ws of workspaces) { diff --git a/models/server-activity/src/migration.ts b/models/server-activity/src/migration.ts index 5a9c729c5d..ef5474472a 100644 --- a/models/server-activity/src/migration.ts +++ b/models/server-activity/src/migration.ts @@ -25,6 +25,7 @@ import core, { type Class, type Doc, type Ref, + type Tx, type TxCUD, type TxCollectionCUD, type TxCreateDoc @@ -55,15 +56,18 @@ async function generateDocUpdateMessageByTx ( tx: TxCUD, control: ActivityControl, client: MigrationClient, - objectCache?: DocObjectCache + objectCache?: DocObjectCache, + existsMap?: Set> ): Promise { - const existsMessages = await client.find( - DOMAIN_ACTIVITY, - { _class: activity.class.DocUpdateMessage, txId: tx._id }, - { projection: { _id: 1 } } - ) + const existsMessages = + existsMap?.has(tx._id) ?? + (await client.find( + DOMAIN_ACTIVITY, + { _class: activity.class.DocUpdateMessage, txId: tx._id }, + { projection: { _id: 1 } } + )) - if (existsMessages.length > 0) { + if (existsMessages === true || (Array.isArray(existsMessages) && existsMessages.length > 0)) { return } @@ -172,6 +176,33 @@ async function createDocUpdateMessages (client: MigrationClient): Promise } } + const docCache = { + docs: docIds, + transactions: allTransactions + } + const txIds = new Set>() + for (const d of docs) { + processed += 1 + if (processed % 1000 === 0) { + console.log('processed', processed) + } + const transactions = allTransactions.get(d._id) ?? [] + for (const tx of transactions) { + const innerTx = TxProcessor.extractTx(tx) as TxCUD + txIds.add(innerTx._id) + } + } + + const ids = ( + await client.find( + DOMAIN_ACTIVITY, + { _class: activity.class.DocUpdateMessage, txId: { $in: Array.from(txIds) as Ref>[] } }, + { projection: { _id: 1, txId: 1 } } + ) + ).map((p) => p.txId as Ref) + + const existsMessages = new Set(ids) + for (const d of docs) { processed += 1 if (processed % 1000 === 0) { @@ -192,10 +223,7 @@ async function createDocUpdateMessages (client: MigrationClient): Promise } try { - await generateDocUpdateMessageByTx(tx, notificationControl, client, { - docs: docIds, - transactions: allTransactions - }) + await generateDocUpdateMessageByTx(tx, notificationControl, client, docCache, existsMessages) } catch (e: any) { console.error('error processing:', d._id, e.stack) } diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts index 8a66383a51..a95bfd7e2c 100644 --- a/models/task/src/migration.ts +++ b/models/task/src/migration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { TxOperations, type Class, type Doc, type Ref, toIdMap } from '@hcengineering/core' +import { ClassifierKind, TxOperations, toIdMap, type Class, type Doc, type Ref } from '@hcengineering/core' import { createOrUpdate, tryMigrate, @@ -24,9 +24,10 @@ import { } from '@hcengineering/model' import core, { DOMAIN_SPACE } from '@hcengineering/model-core' import tags from '@hcengineering/model-tags' -import { type TaskType, taskId } from '@hcengineering/task' -import task from './plugin' +import { getEmbeddedLabel } from '@hcengineering/platform' +import { taskId, type TaskType } from '@hcengineering/task' import { DOMAIN_TASK } from '.' +import task from './plugin' /** * @public @@ -110,6 +111,62 @@ async function createDefaults (tx: TxOperations): Promise { await createDefaultStatesSpace(tx) } +async function fixProjectTypeMissingClass (client: MigrationUpgradeClient): Promise { + const projectTypes = await client.findAll(task.class.ProjectType, {}) + const ops = new TxOperations(client, core.account.ConfigUser) + + const h = ops.getHierarchy() + for (const pt of projectTypes) { + console.log('Checking:', pt.name) + try { + if (!h.hasClass(pt.targetClass)) { + const categoryObj = ops.getModel().findObject(pt.descriptor) + if (categoryObj === undefined) { + throw new Error('category is not found in model') + } + const baseClassClass = h.getClass(categoryObj.baseClass) + await ops.createDoc( + core.class.Class, + core.space.Model, + { + extends: categoryObj.baseClass, + kind: ClassifierKind.MIXIN, + label: baseClassClass.label, + icon: baseClassClass.icon + }, + pt.targetClass, + Date.now(), + core.account.ConfigUser + ) + } + + const taskTypes = await ops.findAll(task.class.TaskType, { parent: pt._id }) + + for (const tt of taskTypes) { + if (!h.hasClass(tt.targetClass)) { + const ofClassClass = h.getClass(tt.ofClass) + await ops.createDoc( + core.class.Class, + core.space.Model, + { + extends: tt.ofClass, + kind: ClassifierKind.MIXIN, + label: getEmbeddedLabel(tt.name), + icon: ofClassClass.icon + }, + pt.targetClass, + Date.now(), + core.account.ConfigUser + ) + } + } + } catch (err: any) { + // + console.error(err) + } + } +} + export const taskOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, taskId, [ @@ -167,6 +224,10 @@ export const taskOperation: MigrateOperation = { { state: 'reorderStates', func: reorderStates + }, + { + state: 'fix-project-type-missing-class', + func: fixProjectTypeMissingClass } ]) } diff --git a/plugins/setting-resources/src/components/ClassAttributeRow.svelte b/plugins/setting-resources/src/components/ClassAttributeRow.svelte index 98ca5f1713..a976d92ed3 100644 --- a/plugins/setting-resources/src/components/ClassAttributeRow.svelte +++ b/plugins/setting-resources/src/components/ClassAttributeRow.svelte @@ -42,8 +42,10 @@ } function getArrayName (type: Type): IntlString | undefined { const ref = (type as ArrOf).of - const res = client.getHierarchy().getClass((ref as RefTo).to) - return res?.label + if (client.getHierarchy().hasClass((ref as RefTo).to)) { + const res = client.getHierarchy().getClass((ref as RefTo).to) + return res?.label + } } diff --git a/server/account/src/index.ts b/server/account/src/index.ts index 26a8d7ca05..ca29a311f1 100644 --- a/server/account/src/index.ts +++ b/server/account/src/index.ts @@ -604,6 +604,27 @@ export async function listWorkspaces (db: Db, productId: string): Promise { + return (await db.collection(WORKSPACE_COLLECTION).find(withProductId(productId, {})).toArray()).map( + (it) => ({ ...it, productId }) + ) +} + +/** + * @public + */ +export async function updateWorkspace ( + db: Db, + productId: string, + info: Workspace, + ops: Partial +): Promise { + await db.collection(WORKSPACE_COLLECTION).updateOne({ _id: info._id }, { $set: { ...info, ...ops } }) +} + /** * @public */ @@ -772,7 +793,7 @@ export async function upgradeWorkspace ( const versionStr = versionToString(version) console.log( - `${forceUpdate ? 'force-' : ''}upgrade from "${ + `${workspaceUrl} - ${forceUpdate ? 'force-' : ''}upgrade from "${ ws?.version !== undefined ? versionToString(ws.version) : '' }" to "${versionStr}"` ) diff --git a/server/tool/src/upgrade.ts b/server/tool/src/upgrade.ts index 047697c9d0..dbb3dd82c1 100644 --- a/server/tool/src/upgrade.ts +++ b/server/tool/src/upgrade.ts @@ -183,7 +183,9 @@ export class MigrateClientImpl implements MigrationClient { async create(domain: Domain, doc: T | T[]): Promise { if (Array.isArray(doc)) { - await this.db.collection(domain).insertMany(doc as Document[]) + if (doc.length > 0) { + await this.db.collection(domain).insertMany(doc as Document[]) + } } else { await this.db.collection(domain).insertOne(doc as Document) }