UBERF-5986: Upgrade fixes (#4957)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

common/scripts/docker_tag_push.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "Tagging release $1 with version $2"  
+docker tag "$1" "$1:$2"
+docker push "$1:$2"

dev/tool/src/index.ts

@@ -26,10 +26,13 @@ import {
   getWorkspaceById,
   listAccounts,
   listWorkspaces,
+  listWorkspacesRaw,
   replacePassword,
   setAccountAdmin,
   setRole,
+  updateWorkspace,
   upgradeWorkspace,
+  type Workspace,
   type WorkspaceInfo
 } from '@hcengineering/account'
 import { setMetadata } from '@hcengineering/platform'
@@ -43,12 +46,12 @@ import {
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
 
-import { type Command, program } from 'commander'
+import { program, type Command } from 'commander'
-import { type Db, MongoClient } from 'mongodb'
+import { MongoClient, type Db } from 'mongodb'
 import { clearTelegramHistory } from './telegram'
 import { diffWorkspace, updateField } from './workspace'
 
-import { type Data, getWorkspaceId, RateLimiter, type Tx, type Version, type AccountRole } from '@hcengineering/core'
+import { RateLimiter, getWorkspaceId, type AccountRole, type Data, type Tx, type Version } from '@hcengineering/core'
 import { type MinioService } from '@hcengineering/minio'
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
 import { openAIConfigDefaults } from '@hcengineering/openai'
@@ -273,10 +276,27 @@ export function devTool (
     .action(async (cmd: { parallel: string, logs: string, retry: string, force: boolean, console: boolean }) => {
       const { mongodbUri, version, txes, migrateOperations } = prepareTools()
       await withDatabase(mongodbUri, async (db) => {
-        const workspaces = await listWorkspaces(db, productId)
+        const workspaces = await listWorkspacesRaw(db, productId)
+
+        // We need to update workspaces with missing workspaceUrl
+        for (const ws of workspaces) {
+          if (ws.workspaceUrl == null) {
+            const upd: Partial<Workspace> = {
+              workspaceUrl: ws.workspace
+            }
+            if (ws.workspaceName == null) {
+              upd.workspaceName = ws.workspace
+            }
+            await updateWorkspace(db, productId, ws, upd)
+          }
+        }
+
         const withError: string[] = []
 
         async function _upgradeWorkspace (ws: WorkspaceInfo): Promise<void> {
+          if (ws.disabled === true) {
+            return
+          }
           const t = Date.now()
           const logger = cmd.console
             ? consoleModelLogger
@@ -308,11 +328,13 @@ export function devTool (
           const parallel = parseInt(cmd.parallel) ?? 1
           const rateLimit = new RateLimiter(parallel)
           console.log('parallel upgrade', parallel, cmd.parallel)
-          for (const ws of workspaces) {
+          await Promise.all(
-            await rateLimit.exec(() => {
+            workspaces.map((it) =>
-              return _upgradeWorkspace(ws)
+              rateLimit.add(() => {
+                return _upgradeWorkspace(it)
               })
-          }
+            )
+          )
         } else {
           console.log('UPGRADE write logs at:', cmd.logs)
           for (const ws of workspaces) {

models/server-activity/src/migration.ts

@@ -25,6 +25,7 @@ import core, {
   type Class,
   type Doc,
   type Ref,
+  type Tx,
   type TxCUD,
   type TxCollectionCUD,
   type TxCreateDoc
@@ -55,15 +56,18 @@ async function generateDocUpdateMessageByTx (
   tx: TxCUD<Doc>,
   control: ActivityControl,
   client: MigrationClient,
-  objectCache?: DocObjectCache
+  objectCache?: DocObjectCache,
+  existsMap?: Set<Ref<Tx>>
 ): Promise<void> {
-  const existsMessages = await client.find<DocUpdateMessage>(
+  const existsMessages =
+    existsMap?.has(tx._id) ??
+    (await client.find<DocUpdateMessage>(
       DOMAIN_ACTIVITY,
       { _class: activity.class.DocUpdateMessage, txId: tx._id },
       { projection: { _id: 1 } }
-  )
+    ))
 
-  if (existsMessages.length > 0) {
+  if (existsMessages === true || (Array.isArray(existsMessages) && existsMessages.length > 0)) {
     return
   }
 
@@ -172,6 +176,33 @@ async function createDocUpdateMessages (client: MigrationClient): Promise<void>
         }
       }
 
+      const docCache = {
+        docs: docIds,
+        transactions: allTransactions
+      }
+      const txIds = new Set<Ref<Tx>>()
+      for (const d of docs) {
+        processed += 1
+        if (processed % 1000 === 0) {
+          console.log('processed', processed)
+        }
+        const transactions = allTransactions.get(d._id) ?? []
+        for (const tx of transactions) {
+          const innerTx = TxProcessor.extractTx(tx) as TxCUD<Doc>
+          txIds.add(innerTx._id)
+        }
+      }
+
+      const ids = (
+        await client.find<DocUpdateMessage>(
+          DOMAIN_ACTIVITY,
+          { _class: activity.class.DocUpdateMessage, txId: { $in: Array.from(txIds) as Ref<TxCUD<Doc>>[] } },
+          { projection: { _id: 1, txId: 1 } }
+        )
+      ).map((p) => p.txId as Ref<Tx>)
+
+      const existsMessages = new Set(ids)
+
       for (const d of docs) {
         processed += 1
         if (processed % 1000 === 0) {
@@ -192,10 +223,7 @@ async function createDocUpdateMessages (client: MigrationClient): Promise<void>
           }
 
           try {
-            await generateDocUpdateMessageByTx(tx, notificationControl, client, {
+            await generateDocUpdateMessageByTx(tx, notificationControl, client, docCache, existsMessages)
-              docs: docIds,
-              transactions: allTransactions
-            })
           } catch (e: any) {
             console.error('error processing:', d._id, e.stack)
           }

models/task/src/migration.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 
-import { TxOperations, type Class, type Doc, type Ref, toIdMap } from '@hcengineering/core'
+import { ClassifierKind, TxOperations, toIdMap, type Class, type Doc, type Ref } from '@hcengineering/core'
 import {
   createOrUpdate,
   tryMigrate,
@@ -24,9 +24,10 @@ import {
 } from '@hcengineering/model'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import tags from '@hcengineering/model-tags'
-import { type TaskType, taskId } from '@hcengineering/task'
+import { getEmbeddedLabel } from '@hcengineering/platform'
-import task from './plugin'
+import { taskId, type TaskType } from '@hcengineering/task'
 import { DOMAIN_TASK } from '.'
+import task from './plugin'
 
 /**
  * @public
@@ -110,6 +111,62 @@ async function createDefaults (tx: TxOperations): Promise<void> {
   await createDefaultStatesSpace(tx)
 }
 
+async function fixProjectTypeMissingClass (client: MigrationUpgradeClient): Promise<void> {
+  const projectTypes = await client.findAll(task.class.ProjectType, {})
+  const ops = new TxOperations(client, core.account.ConfigUser)
+
+  const h = ops.getHierarchy()
+  for (const pt of projectTypes) {
+    console.log('Checking:', pt.name)
+    try {
+      if (!h.hasClass(pt.targetClass)) {
+        const categoryObj = ops.getModel().findObject(pt.descriptor)
+        if (categoryObj === undefined) {
+          throw new Error('category is not found in model')
+        }
+        const baseClassClass = h.getClass(categoryObj.baseClass)
+        await ops.createDoc(
+          core.class.Class,
+          core.space.Model,
+          {
+            extends: categoryObj.baseClass,
+            kind: ClassifierKind.MIXIN,
+            label: baseClassClass.label,
+            icon: baseClassClass.icon
+          },
+          pt.targetClass,
+          Date.now(),
+          core.account.ConfigUser
+        )
+      }
+
+      const taskTypes = await ops.findAll(task.class.TaskType, { parent: pt._id })
+
+      for (const tt of taskTypes) {
+        if (!h.hasClass(tt.targetClass)) {
+          const ofClassClass = h.getClass(tt.ofClass)
+          await ops.createDoc(
+            core.class.Class,
+            core.space.Model,
+            {
+              extends: tt.ofClass,
+              kind: ClassifierKind.MIXIN,
+              label: getEmbeddedLabel(tt.name),
+              icon: ofClassClass.icon
+            },
+            pt.targetClass,
+            Date.now(),
+            core.account.ConfigUser
+          )
+        }
+      }
+    } catch (err: any) {
+      //
+      console.error(err)
+    }
+  }
+}
+
 export const taskOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     await tryMigrate(client, taskId, [
@@ -167,6 +224,10 @@ export const taskOperation: MigrateOperation = {
       {
         state: 'reorderStates',
         func: reorderStates
+      },
+      {
+        state: 'fix-project-type-missing-class',
+        func: fixProjectTypeMissingClass
       }
     ])
   }

plugins/setting-resources/src/components/ClassAttributeRow.svelte

@@ -42,9 +42,11 @@
   }
   function getArrayName (type: Type<any>): IntlString | undefined {
     const ref = (type as ArrOf<any>).of
+    if (client.getHierarchy().hasClass((ref as RefTo<Doc>).to)) {
       const res = client.getHierarchy().getClass((ref as RefTo<Doc>).to)
       return res?.label
     }
+  }
 </script>
 
 <button class="hulyTableAttr-content__row" class:hovered class:selected on:contextmenu on:click>

server/account/src/index.ts

@@ -604,6 +604,27 @@ export async function listWorkspaces (db: Db, productId: string): Promise<Worksp
     .map(trimWorkspaceInfo)
 }
 
+/**
+ * @public
+ */
+export async function listWorkspacesRaw (db: Db, productId: string): Promise<Workspace[]> {
+  return (await db.collection<Workspace>(WORKSPACE_COLLECTION).find(withProductId(productId, {})).toArray()).map(
+    (it) => ({ ...it, productId })
+  )
+}
+
+/**
+ * @public
+ */
+export async function updateWorkspace (
+  db: Db,
+  productId: string,
+  info: Workspace,
+  ops: Partial<Workspace>
+): Promise<void> {
+  await db.collection<Workspace>(WORKSPACE_COLLECTION).updateOne({ _id: info._id }, { $set: { ...info, ...ops } })
+}
+
 /**
  * @public
  */
@@ -772,7 +793,7 @@ export async function upgradeWorkspace (
   const versionStr = versionToString(version)
 
   console.log(
-    `${forceUpdate ? 'force-' : ''}upgrade from "${
+    `${workspaceUrl} - ${forceUpdate ? 'force-' : ''}upgrade from "${
       ws?.version !== undefined ? versionToString(ws.version) : ''
     }" to "${versionStr}"`
   )

server/tool/src/upgrade.ts

@@ -183,7 +183,9 @@ export class MigrateClientImpl implements MigrationClient {
 
   async create<T extends Doc>(domain: Domain, doc: T | T[]): Promise<void> {
     if (Array.isArray(doc)) {
+      if (doc.length > 0) {
         await this.db.collection(domain).insertMany(doc as Document[])
+      }
     } else {
       await this.db.collection(domain).insertOne(doc as Document)
     }