mirror of
https://github.com/hcengineering/platform.git
synced 2024-12-22 19:11:33 +03:00
UBERF-5986: Upgrade fixes (#4957)
Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
parent
768d57348b
commit
27e46ef3ae
4
common/scripts/docker_tag_push.sh
Executable file
4
common/scripts/docker_tag_push.sh
Executable file
@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
echo "Tagging release $1 with version $2"
|
||||
docker tag "$1" "$1:$2"
|
||||
docker push "$1:$2"
|
@ -26,10 +26,13 @@ import {
|
||||
getWorkspaceById,
|
||||
listAccounts,
|
||||
listWorkspaces,
|
||||
listWorkspacesRaw,
|
||||
replacePassword,
|
||||
setAccountAdmin,
|
||||
setRole,
|
||||
updateWorkspace,
|
||||
upgradeWorkspace,
|
||||
type Workspace,
|
||||
type WorkspaceInfo
|
||||
} from '@hcengineering/account'
|
||||
import { setMetadata } from '@hcengineering/platform'
|
||||
@ -43,12 +46,12 @@ import {
|
||||
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
|
||||
import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
|
||||
|
||||
import { type Command, program } from 'commander'
|
||||
import { type Db, MongoClient } from 'mongodb'
|
||||
import { program, type Command } from 'commander'
|
||||
import { MongoClient, type Db } from 'mongodb'
|
||||
import { clearTelegramHistory } from './telegram'
|
||||
import { diffWorkspace, updateField } from './workspace'
|
||||
|
||||
import { type Data, getWorkspaceId, RateLimiter, type Tx, type Version, type AccountRole } from '@hcengineering/core'
|
||||
import { RateLimiter, getWorkspaceId, type AccountRole, type Data, type Tx, type Version } from '@hcengineering/core'
|
||||
import { type MinioService } from '@hcengineering/minio'
|
||||
import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
|
||||
import { openAIConfigDefaults } from '@hcengineering/openai'
|
||||
@ -273,10 +276,27 @@ export function devTool (
|
||||
.action(async (cmd: { parallel: string, logs: string, retry: string, force: boolean, console: boolean }) => {
|
||||
const { mongodbUri, version, txes, migrateOperations } = prepareTools()
|
||||
await withDatabase(mongodbUri, async (db) => {
|
||||
const workspaces = await listWorkspaces(db, productId)
|
||||
const workspaces = await listWorkspacesRaw(db, productId)
|
||||
|
||||
// We need to update workspaces with missing workspaceUrl
|
||||
for (const ws of workspaces) {
|
||||
if (ws.workspaceUrl == null) {
|
||||
const upd: Partial<Workspace> = {
|
||||
workspaceUrl: ws.workspace
|
||||
}
|
||||
if (ws.workspaceName == null) {
|
||||
upd.workspaceName = ws.workspace
|
||||
}
|
||||
await updateWorkspace(db, productId, ws, upd)
|
||||
}
|
||||
}
|
||||
|
||||
const withError: string[] = []
|
||||
|
||||
async function _upgradeWorkspace (ws: WorkspaceInfo): Promise<void> {
|
||||
if (ws.disabled === true) {
|
||||
return
|
||||
}
|
||||
const t = Date.now()
|
||||
const logger = cmd.console
|
||||
? consoleModelLogger
|
||||
@ -308,11 +328,13 @@ export function devTool (
|
||||
const parallel = parseInt(cmd.parallel) ?? 1
|
||||
const rateLimit = new RateLimiter(parallel)
|
||||
console.log('parallel upgrade', parallel, cmd.parallel)
|
||||
for (const ws of workspaces) {
|
||||
await rateLimit.exec(() => {
|
||||
return _upgradeWorkspace(ws)
|
||||
})
|
||||
}
|
||||
await Promise.all(
|
||||
workspaces.map((it) =>
|
||||
rateLimit.add(() => {
|
||||
return _upgradeWorkspace(it)
|
||||
})
|
||||
)
|
||||
)
|
||||
} else {
|
||||
console.log('UPGRADE write logs at:', cmd.logs)
|
||||
for (const ws of workspaces) {
|
||||
|
@ -25,6 +25,7 @@ import core, {
|
||||
type Class,
|
||||
type Doc,
|
||||
type Ref,
|
||||
type Tx,
|
||||
type TxCUD,
|
||||
type TxCollectionCUD,
|
||||
type TxCreateDoc
|
||||
@ -55,15 +56,18 @@ async function generateDocUpdateMessageByTx (
|
||||
tx: TxCUD<Doc>,
|
||||
control: ActivityControl,
|
||||
client: MigrationClient,
|
||||
objectCache?: DocObjectCache
|
||||
objectCache?: DocObjectCache,
|
||||
existsMap?: Set<Ref<Tx>>
|
||||
): Promise<void> {
|
||||
const existsMessages = await client.find<DocUpdateMessage>(
|
||||
DOMAIN_ACTIVITY,
|
||||
{ _class: activity.class.DocUpdateMessage, txId: tx._id },
|
||||
{ projection: { _id: 1 } }
|
||||
)
|
||||
const existsMessages =
|
||||
existsMap?.has(tx._id) ??
|
||||
(await client.find<DocUpdateMessage>(
|
||||
DOMAIN_ACTIVITY,
|
||||
{ _class: activity.class.DocUpdateMessage, txId: tx._id },
|
||||
{ projection: { _id: 1 } }
|
||||
))
|
||||
|
||||
if (existsMessages.length > 0) {
|
||||
if (existsMessages === true || (Array.isArray(existsMessages) && existsMessages.length > 0)) {
|
||||
return
|
||||
}
|
||||
|
||||
@ -172,6 +176,33 @@ async function createDocUpdateMessages (client: MigrationClient): Promise<void>
|
||||
}
|
||||
}
|
||||
|
||||
const docCache = {
|
||||
docs: docIds,
|
||||
transactions: allTransactions
|
||||
}
|
||||
const txIds = new Set<Ref<Tx>>()
|
||||
for (const d of docs) {
|
||||
processed += 1
|
||||
if (processed % 1000 === 0) {
|
||||
console.log('processed', processed)
|
||||
}
|
||||
const transactions = allTransactions.get(d._id) ?? []
|
||||
for (const tx of transactions) {
|
||||
const innerTx = TxProcessor.extractTx(tx) as TxCUD<Doc>
|
||||
txIds.add(innerTx._id)
|
||||
}
|
||||
}
|
||||
|
||||
const ids = (
|
||||
await client.find<DocUpdateMessage>(
|
||||
DOMAIN_ACTIVITY,
|
||||
{ _class: activity.class.DocUpdateMessage, txId: { $in: Array.from(txIds) as Ref<TxCUD<Doc>>[] } },
|
||||
{ projection: { _id: 1, txId: 1 } }
|
||||
)
|
||||
).map((p) => p.txId as Ref<Tx>)
|
||||
|
||||
const existsMessages = new Set(ids)
|
||||
|
||||
for (const d of docs) {
|
||||
processed += 1
|
||||
if (processed % 1000 === 0) {
|
||||
@ -192,10 +223,7 @@ async function createDocUpdateMessages (client: MigrationClient): Promise<void>
|
||||
}
|
||||
|
||||
try {
|
||||
await generateDocUpdateMessageByTx(tx, notificationControl, client, {
|
||||
docs: docIds,
|
||||
transactions: allTransactions
|
||||
})
|
||||
await generateDocUpdateMessageByTx(tx, notificationControl, client, docCache, existsMessages)
|
||||
} catch (e: any) {
|
||||
console.error('error processing:', d._id, e.stack)
|
||||
}
|
||||
|
@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { TxOperations, type Class, type Doc, type Ref, toIdMap } from '@hcengineering/core'
|
||||
import { ClassifierKind, TxOperations, toIdMap, type Class, type Doc, type Ref } from '@hcengineering/core'
|
||||
import {
|
||||
createOrUpdate,
|
||||
tryMigrate,
|
||||
@ -24,9 +24,10 @@ import {
|
||||
} from '@hcengineering/model'
|
||||
import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
|
||||
import tags from '@hcengineering/model-tags'
|
||||
import { type TaskType, taskId } from '@hcengineering/task'
|
||||
import task from './plugin'
|
||||
import { getEmbeddedLabel } from '@hcengineering/platform'
|
||||
import { taskId, type TaskType } from '@hcengineering/task'
|
||||
import { DOMAIN_TASK } from '.'
|
||||
import task from './plugin'
|
||||
|
||||
/**
|
||||
* @public
|
||||
@ -110,6 +111,62 @@ async function createDefaults (tx: TxOperations): Promise<void> {
|
||||
await createDefaultStatesSpace(tx)
|
||||
}
|
||||
|
||||
async function fixProjectTypeMissingClass (client: MigrationUpgradeClient): Promise<void> {
|
||||
const projectTypes = await client.findAll(task.class.ProjectType, {})
|
||||
const ops = new TxOperations(client, core.account.ConfigUser)
|
||||
|
||||
const h = ops.getHierarchy()
|
||||
for (const pt of projectTypes) {
|
||||
console.log('Checking:', pt.name)
|
||||
try {
|
||||
if (!h.hasClass(pt.targetClass)) {
|
||||
const categoryObj = ops.getModel().findObject(pt.descriptor)
|
||||
if (categoryObj === undefined) {
|
||||
throw new Error('category is not found in model')
|
||||
}
|
||||
const baseClassClass = h.getClass(categoryObj.baseClass)
|
||||
await ops.createDoc(
|
||||
core.class.Class,
|
||||
core.space.Model,
|
||||
{
|
||||
extends: categoryObj.baseClass,
|
||||
kind: ClassifierKind.MIXIN,
|
||||
label: baseClassClass.label,
|
||||
icon: baseClassClass.icon
|
||||
},
|
||||
pt.targetClass,
|
||||
Date.now(),
|
||||
core.account.ConfigUser
|
||||
)
|
||||
}
|
||||
|
||||
const taskTypes = await ops.findAll(task.class.TaskType, { parent: pt._id })
|
||||
|
||||
for (const tt of taskTypes) {
|
||||
if (!h.hasClass(tt.targetClass)) {
|
||||
const ofClassClass = h.getClass(tt.ofClass)
|
||||
await ops.createDoc(
|
||||
core.class.Class,
|
||||
core.space.Model,
|
||||
{
|
||||
extends: tt.ofClass,
|
||||
kind: ClassifierKind.MIXIN,
|
||||
label: getEmbeddedLabel(tt.name),
|
||||
icon: ofClassClass.icon
|
||||
},
|
||||
pt.targetClass,
|
||||
Date.now(),
|
||||
core.account.ConfigUser
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
//
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const taskOperation: MigrateOperation = {
|
||||
async migrate (client: MigrationClient): Promise<void> {
|
||||
await tryMigrate(client, taskId, [
|
||||
@ -167,6 +224,10 @@ export const taskOperation: MigrateOperation = {
|
||||
{
|
||||
state: 'reorderStates',
|
||||
func: reorderStates
|
||||
},
|
||||
{
|
||||
state: 'fix-project-type-missing-class',
|
||||
func: fixProjectTypeMissingClass
|
||||
}
|
||||
])
|
||||
}
|
||||
|
@ -42,8 +42,10 @@
|
||||
}
|
||||
function getArrayName (type: Type<any>): IntlString | undefined {
|
||||
const ref = (type as ArrOf<any>).of
|
||||
const res = client.getHierarchy().getClass((ref as RefTo<Doc>).to)
|
||||
return res?.label
|
||||
if (client.getHierarchy().hasClass((ref as RefTo<Doc>).to)) {
|
||||
const res = client.getHierarchy().getClass((ref as RefTo<Doc>).to)
|
||||
return res?.label
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
@ -604,6 +604,27 @@ export async function listWorkspaces (db: Db, productId: string): Promise<Worksp
|
||||
.map(trimWorkspaceInfo)
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export async function listWorkspacesRaw (db: Db, productId: string): Promise<Workspace[]> {
|
||||
return (await db.collection<Workspace>(WORKSPACE_COLLECTION).find(withProductId(productId, {})).toArray()).map(
|
||||
(it) => ({ ...it, productId })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export async function updateWorkspace (
|
||||
db: Db,
|
||||
productId: string,
|
||||
info: Workspace,
|
||||
ops: Partial<Workspace>
|
||||
): Promise<void> {
|
||||
await db.collection<Workspace>(WORKSPACE_COLLECTION).updateOne({ _id: info._id }, { $set: { ...info, ...ops } })
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
@ -772,7 +793,7 @@ export async function upgradeWorkspace (
|
||||
const versionStr = versionToString(version)
|
||||
|
||||
console.log(
|
||||
`${forceUpdate ? 'force-' : ''}upgrade from "${
|
||||
`${workspaceUrl} - ${forceUpdate ? 'force-' : ''}upgrade from "${
|
||||
ws?.version !== undefined ? versionToString(ws.version) : ''
|
||||
}" to "${versionStr}"`
|
||||
)
|
||||
|
@ -183,7 +183,9 @@ export class MigrateClientImpl implements MigrationClient {
|
||||
|
||||
async create<T extends Doc>(domain: Domain, doc: T | T[]): Promise<void> {
|
||||
if (Array.isArray(doc)) {
|
||||
await this.db.collection(domain).insertMany(doc as Document[])
|
||||
if (doc.length > 0) {
|
||||
await this.db.collection(domain).insertMany(doc as Document[])
|
||||
}
|
||||
} else {
|
||||
await this.db.collection(domain).insertOne(doc as Document)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user