Init workspace via script (#6007)

Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com>

common/config/rush/pnpm-lock.yaml

@@ -22580,7 +22580,7 @@ packages:
     dev: false
 
   file:projects/model.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-st7Z7xVdtaQf3+WTi8BcUYBwuZzi+2iWF+gyYIFi6AI1beJh4Mdjtlu6NYfEGRkHONpL/czV4laJly2Dxkc1Ew==, tarball: file:projects/model.tgz}
+    resolution: {integrity: sha512-8Rge/rE550GSKnZADVxI1ykybyTMnaYeRp872Miu2IVHPknVie3N2ZZaLlH0T6M2tQBxqlnAE8VOr1ey9KNWIQ==, tarball: file:projects/model.tgz}
     id: file:projects/model.tgz
     name: '@rush-temp/model'
     version: 0.0.0
@@ -23062,7 +23062,7 @@ packages:
     dev: false
 
   file:projects/pod-server.tgz:
-    resolution: {integrity: sha512-GjJqgPc/ux0jjGydLB1tmiwo9f4tTZL4KECyHmiLemLahnZojN8AIIFQTsqUkiFh0dwULxPvJkRWnYdGLA0iZg==, tarball: file:projects/pod-server.tgz}
+    resolution: {integrity: sha512-e3S57PlJ3ME1jMupiLtYmdZFewiWjjAeY/uyOZ1yENsAf6WnET/uLgDpgRqHy/Kw/rLPvyAfWQR69mTFXpAOZA==, tarball: file:projects/pod-server.tgz}
     name: '@rush-temp/pod-server'
     version: 0.0.0
     dependencies:
@@ -24091,7 +24091,7 @@ packages:
     dev: false
 
   file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-Zpr44q67hLKNSbFvWVJEITtDNValbilYERD5KeWImI44rVHVotGtAISj0GqAtGr0mzGLuosOyP6dhqs/+bfOgw==, tarball: file:projects/server-backup.tgz}
+    resolution: {integrity: sha512-LuaW1naKF+DcYgzDYKohAfO1mMtqzpnglsAfNyAl0hNYUWA/F7M5+/qcy0LpHuwRTltol+RWBBLVeLMzMX9A/Q==, tarball: file:projects/server-backup.tgz}
     id: file:projects/server-backup.tgz
     name: '@rush-temp/server-backup'
     version: 0.0.0
@@ -24969,7 +24969,7 @@ packages:
     dev: false
 
   file:projects/server-notification-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2):
-    resolution: {integrity: sha512-nQJkxhWMtbfdsgmIvt3/qQ/Dz2dsVx6JI5MbNyrJmixplMRt+cNGb08lAVJLsZkFF2UReTss6jNyKLx8WSki/w==, tarball: file:projects/server-notification-resources.tgz}
+    resolution: {integrity: sha512-DhxiRHwKgwqkX4WP2ZmqGL7fHs0Ev3rtxHT/uylk5SPCPbKmBe20xyEzheSahP0Vm6kIS9XerGQuxGrI31eVBw==, tarball: file:projects/server-notification-resources.tgz}
     id: file:projects/server-notification-resources.tgz
     name: '@rush-temp/server-notification-resources'
     version: 0.0.0
@@ -25631,12 +25631,13 @@ packages:
     dev: false
 
   file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3):
-    resolution: {integrity: sha512-ErBexvPUdBHDpLANxgrqNef832SQXZicJp7U+UDiqr9EsTjKjJ71ZgTYlGtj7unI6XfBEoUlH19cAu7ExV3veA==, tarball: file:projects/server-tool.tgz}
+    resolution: {integrity: sha512-7THPdPOTi6rdVo4B0AUJz4mx3urr/lNWZ88ZzgUjPXTl7T2w2PyplAEoCFvxN8A184+6KM9Wb0trUlnftH72fA==, tarball: file:projects/server-tool.tgz}
     id: file:projects/server-tool.tgz
     name: '@rush-temp/server-tool'
     version: 0.0.0
     dependencies:
       '@types/jest': 29.5.12
+      '@types/uuid': 8.3.4
       '@types/ws': 8.5.10
       '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3)
       '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3)
@@ -25651,6 +25652,7 @@ packages:
       prettier: 3.2.5
       ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3)
       typescript: 5.3.3
+      uuid: 8.3.2
       ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3)
     transitivePeerDependencies:
       - '@aws-sdk/credential-providers'

packages/text/src/nodes/image.ts

@@ -94,7 +94,6 @@ export const ImageNode = Node.create<ImageOptions>({
       'data-type': this.name,
       'data-align': node.attrs.align
     }
-
     const imgAttributes = mergeAttributes(
       {
         'data-type': this.name
@@ -114,7 +113,6 @@ export const ImageNode = Node.create<ImageOptions>({
       const container = document.createElement('div')
       const imgElement = document.createElement('img')
       container.append(imgElement)
-
       const divAttributes = {
         class: 'text-editor-image-container',
         'data-type': this.name,
@@ -149,6 +147,13 @@ export const ImageNode = Node.create<ImageOptions>({
             imgElement.srcset = val.srcset
           })
         }
+      } else {
+        if (imgAttributes.srcset != null) {
+          imgElement.srcset = imgAttributes.srcset
+        }
+        if (imgAttributes.src != null) {
+          imgElement.src = imgAttributes.src
+        }
       }
 
       return {

server/account/src/operations.ts

@@ -46,11 +46,9 @@ import core, {
   type Branding
 } from '@hcengineering/core'
 import { consoleModelLogger, MigrateOperation, ModelLogger } from '@hcengineering/model'
-import { getModelVersion } from '@hcengineering/model-all'
 import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform'
-import { cloneWorkspace } from '@hcengineering/server-backup'
 import { decodeToken, generateToken } from '@hcengineering/server-token'
-import toolPlugin, { connect, getStorageAdapter, initModel, upgradeModel } from '@hcengineering/server-tool'
+import toolPlugin, { connect, initializeWorkspace, initModel, upgradeModel } from '@hcengineering/server-tool'
 import { pbkdf2Sync, randomBytes } from 'crypto'
 import { Binary, Db, Filter, ObjectId, type MongoClient } from 'mongodb'
 import fetch from 'node-fetch'
@@ -940,69 +938,19 @@ export async function createWorkspace (
         childLogger.error(msg, data)
       }
     }
-    let model: Tx[] = []
+    const model: Tx[] = []
     try {
-      const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
       const wsId = getWorkspaceId(workspaceInfo.workspace, productId)
 
-      // We should not try to clone INIT_WS into INIT_WS during it's creation.
+      await childLogger.withLog('init-workspace', {}, async (ctx) => {
-      let initWSInfo: Workspace | undefined
+        await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger, async (value) => {
-      if (initWS !== undefined) {
+          await updateInfo({ createProgress: 10 + Math.round((Math.min(value, 100) / 100) * 20) })
-        initWSInfo = (await getWorkspaceById(db, productId, initWS)) ?? undefined
-      }
-      if (initWS !== undefined && initWSInfo !== undefined && initWS !== workspaceInfo.workspace) {
-        // Just any valid model for transactor to be able to function
-        await childLogger.with('init-model', {}, async (ctx) => {
-          await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, async (value) => {
-            await updateInfo({ createProgress: Math.round((Math.min(value, 100) / 100) * 20) })
-          })
         })
+      })
 
-        await updateInfo({ createProgress: 20 })
+      await initializeWorkspace(ctx, branding, getTransactor(), wsId, ctxModellogger, async (value) => {
-
+        await updateInfo({ createProgress: 30 + Math.round((Math.min(value, 100) / 100) * 65) })
-        // Clone init workspace.
+      })
-        await cloneWorkspace(
-          childLogger,
-          getTransactor(),
-          getWorkspaceId(initWS, productId),
-          getWorkspaceId(workspaceInfo.workspace, productId),
-          true,
-          async (value) => {
-            await updateInfo({ createProgress: 20 + Math.round((Math.min(value, 100) / 100) * 70) })
-          },
-          getStorageAdapter()
-        )
-        const modelVersion = getModelVersion()
-        await updateInfo({ createProgress: 90 })
-
-        // Skip tx update if version of init workspace are proper one.
-        const skipTxUpdate =
-          versionToString(modelVersion) === versionToString(initWSInfo.version ?? { major: 0, minor: 0, patch: 0 })
-        model = await childLogger.withLog(
-          'upgrade-model',
-          {},
-          async (ctx) =>
-            await upgradeModel(
-              ctx,
-              getTransactor(),
-              wsId,
-              txes,
-              migrationOperation,
-              ctxModellogger,
-              skipTxUpdate,
-              async (value) => {
-                await updateInfo({ createProgress: Math.round(90 + (Math.min(value, 100) / 100) * 10) })
-              }
-            )
-        )
-        await updateInfo({ createProgress: 99 })
-      } else {
-        await childLogger.withLog('init-workspace', {}, async (ctx) => {
-          await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger, async (value) => {
-            await updateInfo({ createProgress: Math.round(Math.min(value, 100)) })
-          })
-        })
-      }
     } catch (err: any) {
       Analytics.handleError(err)
       return { workspaceInfo, err, client: null as any }
@@ -1124,7 +1072,7 @@ export const createUserWorkspace =
           notifyHandler,
           async (workspace, model) => {
             const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
-            const shouldUpdateAccount = initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null
+            const shouldUpdateAccount = initWS !== undefined
             const client = await connect(
               getTransactor(),
               getWorkspaceId(workspace.workspace, productId),

server/tool/package.json

@@ -32,6 +32,7 @@
     "eslint-config-standard-with-typescript": "^40.0.0",
     "prettier": "^3.1.0",
     "typescript": "^5.3.3",
+    "@types/uuid": "^8.3.1",
     "@types/ws": "^8.5.3",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
@@ -46,6 +47,8 @@
     "@hcengineering/client": "^0.6.18",
     "ws": "^8.16.0",
     "@hcengineering/model": "^0.6.11",
+    "@hcengineering/rank": "^0.6.4",
+    "uuid": "^8.3.2",
     "@hcengineering/server-token": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server": "^0.6.4",

server/tool/src/index.ts

@@ -16,6 +16,7 @@
 import contact from '@hcengineering/contact'
 import core, {
   BackupClient,
+  Branding,
   Client as CoreClient,
   DOMAIN_MIGRATION,
   DOMAIN_MODEL,
@@ -37,9 +38,11 @@ import { DomainIndexHelperImpl, StorageAdapter, StorageConfiguration } from '@hc
 import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { Db, Document } from 'mongodb'
 import { connect } from './connect'
+import { createWorkspaceData, InitScript } from './initializer'
 import toolPlugin from './plugin'
 import { MigrateClientImpl } from './upgrade'
 
+import { getMetadata } from '@hcengineering/platform'
 import fs from 'fs'
 import path from 'path'
 
@@ -47,6 +50,9 @@ export * from './connect'
 export * from './plugin'
 export { toolPlugin as default }
 
+export const CONFIG_DB = '%config'
+const scriptsCol = 'initScripts'
+
 export class FileModelLogger implements ModelLogger {
   handle: fs.WriteStream
   constructor (readonly file: string) {
@@ -181,6 +187,61 @@ export async function initModel (
   }
 }
 
+/**
+ * @public
+ */
+export async function initializeWorkspace (
+  ctx: MeasureContext,
+  branding: Branding | null,
+  transactorUrl: string,
+  workspaceId: WorkspaceId,
+  logger: ModelLogger = consoleModelLogger,
+  progress: (value: number) => Promise<void>
+): Promise<void> {
+  const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
+  if (initWS === undefined) return
+
+  const { mongodbUri } = prepareTools([])
+
+  const _client = getMongoClient(mongodbUri)
+  const client = await _client.getClient()
+  let connection: (CoreClient & BackupClient) | undefined
+  const storageConfig: StorageConfiguration = storageConfigFromEnv()
+  const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
+  try {
+    const db = client.db(CONFIG_DB)
+    const scripts = await db.collection<InitScript>(scriptsCol).find({}).toArray()
+    let script: InitScript | undefined
+    if (initWS !== undefined) {
+      script = scripts.find((it) => it.name === initWS)
+    }
+    if (script === undefined) {
+      script = scripts.find((it) => it.default)
+    }
+    if (script === undefined) {
+      return
+    }
+    try {
+      connection = (await connect(transactorUrl, workspaceId, undefined, {
+        model: 'upgrade',
+        admin: 'true'
+      })) as unknown as CoreClient & BackupClient
+      await createWorkspaceData(ctx, connection, storageAdapter, workspaceId, script, logger, progress)
+    } catch (e: any) {
+      logger.error('error', { error: e })
+      throw e
+    }
+  } catch (err: any) {
+    ctx.error('Failed to create workspace', { error: err })
+    throw err
+  } finally {
+    await storageAdapter.close()
+    await connection?.sendForceClose()
+    await connection?.close()
+    _client.close()
+  }
+}
+
 export function getStorageAdapter (): StorageAdapter {
   const { mongodbUri } = prepareTools([])
 

server/tool/src/initializer.ts

@@ -0,0 +1,224 @@
+import core, {
+  AttachedDoc,
+  Class,
+  Client,
+  Data,
+  Doc,
+  MeasureContext,
+  Mixin,
+  Ref,
+  Space,
+  TxOperations,
+  WorkspaceId
+} from '@hcengineering/core'
+import { ModelLogger } from '@hcengineering/model'
+import { makeRank } from '@hcengineering/rank'
+import { AggregatorStorageAdapter } from '@hcengineering/server-core'
+import { v4 as uuid } from 'uuid'
+
+const fieldRegexp = /\${\S+?}/
+
+export interface InitScript {
+  name: string
+  lang?: string
+  default: boolean
+  steps: InitStep<Doc>[]
+}
+
+export type InitStep<T extends Doc> = CreateStep<T> | MixinStep<T, T> | UpdateStep<T> | FindStep<T> | UploadStep
+
+export interface CreateStep<T extends Doc> {
+  type: 'create'
+  _class: Ref<Class<T>>
+  data: Props<T>
+  resultVariable?: string
+}
+
+export interface MixinStep<T extends Doc, M extends T> {
+  type: 'mixin'
+  _class: Ref<Class<T>>
+  mixin: Ref<Mixin<M>>
+  data: Props<T>
+}
+
+export interface UpdateStep<T extends Doc> {
+  type: 'update'
+  _class: Ref<Class<T>>
+  data: Props<T>
+}
+
+export interface FindStep<T extends Doc> {
+  type: 'find'
+  _class: Ref<Class<T>>
+  query: Partial<T>
+  resultVariable?: string
+}
+
+export interface UploadStep {
+  type: 'upload'
+  fromUrl: string
+  contentType: string
+  size?: number
+  resultVariable?: string
+}
+
+export type Props<T extends Doc> = Data<T> & Partial<Doc> & { space: Ref<Space> }
+
+const nextRank = '#nextRank'
+const now = '#now'
+
+export async function createWorkspaceData (
+  ctx: MeasureContext,
+  connection: Client,
+  storageAdapter: AggregatorStorageAdapter,
+  workspaceId: WorkspaceId,
+  script: InitScript,
+  logger: ModelLogger,
+  progress: (value: number) => Promise<void>
+): Promise<void> {
+  const client = new TxOperations(connection, core.account.System)
+  const vars: Record<string, any> = {}
+  for (let index = 0; index < script.steps.length; index++) {
+    const step = script.steps[index]
+    if (step.type === 'create') {
+      await processCreate(client, step, vars)
+    } else if (step.type === 'update') {
+      await processUpdate(client, step, vars)
+    } else if (step.type === 'mixin') {
+      await processMixin(client, step, vars)
+    } else if (step.type === 'find') {
+      await processFind(client, step, vars)
+    } else if (step.type === 'upload') {
+      await processUpload(ctx, storageAdapter, workspaceId, step, vars, logger)
+    }
+
+    await progress(Math.round(((index + 1) * 100) / script.steps.length))
+  }
+}
+
+async function processUpload (
+  ctx: MeasureContext,
+  storageAdapter: AggregatorStorageAdapter,
+  workspaceId: WorkspaceId,
+  step: UploadStep,
+  vars: Record<string, any>,
+  logger: ModelLogger
+): Promise<void> {
+  try {
+    const id = uuid()
+    const resp = await fetch(step.fromUrl)
+    const buffer = Buffer.from(await resp.arrayBuffer())
+    await storageAdapter.put(ctx, workspaceId, id, buffer, step.contentType, step.size)
+    if (step.resultVariable !== undefined) {
+      vars[step.resultVariable] = id
+    }
+  } catch (error) {
+    logger.error('Upload failed', error)
+  }
+}
+
+async function processFind<T extends Doc> (
+  client: TxOperations,
+  step: FindStep<T>,
+  vars: Record<string, any>
+): Promise<void> {
+  const query = fillProps(step.query, vars)
+  const res = await client.findOne(step._class, { ...(query as any) })
+  if (res === undefined) {
+    throw new Error(`Document not found: ${JSON.stringify(query)}`)
+  }
+  if (step.resultVariable !== undefined) {
+    vars[step.resultVariable] = res
+  }
+}
+
+async function processMixin<T extends Doc> (
+  client: TxOperations,
+  step: MixinStep<T, T>,
+  vars: Record<string, any>
+): Promise<void> {
+  const data = fillProps(step.data, vars)
+  const { _id, space, ...props } = data
+  if (_id === undefined || space === undefined) {
+    throw new Error('Mixin step must have _id and space')
+  }
+  await client.createMixin(_id, step._class, space, step.mixin, props)
+}
+
+async function processUpdate<T extends Doc> (
+  client: TxOperations,
+  step: UpdateStep<T>,
+  vars: Record<string, any>
+): Promise<void> {
+  const data = fillProps(step.data, vars)
+  const { _id, space, ...props } = data
+  if (_id === undefined || space === undefined) {
+    throw new Error('Update step must have _id and space')
+  }
+  await client.updateDoc(step._class, space, _id as Ref<Doc>, props)
+}
+
+async function processCreate<T extends Doc> (
+  client: TxOperations,
+  step: CreateStep<T>,
+  vars: Record<string, any>
+): Promise<void> {
+  const data = fillProps(step.data, vars)
+  const res = await create(client, step._class, data)
+  if (step.resultVariable !== undefined) {
+    vars[step.resultVariable] = res
+  }
+}
+
+async function create<T extends Doc> (client: TxOperations, _class: Ref<Class<T>>, data: Props<T>): Promise<Ref<T>> {
+  const hierarchy = client.getHierarchy()
+  if (hierarchy.isDerived(_class, core.class.AttachedDoc)) {
+    const { space, attachedTo, attachedToClass, collection, ...props } = data as unknown as Props<AttachedDoc>
+    if (attachedTo === undefined || space === undefined || attachedToClass === undefined || collection === undefined) {
+      throw new Error('Add collection step must have attachedTo, attachedToClass, collection and space')
+    }
+    return (await client.addCollection(
+      _class,
+      space,
+      attachedTo,
+      attachedToClass,
+      collection,
+      props
+    )) as unknown as Ref<T>
+  } else {
+    const { space, ...props } = data
+    if (space === undefined) {
+      throw new Error('Create step must have space')
+    }
+    return await client.createDoc<T>(_class, space, props as Data<T>)
+  }
+}
+
+function fillProps<T extends Doc, P extends Partial<T> | Props<T>> (data: P, vars: Record<string, any>): P {
+  for (const key in data) {
+    let value = (data as any)[key]
+    if (typeof value === 'object') {
+      ;(data as any)[key] = fillProps(value, vars)
+    } else if (typeof value === 'string') {
+      if (value === nextRank) {
+        const rank = makeRank(vars[nextRank], undefined)
+        ;(data as any)[key] = rank
+        vars[nextRank] = rank
+      } else if (value === now) {
+        ;(data as any)[key] = new Date().getTime()
+      } else {
+        while (true) {
+          const matched = fieldRegexp.exec(value)
+          if (matched === null) break
+          const result = vars[matched[0]]
+          if (result !== undefined && typeof result === 'string') {
+            value = value.replaceAll(matched[0], result)
+            fieldRegexp.lastIndex = 0
+          }
+        }
+        ;(data as any)[key] = value
+      }
+    }
+  }
+  return data
+}