UBERF-8379: Fix workspace creation and missing plugin configuration (#6832)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
2024-11-22 03:14:40 +03:00 · 2024-10-08 19:43:33 +07:00 · 2024-10-08 19:43:33 +07:00 · a698889f61
commit a698889f61
parent 27548a49dd
10 changed files with 108 additions and 175 deletions
--- a/dev/docker-compose.yaml
+++ b/dev/docker-compose.yaml
@ -113,6 +113,7 @@ services:
      - MODEL_ENABLED=*
      - ACCOUNTS_URL=http://host.docker.internal:3000
      - BRANDING_PATH=/var/cfg/branding.json
+      - NOTIFY_INBOX_ONLY=true
      # - PARALLEL=2
      # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
      # - INIT_WORKSPACE=onboarding
--- a/dev/tool/src/index.ts
+++ b/dev/tool/src/index.ts
@ -48,7 +48,7 @@ import {
  restore
 } from '@hcengineering/server-backup'
 import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client'
-import { getServerPipeline } from '@hcengineering/server-pipeline'
+import { getServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
@ -163,10 +163,6 @@ export function devTool (
    return elasticUrl
  }

-  const initWS = process.env.INIT_WORKSPACE
-  if (initWS !== undefined) {
-    setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
-  }
  const initScriptUrl = process.env.INIT_SCRIPT_URL
  if (initScriptUrl !== undefined) {
    setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl)
@ -1569,6 +1565,9 @@ export function devTool (
                workspaceUrl: workspace.workspaceUrl ?? ''
              }

+              registerServerPlugins()
+              registerStringLoaders()
+
              const { pipeline } = await getServerPipeline(toolCtx, txes, mongodbUri ?? dbUrl, dbUrl, wsUrl)

              await migrateMarkup(toolCtx, adapter, wsId, _client, pipeline, parseInt(cmd.concurrency))
--- a/models/all/src/index.ts
+++ b/models/all/src/index.ts
@ -256,7 +256,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
      {
        label: inventory.string.ConfigLabel,
        description: inventory.string.ConfigDescription,
-        enabled: false,
+        enabled: true,
        beta: false,
        classFilter: defaultFilter
      }
@ -267,8 +267,8 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
      {
        label: hr.string.ConfigLabel,
        description: hr.string.ConfigDescription,
-        enabled: false,
-        beta: true,
+        enabled: true,
+        beta: false,
        icon: hr.icon.Structure,
        classFilter: defaultFilter
      }
@ -292,7 +292,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
        label: document.string.ConfigLabel,
        description: document.string.ConfigDescription,
        enabled: true,
-        beta: true,
+        beta: false,
        icon: document.icon.DocumentApplication,
        classFilter: defaultFilter
      }
@ -343,7 +343,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
        label: github.string.ConfigLabel,
        description: github.string.ConfigDescription,
        enabled: true,
-        beta: true,
+        beta: false,
        icon: github.icon.Github
      }
    ],
--- a/server/account-service/src/index.ts
+++ b/server/account-service/src/index.ts
@ -77,10 +77,6 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap

  setMetadata(serverToken.metadata.Secret, serverSecret)

-  const initWS = process.env.INIT_WORKSPACE
-  if (initWS !== undefined) {
-    setMetadata(toolPlugin.metadata.InitWorkspace, initWS)
-  }
  const initScriptUrl = process.env.INIT_SCRIPT_URL
  if (initScriptUrl !== undefined) {
    setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl)
--- a/server/account/src/operations.ts
+++ b/server/account/src/operations.ts
@ -47,7 +47,7 @@ import core, {
 import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform'
 import { type StorageAdapter } from '@hcengineering/server-core'
 import { decodeToken as decodeTokenRaw, generateToken, type Token } from '@hcengineering/server-token'
-import toolPlugin, { connect } from '@hcengineering/server-tool'
+import { connect } from '@hcengineering/server-tool'
 import { randomBytes } from 'crypto'
 import { type MongoClient } from 'mongodb'
 import otpGenerator from 'otp-generator'
@ -55,8 +55,8 @@ import otpGenerator from 'otp-generator'
 import { accountPlugin } from './plugin'
 import type {
  Account,
-  AccountInfo,
  AccountDB,
+  AccountInfo,
  ClientWorkspaceInfo,
  Invite,
  LoginInfo,
@ -71,14 +71,14 @@ import type {
  WorkspaceOperation
 } from './types'
 import {
-  toAccountInfo,
-  getEndpoint,
-  EndpointKind,
+  areDbIdsEqual,
  cleanEmail,
+  EndpointKind,
+  getEndpoint,
  hashWithSalt,
  isEmail,
-  verifyPassword,
-  areDbIdsEqual
+  toAccountInfo,
+  verifyPassword
 } from './utils'

 /**
@ -1177,8 +1177,6 @@ async function postCreateUserWorkspace (
  branding: Branding | null,
  workspace: Workspace
 ): Promise<void> {
-  const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
-  const shouldUpdateAccount = initWS !== undefined
  const client = await connect(
    getEndpoint(ctx, workspace, EndpointKind.Internal),
    getWorkspaceId(workspace.workspace),
@ -1196,7 +1194,7 @@ async function postCreateUserWorkspace (
      workspace.workspace,
      AccountRole.Owner,
      undefined,
-      shouldUpdateAccount,
+      true,
      client
    )
    ctx.info('Creating server side done', { workspaceName: workspace.workspaceName, email: workspace.workspaceName })
@ -1590,12 +1588,6 @@ export async function assignWorkspace (
  personAccountId?: Ref<PersonAccount>
 ): Promise<Workspace> {
  const email = cleanEmail(_email)
-  const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace)
-  if (initWS !== undefined && initWS === workspaceId) {
-    Analytics.handleError(new Error(`assign-workspace failed ${email} ${workspaceId}`))
-    ctx.error('assign-workspace failed', { email, workspaceId, reason: 'initWs === workspaceId' })
-    throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {}))
-  }
  const workspaceInfo = await getWorkspaceAndAccount(ctx, db, email, workspaceId)

  if (workspaceInfo.account !== null) {
--- a/server/server-pipeline/src/pipeline.ts
+++ b/server/server-pipeline/src/pipeline.ts
@ -29,13 +29,13 @@ import {
  MarkDerivedEntryMiddleware,
  ModelMiddleware,
  ModifiedMiddleware,
+  NotificationsMiddleware,
  PrivateMiddleware,
  QueryJoinMiddleware,
  SpacePermissionsMiddleware,
  SpaceSecurityMiddleware,
  TriggersMiddleware,
-  TxMiddleware,
-  NotificationsMiddleware
+  TxMiddleware
 } from '@hcengineering/middleware'
 import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
 import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
@ -54,7 +54,6 @@ import {
  DummyDbAdapter,
  DummyFullTextAdapter,
  FullTextMiddleware,
-  type AggregatorStorageAdapter,
  type DbAdapterFactory,
  type DbConfiguration,
  type Middleware,
@ -236,7 +235,7 @@ export async function getServerPipeline (
  wsUrl: WorkspaceIdWithUrl
 ): Promise<{
    pipeline: Pipeline
-    storageAdapter: AggregatorStorageAdapter
+    storageAdapter: StorageAdapter
  }> {
  const dbUrls = mongodbUri !== undefined && mongodbUri !== dbUrl ? `${dbUrl};${mongodbUri}` : dbUrl

@ -258,7 +257,7 @@ export async function getServerPipeline (
      indexProcessing: 0,
      rekoniUrl: '',
      usePassedCtx: true,
-      disableTriggers: true
+      disableTriggers: false
    },
    {
      fulltextAdapter: {
--- a/server/tool/src/index.ts
+++ b/server/tool/src/index.ts
@ -13,7 +13,6 @@
 // limitations under the License.
 //

-import contact from '@hcengineering/contact'
 import core, {
  BackupClient,
  Branding,
@ -36,17 +35,10 @@ import core, {
  WorkspaceId,
  WorkspaceIdWithUrl,
  type Doc,
-  type Ref,
-  type TxCUD
+  type Ref
 } from '@hcengineering/core'
 import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model'
-import {
-  AggregatorStorageAdapter,
-  DomainIndexHelperImpl,
-  Pipeline,
-  StorageAdapter,
-  type DbAdapter
-} from '@hcengineering/server-core'
+import { DomainIndexHelperImpl, Pipeline, StorageAdapter, type DbAdapter } from '@hcengineering/server-core'
 import { connect } from './connect'
 import { InitScript, WorkspaceInitializer } from './initializer'
 import toolPlugin from './plugin'
@ -113,10 +105,9 @@ export async function initModel (
  workspaceId: WorkspaceId,
  rawTxes: Tx[],
  adapter: DbAdapter,
-  storageAdapter: AggregatorStorageAdapter,
+  storageAdapter: StorageAdapter,
  logger: ModelLogger = consoleModelLogger,
-  progress: (value: number) => Promise<void>,
-  deleteFirst: boolean = false
+  progress: (value: number) => Promise<void>
 ): Promise<void> {
  const { txes } = prepareTools(rawTxes)
  if (txes.some((tx) => tx.objectSpace !== core.space.Model)) {
@ -124,23 +115,6 @@ export async function initModel (
  }

  try {
-    if (deleteFirst) {
-      logger.log('deleting model...', workspaceId)
-      await ctx.with('mongo-delete', {}, async () => {
-        const toRemove = await adapter.rawFindAll(DOMAIN_TX, {
-          objectSpace: core.space.Model,
-          modifiedBy: core.account.System,
-          objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] }
-        })
-        await adapter.clean(
-          ctx,
-          DOMAIN_TX,
-          toRemove.map((p) => p._id)
-        )
-      })
-      logger.log('transactions deleted.', { workspaceId: workspaceId.name })
-    }
-
    logger.log('creating database...', workspaceId)
    await adapter.upload(ctx, DOMAIN_TX, [
      {
@ -219,7 +193,7 @@ export async function initializeWorkspace (
  ctx: MeasureContext,
  branding: Branding | null,
  wsUrl: WorkspaceIdWithUrl,
-  storageAdapter: AggregatorStorageAdapter,
+  storageAdapter: StorageAdapter,
  client: TxOperations,
  logger: ModelLogger = consoleModelLogger,
  progress: (value: number) => Promise<void>
@ -263,7 +237,6 @@ export async function upgradeModel (
  storageAdapter: StorageAdapter,
  migrateOperations: [string, MigrateOperation][],
  logger: ModelLogger = consoleModelLogger,
-  skipTxUpdate: boolean = false,
  progress: (value: number) => Promise<void>,
  forceIndexes: boolean = false
 ): Promise<Tx[]> {
@ -309,15 +282,6 @@ export async function upgradeModel (
    }
  })

-  if (!skipTxUpdate) {
-    if (pipeline.context.lowLevelStorage === undefined) {
-      throw new PlatformError(unknownError('Low level storage is not available'))
-    }
-    logger.log('removing model...', { workspaceId: workspaceId.name })
-    await progress(10)
-    logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: txes.length })
-    await progress(20)
-  }
  const { migrateClient, migrateState } = await prepareMigrationClient(
    pipeline,
    hierarchy,
@ -366,21 +330,6 @@ export async function upgradeModel (
      {
        state: 'indexes-v5',
        func: upgradeIndexes
-      },
-      {
-        state: 'delete-model',
-        func: async (client) => {
-          const model = await client.find<Tx>(DOMAIN_TX, { objectSpace: core.space.Model })
-
-          // Ignore Employee accounts.
-          const isUserTx = (it: Tx): boolean =>
-            it.modifiedBy !== core.account.System ||
-            (it as TxCUD<Doc>).objectClass === 'contact:class:Person' ||
-            (it as TxCUD<Doc>).objectClass === 'contact:class:PersonAccount'
-
-          const toDelete = model.filter((it) => !isUserTx(it)).map((it) => it._id)
-          await client.deleteMany(DOMAIN_TX, { _id: { $in: toDelete } })
-        }
      }
    ])
  })
--- a/server/tool/src/initializer.ts
+++ b/server/tool/src/initializer.ts
@ -15,7 +15,7 @@ import core, {
 } from '@hcengineering/core'
 import { ModelLogger } from '@hcengineering/model'
 import { makeRank } from '@hcengineering/rank'
-import { AggregatorStorageAdapter } from '@hcengineering/server-core'
+import type { StorageAdapter } from '@hcengineering/server-core'
 import { jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text'
 import { v4 as uuid } from 'uuid'

@ -91,7 +91,7 @@ export class WorkspaceInitializer {

  constructor (
    private readonly ctx: MeasureContext,
-    private readonly storageAdapter: AggregatorStorageAdapter,
+    private readonly storageAdapter: StorageAdapter,
    private readonly wsUrl: WorkspaceIdWithUrl,
    private readonly client: TxOperations
  ) {}
--- a/server/workspace-service/src/service.ts
+++ b/server/workspace-service/src/service.ts
@ -13,29 +13,29 @@
 // limitations under the License.
 //
 import {
-  type BrandingMap,
-  systemAccountEmail,
  type BaseWorkspaceInfo,
+  type BrandingMap,
  type Data,
  type MeasureContext,
  type Tx,
  type Version,
  getBranding,
-  getWorkspaceId
+  getWorkspaceId,
+  systemAccountEmail
 } from '@hcengineering/core'
 import { type MigrateOperation, type ModelLogger } from '@hcengineering/model'
 import {
  getPendingWorkspace,
  updateWorkspaceInfo,
-  workerHandshake,
+  withRetryConnUntilSuccess,
  withRetryConnUntilTimeout,
-  withRetryConnUntilSuccess
+  workerHandshake
 } from '@hcengineering/server-client'
 import { generateToken } from '@hcengineering/server-token'
 import { FileModelLogger } from '@hcengineering/server-tool'
 import path from 'path'

-import { upgradeWorkspace, createWorkspace } from './ws-operations'
+import { createWorkspace, upgradeWorkspace } from './ws-operations'

 export interface WorkspaceOptions {
  errorHandler: (workspace: BaseWorkspaceInfo, error: any) => Promise<void>
--- a/server/workspace-service/src/ws-operations.ts
+++ b/server/workspace-service/src/ws-operations.ts
@ -16,23 +16,13 @@ import core, {
 } from '@hcengineering/core'
 import { consoleModelLogger, type MigrateOperation, type ModelLogger } from '@hcengineering/model'
 import { getTransactorEndpoint } from '@hcengineering/server-client'
+import { SessionDataImpl, type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
 import {
-  DummyFullTextAdapter,
-  SessionDataImpl,
-  type Pipeline,
-  type PipelineFactory,
-  type StorageAdapter,
-  type StorageConfiguration
-} from '@hcengineering/server-core'
-import {
-  createIndexStages,
-  createServerPipeline,
  getServerPipeline,
  getTxAdapterFactory,
  registerServerPlugins,
  registerStringLoaders
 } from '@hcengineering/server-pipeline'
-import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import { generateToken } from '@hcengineering/server-token'
 import { initializeWorkspace, initModel, prepareTools, updateModel, upgradeModel } from '@hcengineering/server-tool'

@ -130,9 +120,10 @@ export async function createWorkspace (
    const dbUrls = mongodbUri !== undefined && dbUrl !== mongodbUri ? `${dbUrl};${mongodbUri}` : dbUrl
    const hierarchy = new Hierarchy()
    const modelDb = new ModelDb(hierarchy)
+    registerServerPlugins()
+    registerStringLoaders()

-    const storageConfig: StorageConfiguration = storageConfigFromEnv()
-    const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri)
+    const { pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, wsUrl)

    try {
      const txFactory = getTxAdapterFactory(ctx, dbUrls, wsUrl, null, {
@ -146,57 +137,12 @@ export async function createWorkspace (
      const txAdapter = await txFactory(ctx, hierarchy, dbUrl ?? mongodbUri, wsId, modelDb, storageAdapter)

      await childLogger.withLog('init-workspace', {}, async (ctx) => {
-        const deleteModelFirst = workspaceInfo.mode === 'creating'
-
-        await initModel(
-          ctx,
-          wsId,
-          txes,
-          txAdapter,
-          storageAdapter,
-          ctxModellogger,
-          async (value) => {
-            await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10))
-          },
-          deleteModelFirst
-        )
+        await initModel(ctx, wsId, txes, txAdapter, storageAdapter, ctxModellogger, async (value) => {
+          await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10))
+        })
      })
-      registerServerPlugins()
-      registerStringLoaders()
-      const factory: PipelineFactory = createServerPipeline(
-        ctx,
-        dbUrls,
-        txes,
-        {
-          externalStorage: storageAdapter,
-          fullTextUrl: 'http://localhost:9200',
-          indexParallel: 0,
-          indexProcessing: 0,
-          rekoniUrl: '',
-          usePassedCtx: true
-        },
-        {
-          fulltextAdapter: {
-            factory: async () => new DummyFullTextAdapter(),
-            url: '',
-            stages: (adapter, storage, storageAdapter, contentAdapter) =>
-              createIndexStages(
-                ctx.newChild('stages', {}),
-                wsUrl,
-                branding,
-                adapter,
-                storage,
-                storageAdapter,
-                contentAdapter,
-                0,
-                0
-              )
-          }
-        }
-      )

-      const pipeline = await factory(ctx, wsUrl, true, () => {}, null)
-      const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.System)
+      const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.ConfigUser)

      await updateModel(ctx, wsId, migrationOperation, client, pipeline, ctxModellogger, async (value) => {
        await handleWsEvent?.('progress', version, 20 + Math.round((Math.min(value, 100) / 100) * 10))
@ -208,12 +154,14 @@ export async function createWorkspace (
        await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 60))
      })

-      await upgradeWorkspace(
+      await upgradeWorkspaceWith(
        ctx,
        version,
        txes,
        migrationOperation,
        workspaceInfo,
+        pipeline,
+        storageAdapter,
        ctxModellogger,
        async (event, version, value) => {
          ctx.info('Init script progress', { event, value })
@ -223,12 +171,11 @@ export async function createWorkspace (
        false
      )

-      await pipeline.close()
-
      await handleWsEvent?.('create-done', version, 100, '')
    } catch (err: any) {
      await handleWsEvent?.('ping', version, 0, `Create failed: ${err.message}`)
    } finally {
+      await pipeline.close()
      await storageAdapter.close()
    }
  } finally {
@ -256,6 +203,67 @@ export async function upgradeWorkspace (
  forceUpdate: boolean = true,
  forceIndexes: boolean = false,
  external: boolean = false
+): Promise<void> {
+  const { mongodbUri, dbUrl } = prepareTools([])
+  if (mongodbUri === undefined) {
+    throw new Error('No MONGO_URL specified')
+  }
+  let pipeline: Pipeline | undefined
+  let storageAdapter: StorageAdapter | undefined
+
+  registerServerPlugins()
+  registerStringLoaders()
+  try {
+    ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, {
+      name: ws.workspace,
+      workspaceName: ws.workspaceName ?? '',
+      workspaceUrl: ws.workspaceUrl ?? ''
+    }))
+    if (pipeline === undefined || storageAdapter === undefined) {
+      return
+    }
+
+    await upgradeWorkspaceWith(
+      ctx,
+      version,
+      txes,
+      migrationOperation,
+      ws,
+      pipeline,
+      storageAdapter,
+      logger,
+      handleWsEvent,
+      forceUpdate,
+      forceIndexes,
+      external
+    )
+  } finally {
+    await pipeline?.close()
+    await storageAdapter?.close()
+  }
+}
+
+/**
+ * @public
+ */
+export async function upgradeWorkspaceWith (
+  ctx: MeasureContext,
+  version: Data<Version>,
+  txes: Tx[],
+  migrationOperation: [string, MigrateOperation][],
+  ws: BaseWorkspaceInfo,
+  pipeline: Pipeline,
+  storageAdapter: StorageAdapter,
+  logger: ModelLogger = consoleModelLogger,
+  handleWsEvent?: (
+    event: 'upgrade-started' | 'progress' | 'upgrade-done' | 'ping',
+    version: Data<Version>,
+    progress: number,
+    message?: string
+  ) => Promise<void>,
+  forceUpdate: boolean = true,
+  forceIndexes: boolean = false,
+  external: boolean = false
 ): Promise<void> {
  const versionStr = versionToString(version)

@ -282,20 +290,12 @@ export async function upgradeWorkspace (
    void handleWsEvent?.('progress', version, progress)
  }, 5000)

-  const { mongodbUri, dbUrl } = prepareTools([])
-  if (mongodbUri === undefined) {
-    throw new Error('No MONGO_URL specified')
-  }
-
  const wsUrl: WorkspaceIdWithUrl = {
    name: ws.workspace,
    workspaceName: ws.workspaceName ?? '',
    workspaceUrl: ws.workspaceUrl ?? ''
  }
-  let pipeline: Pipeline | undefined
-  let storageAdapter: StorageAdapter | undefined
  try {
-    ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, wsUrl))
    const contextData = new SessionDataImpl(
      systemAccountEmail,
      'backup',
@ -320,7 +320,6 @@ export async function upgradeWorkspace (
      storageAdapter,
      migrationOperation,
      logger,
-      false,
      async (value) => {
        progress = value
      },
@ -333,8 +332,6 @@ export async function upgradeWorkspace (
    await handleWsEvent?.('ping', version, 0, `Upgrade failed: ${err.message}`)
    throw err
  } finally {
-    await pipeline?.close()
-    await storageAdapter?.close()
    clearInterval(updateProgressHandle)
  }
 }