From a698889f616f97be26b80ecb2f9d925eb3d93d48 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Tue, 8 Oct 2024 19:43:33 +0700 Subject: [PATCH] UBERF-8379: Fix workspace creation and missing plugin configuration (#6832) Signed-off-by: Andrey Sobolev --- dev/docker-compose.yaml | 1 + dev/tool/src/index.ts | 9 +- models/all/src/index.ts | 10 +- server/account-service/src/index.ts | 4 - server/account/src/operations.ts | 24 +-- server/server-pipeline/src/pipeline.ts | 9 +- server/tool/src/index.ts | 61 +------ server/tool/src/initializer.ts | 4 +- server/workspace-service/src/service.ts | 12 +- server/workspace-service/src/ws-operations.ts | 149 +++++++++--------- 10 files changed, 108 insertions(+), 175 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 5f2ea08269..7d215c3f17 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -113,6 +113,7 @@ services: - MODEL_ENABLED=* - ACCOUNTS_URL=http://host.docker.internal:3000 - BRANDING_PATH=/var/cfg/branding.json + - NOTIFY_INBOX_ONLY=true # - PARALLEL=2 # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 539bac113d..3e2dbe5711 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -48,7 +48,7 @@ import { restore } from '@hcengineering/server-backup' import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client' -import { getServerPipeline } from '@hcengineering/server-pipeline' +import { getServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' @@ -163,10 +163,6 @@ export function devTool ( return elasticUrl } - const initWS = process.env.INIT_WORKSPACE - if (initWS !== undefined) { - setMetadata(toolPlugin.metadata.InitWorkspace, initWS) - } const initScriptUrl = process.env.INIT_SCRIPT_URL if (initScriptUrl !== undefined) { setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) @@ -1569,6 +1565,9 @@ export function devTool ( workspaceUrl: workspace.workspaceUrl ?? '' } + registerServerPlugins() + registerStringLoaders() + const { pipeline } = await getServerPipeline(toolCtx, txes, mongodbUri ?? dbUrl, dbUrl, wsUrl) await migrateMarkup(toolCtx, adapter, wsId, _client, pipeline, parseInt(cmd.concurrency)) diff --git a/models/all/src/index.ts b/models/all/src/index.ts index ff707155a1..8548b9751c 100644 --- a/models/all/src/index.ts +++ b/models/all/src/index.ts @@ -256,7 +256,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ { label: inventory.string.ConfigLabel, description: inventory.string.ConfigDescription, - enabled: false, + enabled: true, beta: false, classFilter: defaultFilter } @@ -267,8 +267,8 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ { label: hr.string.ConfigLabel, description: hr.string.ConfigDescription, - enabled: false, - beta: true, + enabled: true, + beta: false, icon: hr.icon.Structure, classFilter: defaultFilter } @@ -292,7 +292,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ label: document.string.ConfigLabel, description: document.string.ConfigDescription, enabled: true, - beta: true, + beta: false, icon: document.icon.DocumentApplication, classFilter: defaultFilter } @@ -343,7 +343,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ label: github.string.ConfigLabel, description: github.string.ConfigDescription, enabled: true, - beta: true, + beta: false, icon: github.icon.Github } ], diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index 2ea0284961..bafb25415f 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -77,10 +77,6 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap setMetadata(serverToken.metadata.Secret, serverSecret) - const initWS = process.env.INIT_WORKSPACE - if (initWS !== undefined) { - setMetadata(toolPlugin.metadata.InitWorkspace, initWS) - } const initScriptUrl = process.env.INIT_SCRIPT_URL if (initScriptUrl !== undefined) { setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 13541db7a4..773b2d82f5 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -47,7 +47,7 @@ import core, { import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform' import { type StorageAdapter } from '@hcengineering/server-core' import { decodeToken as decodeTokenRaw, generateToken, type Token } from '@hcengineering/server-token' -import toolPlugin, { connect } from '@hcengineering/server-tool' +import { connect } from '@hcengineering/server-tool' import { randomBytes } from 'crypto' import { type MongoClient } from 'mongodb' import otpGenerator from 'otp-generator' @@ -55,8 +55,8 @@ import otpGenerator from 'otp-generator' import { accountPlugin } from './plugin' import type { Account, - AccountInfo, AccountDB, + AccountInfo, ClientWorkspaceInfo, Invite, LoginInfo, @@ -71,14 +71,14 @@ import type { WorkspaceOperation } from './types' import { - toAccountInfo, - getEndpoint, - EndpointKind, + areDbIdsEqual, cleanEmail, + EndpointKind, + getEndpoint, hashWithSalt, isEmail, - verifyPassword, - areDbIdsEqual + toAccountInfo, + verifyPassword } from './utils' /** @@ -1177,8 +1177,6 @@ async function postCreateUserWorkspace ( branding: Branding | null, workspace: Workspace ): Promise { - const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) - const shouldUpdateAccount = initWS !== undefined const client = await connect( getEndpoint(ctx, workspace, EndpointKind.Internal), getWorkspaceId(workspace.workspace), @@ -1196,7 +1194,7 @@ async function postCreateUserWorkspace ( workspace.workspace, AccountRole.Owner, undefined, - shouldUpdateAccount, + true, client ) ctx.info('Creating server side done', { workspaceName: workspace.workspaceName, email: workspace.workspaceName }) @@ -1590,12 +1588,6 @@ export async function assignWorkspace ( personAccountId?: Ref ): Promise { const email = cleanEmail(_email) - const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) - if (initWS !== undefined && initWS === workspaceId) { - Analytics.handleError(new Error(`assign-workspace failed ${email} ${workspaceId}`)) - ctx.error('assign-workspace failed', { email, workspaceId, reason: 'initWs === workspaceId' }) - throw new PlatformError(new Status(Severity.ERROR, platform.status.Forbidden, {})) - } const workspaceInfo = await getWorkspaceAndAccount(ctx, db, email, workspaceId) if (workspaceInfo.account !== null) { diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts index cf7b1ca931..a255451f03 100644 --- a/server/server-pipeline/src/pipeline.ts +++ b/server/server-pipeline/src/pipeline.ts @@ -29,13 +29,13 @@ import { MarkDerivedEntryMiddleware, ModelMiddleware, ModifiedMiddleware, + NotificationsMiddleware, PrivateMiddleware, QueryJoinMiddleware, SpacePermissionsMiddleware, SpaceSecurityMiddleware, TriggersMiddleware, - TxMiddleware, - NotificationsMiddleware + TxMiddleware } from '@hcengineering/middleware' import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo' import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres' @@ -54,7 +54,6 @@ import { DummyDbAdapter, DummyFullTextAdapter, FullTextMiddleware, - type AggregatorStorageAdapter, type DbAdapterFactory, type DbConfiguration, type Middleware, @@ -236,7 +235,7 @@ export async function getServerPipeline ( wsUrl: WorkspaceIdWithUrl ): Promise<{ pipeline: Pipeline - storageAdapter: AggregatorStorageAdapter + storageAdapter: StorageAdapter }> { const dbUrls = mongodbUri !== undefined && mongodbUri !== dbUrl ? `${dbUrl};${mongodbUri}` : dbUrl @@ -258,7 +257,7 @@ export async function getServerPipeline ( indexProcessing: 0, rekoniUrl: '', usePassedCtx: true, - disableTriggers: true + disableTriggers: false }, { fulltextAdapter: { diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 81952c0315..c2268f7d93 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -13,7 +13,6 @@ // limitations under the License. // -import contact from '@hcengineering/contact' import core, { BackupClient, Branding, @@ -36,17 +35,10 @@ import core, { WorkspaceId, WorkspaceIdWithUrl, type Doc, - type Ref, - type TxCUD + type Ref } from '@hcengineering/core' import { consoleModelLogger, MigrateOperation, ModelLogger, tryMigrate } from '@hcengineering/model' -import { - AggregatorStorageAdapter, - DomainIndexHelperImpl, - Pipeline, - StorageAdapter, - type DbAdapter -} from '@hcengineering/server-core' +import { DomainIndexHelperImpl, Pipeline, StorageAdapter, type DbAdapter } from '@hcengineering/server-core' import { connect } from './connect' import { InitScript, WorkspaceInitializer } from './initializer' import toolPlugin from './plugin' @@ -113,10 +105,9 @@ export async function initModel ( workspaceId: WorkspaceId, rawTxes: Tx[], adapter: DbAdapter, - storageAdapter: AggregatorStorageAdapter, + storageAdapter: StorageAdapter, logger: ModelLogger = consoleModelLogger, - progress: (value: number) => Promise, - deleteFirst: boolean = false + progress: (value: number) => Promise ): Promise { const { txes } = prepareTools(rawTxes) if (txes.some((tx) => tx.objectSpace !== core.space.Model)) { @@ -124,23 +115,6 @@ export async function initModel ( } try { - if (deleteFirst) { - logger.log('deleting model...', workspaceId) - await ctx.with('mongo-delete', {}, async () => { - const toRemove = await adapter.rawFindAll(DOMAIN_TX, { - objectSpace: core.space.Model, - modifiedBy: core.account.System, - objectClass: { $nin: [contact.class.PersonAccount, 'contact:class:EmployeeAccount'] } - }) - await adapter.clean( - ctx, - DOMAIN_TX, - toRemove.map((p) => p._id) - ) - }) - logger.log('transactions deleted.', { workspaceId: workspaceId.name }) - } - logger.log('creating database...', workspaceId) await adapter.upload(ctx, DOMAIN_TX, [ { @@ -219,7 +193,7 @@ export async function initializeWorkspace ( ctx: MeasureContext, branding: Branding | null, wsUrl: WorkspaceIdWithUrl, - storageAdapter: AggregatorStorageAdapter, + storageAdapter: StorageAdapter, client: TxOperations, logger: ModelLogger = consoleModelLogger, progress: (value: number) => Promise @@ -263,7 +237,6 @@ export async function upgradeModel ( storageAdapter: StorageAdapter, migrateOperations: [string, MigrateOperation][], logger: ModelLogger = consoleModelLogger, - skipTxUpdate: boolean = false, progress: (value: number) => Promise, forceIndexes: boolean = false ): Promise { @@ -309,15 +282,6 @@ export async function upgradeModel ( } }) - if (!skipTxUpdate) { - if (pipeline.context.lowLevelStorage === undefined) { - throw new PlatformError(unknownError('Low level storage is not available')) - } - logger.log('removing model...', { workspaceId: workspaceId.name }) - await progress(10) - logger.log('model transactions inserted.', { workspaceId: workspaceId.name, count: txes.length }) - await progress(20) - } const { migrateClient, migrateState } = await prepareMigrationClient( pipeline, hierarchy, @@ -366,21 +330,6 @@ export async function upgradeModel ( { state: 'indexes-v5', func: upgradeIndexes - }, - { - state: 'delete-model', - func: async (client) => { - const model = await client.find(DOMAIN_TX, { objectSpace: core.space.Model }) - - // Ignore Employee accounts. - const isUserTx = (it: Tx): boolean => - it.modifiedBy !== core.account.System || - (it as TxCUD).objectClass === 'contact:class:Person' || - (it as TxCUD).objectClass === 'contact:class:PersonAccount' - - const toDelete = model.filter((it) => !isUserTx(it)).map((it) => it._id) - await client.deleteMany(DOMAIN_TX, { _id: { $in: toDelete } }) - } } ]) }) diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index 4a15468c74..3b2aa0e263 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -15,7 +15,7 @@ import core, { } from '@hcengineering/core' import { ModelLogger } from '@hcengineering/model' import { makeRank } from '@hcengineering/rank' -import { AggregatorStorageAdapter } from '@hcengineering/server-core' +import type { StorageAdapter } from '@hcengineering/server-core' import { jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text' import { v4 as uuid } from 'uuid' @@ -91,7 +91,7 @@ export class WorkspaceInitializer { constructor ( private readonly ctx: MeasureContext, - private readonly storageAdapter: AggregatorStorageAdapter, + private readonly storageAdapter: StorageAdapter, private readonly wsUrl: WorkspaceIdWithUrl, private readonly client: TxOperations ) {} diff --git a/server/workspace-service/src/service.ts b/server/workspace-service/src/service.ts index 9a0c58f757..e348acdb32 100644 --- a/server/workspace-service/src/service.ts +++ b/server/workspace-service/src/service.ts @@ -13,29 +13,29 @@ // limitations under the License. // import { - type BrandingMap, - systemAccountEmail, type BaseWorkspaceInfo, + type BrandingMap, type Data, type MeasureContext, type Tx, type Version, getBranding, - getWorkspaceId + getWorkspaceId, + systemAccountEmail } from '@hcengineering/core' import { type MigrateOperation, type ModelLogger } from '@hcengineering/model' import { getPendingWorkspace, updateWorkspaceInfo, - workerHandshake, + withRetryConnUntilSuccess, withRetryConnUntilTimeout, - withRetryConnUntilSuccess + workerHandshake } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' import { FileModelLogger } from '@hcengineering/server-tool' import path from 'path' -import { upgradeWorkspace, createWorkspace } from './ws-operations' +import { createWorkspace, upgradeWorkspace } from './ws-operations' export interface WorkspaceOptions { errorHandler: (workspace: BaseWorkspaceInfo, error: any) => Promise diff --git a/server/workspace-service/src/ws-operations.ts b/server/workspace-service/src/ws-operations.ts index df59258171..47c41fa1db 100644 --- a/server/workspace-service/src/ws-operations.ts +++ b/server/workspace-service/src/ws-operations.ts @@ -16,23 +16,13 @@ import core, { } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation, type ModelLogger } from '@hcengineering/model' import { getTransactorEndpoint } from '@hcengineering/server-client' +import { SessionDataImpl, type Pipeline, type StorageAdapter } from '@hcengineering/server-core' import { - DummyFullTextAdapter, - SessionDataImpl, - type Pipeline, - type PipelineFactory, - type StorageAdapter, - type StorageConfiguration -} from '@hcengineering/server-core' -import { - createIndexStages, - createServerPipeline, getServerPipeline, getTxAdapterFactory, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' -import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { generateToken } from '@hcengineering/server-token' import { initializeWorkspace, initModel, prepareTools, updateModel, upgradeModel } from '@hcengineering/server-tool' @@ -130,9 +120,10 @@ export async function createWorkspace ( const dbUrls = mongodbUri !== undefined && dbUrl !== mongodbUri ? `${dbUrl};${mongodbUri}` : dbUrl const hierarchy = new Hierarchy() const modelDb = new ModelDb(hierarchy) + registerServerPlugins() + registerStringLoaders() - const storageConfig: StorageConfiguration = storageConfigFromEnv() - const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri) + const { pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, wsUrl) try { const txFactory = getTxAdapterFactory(ctx, dbUrls, wsUrl, null, { @@ -146,57 +137,12 @@ export async function createWorkspace ( const txAdapter = await txFactory(ctx, hierarchy, dbUrl ?? mongodbUri, wsId, modelDb, storageAdapter) await childLogger.withLog('init-workspace', {}, async (ctx) => { - const deleteModelFirst = workspaceInfo.mode === 'creating' - - await initModel( - ctx, - wsId, - txes, - txAdapter, - storageAdapter, - ctxModellogger, - async (value) => { - await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10)) - }, - deleteModelFirst - ) + await initModel(ctx, wsId, txes, txAdapter, storageAdapter, ctxModellogger, async (value) => { + await handleWsEvent?.('progress', version, 10 + Math.round((Math.min(value, 100) / 100) * 10)) + }) }) - registerServerPlugins() - registerStringLoaders() - const factory: PipelineFactory = createServerPipeline( - ctx, - dbUrls, - txes, - { - externalStorage: storageAdapter, - fullTextUrl: 'http://localhost:9200', - indexParallel: 0, - indexProcessing: 0, - rekoniUrl: '', - usePassedCtx: true - }, - { - fulltextAdapter: { - factory: async () => new DummyFullTextAdapter(), - url: '', - stages: (adapter, storage, storageAdapter, contentAdapter) => - createIndexStages( - ctx.newChild('stages', {}), - wsUrl, - branding, - adapter, - storage, - storageAdapter, - contentAdapter, - 0, - 0 - ) - } - } - ) - const pipeline = await factory(ctx, wsUrl, true, () => {}, null) - const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.System) + const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.ConfigUser) await updateModel(ctx, wsId, migrationOperation, client, pipeline, ctxModellogger, async (value) => { await handleWsEvent?.('progress', version, 20 + Math.round((Math.min(value, 100) / 100) * 10)) @@ -208,12 +154,14 @@ export async function createWorkspace ( await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 60)) }) - await upgradeWorkspace( + await upgradeWorkspaceWith( ctx, version, txes, migrationOperation, workspaceInfo, + pipeline, + storageAdapter, ctxModellogger, async (event, version, value) => { ctx.info('Init script progress', { event, value }) @@ -223,12 +171,11 @@ export async function createWorkspace ( false ) - await pipeline.close() - await handleWsEvent?.('create-done', version, 100, '') } catch (err: any) { await handleWsEvent?.('ping', version, 0, `Create failed: ${err.message}`) } finally { + await pipeline.close() await storageAdapter.close() } } finally { @@ -256,6 +203,67 @@ export async function upgradeWorkspace ( forceUpdate: boolean = true, forceIndexes: boolean = false, external: boolean = false +): Promise { + const { mongodbUri, dbUrl } = prepareTools([]) + if (mongodbUri === undefined) { + throw new Error('No MONGO_URL specified') + } + let pipeline: Pipeline | undefined + let storageAdapter: StorageAdapter | undefined + + registerServerPlugins() + registerStringLoaders() + try { + ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, { + name: ws.workspace, + workspaceName: ws.workspaceName ?? '', + workspaceUrl: ws.workspaceUrl ?? '' + })) + if (pipeline === undefined || storageAdapter === undefined) { + return + } + + await upgradeWorkspaceWith( + ctx, + version, + txes, + migrationOperation, + ws, + pipeline, + storageAdapter, + logger, + handleWsEvent, + forceUpdate, + forceIndexes, + external + ) + } finally { + await pipeline?.close() + await storageAdapter?.close() + } +} + +/** + * @public + */ +export async function upgradeWorkspaceWith ( + ctx: MeasureContext, + version: Data, + txes: Tx[], + migrationOperation: [string, MigrateOperation][], + ws: BaseWorkspaceInfo, + pipeline: Pipeline, + storageAdapter: StorageAdapter, + logger: ModelLogger = consoleModelLogger, + handleWsEvent?: ( + event: 'upgrade-started' | 'progress' | 'upgrade-done' | 'ping', + version: Data, + progress: number, + message?: string + ) => Promise, + forceUpdate: boolean = true, + forceIndexes: boolean = false, + external: boolean = false ): Promise { const versionStr = versionToString(version) @@ -282,20 +290,12 @@ export async function upgradeWorkspace ( void handleWsEvent?.('progress', version, progress) }, 5000) - const { mongodbUri, dbUrl } = prepareTools([]) - if (mongodbUri === undefined) { - throw new Error('No MONGO_URL specified') - } - const wsUrl: WorkspaceIdWithUrl = { name: ws.workspace, workspaceName: ws.workspaceName ?? '', workspaceUrl: ws.workspaceUrl ?? '' } - let pipeline: Pipeline | undefined - let storageAdapter: StorageAdapter | undefined try { - ;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, mongodbUri, dbUrl, wsUrl)) const contextData = new SessionDataImpl( systemAccountEmail, 'backup', @@ -320,7 +320,6 @@ export async function upgradeWorkspace ( storageAdapter, migrationOperation, logger, - false, async (value) => { progress = value }, @@ -333,8 +332,6 @@ export async function upgradeWorkspace ( await handleWsEvent?.('ping', version, 0, `Upgrade failed: ${err.message}`) throw err } finally { - await pipeline?.close() - await storageAdapter?.close() clearInterval(updateProgressHandle) } }