From 8d471bed1b7f63114a4df2c6130841343e4f6ebd Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Fri, 5 Jul 2024 19:26:02 +0500 Subject: [PATCH] Init workspace via script (#6007) Signed-off-by: Denis Bykhov --- common/config/rush/pnpm-lock.yaml | 12 +- packages/text/src/nodes/image.ts | 9 +- server/account/src/operations.ts | 72 ++-------- server/tool/package.json | 3 + server/tool/src/index.ts | 61 ++++++++ server/tool/src/initializer.ts | 224 ++++++++++++++++++++++++++++++ 6 files changed, 312 insertions(+), 69 deletions(-) create mode 100644 server/tool/src/initializer.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 29a2726a83..0295cdbeef 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -22580,7 +22580,7 @@ packages: dev: false file:projects/model.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-st7Z7xVdtaQf3+WTi8BcUYBwuZzi+2iWF+gyYIFi6AI1beJh4Mdjtlu6NYfEGRkHONpL/czV4laJly2Dxkc1Ew==, tarball: file:projects/model.tgz} + resolution: {integrity: sha512-8Rge/rE550GSKnZADVxI1ykybyTMnaYeRp872Miu2IVHPknVie3N2ZZaLlH0T6M2tQBxqlnAE8VOr1ey9KNWIQ==, tarball: file:projects/model.tgz} id: file:projects/model.tgz name: '@rush-temp/model' version: 0.0.0 @@ -23062,7 +23062,7 @@ packages: dev: false file:projects/pod-server.tgz: - resolution: {integrity: sha512-GjJqgPc/ux0jjGydLB1tmiwo9f4tTZL4KECyHmiLemLahnZojN8AIIFQTsqUkiFh0dwULxPvJkRWnYdGLA0iZg==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-e3S57PlJ3ME1jMupiLtYmdZFewiWjjAeY/uyOZ1yENsAf6WnET/uLgDpgRqHy/Kw/rLPvyAfWQR69mTFXpAOZA==, tarball: file:projects/pod-server.tgz} name: '@rush-temp/pod-server' version: 0.0.0 dependencies: @@ -24091,7 +24091,7 @@ packages: dev: false file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Zpr44q67hLKNSbFvWVJEITtDNValbilYERD5KeWImI44rVHVotGtAISj0GqAtGr0mzGLuosOyP6dhqs/+bfOgw==, tarball: file:projects/server-backup.tgz} + resolution: {integrity: sha512-LuaW1naKF+DcYgzDYKohAfO1mMtqzpnglsAfNyAl0hNYUWA/F7M5+/qcy0LpHuwRTltol+RWBBLVeLMzMX9A/Q==, tarball: file:projects/server-backup.tgz} id: file:projects/server-backup.tgz name: '@rush-temp/server-backup' version: 0.0.0 @@ -24969,7 +24969,7 @@ packages: dev: false file:projects/server-notification-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-nQJkxhWMtbfdsgmIvt3/qQ/Dz2dsVx6JI5MbNyrJmixplMRt+cNGb08lAVJLsZkFF2UReTss6jNyKLx8WSki/w==, tarball: file:projects/server-notification-resources.tgz} + resolution: {integrity: sha512-DhxiRHwKgwqkX4WP2ZmqGL7fHs0Ev3rtxHT/uylk5SPCPbKmBe20xyEzheSahP0Vm6kIS9XerGQuxGrI31eVBw==, tarball: file:projects/server-notification-resources.tgz} id: file:projects/server-notification-resources.tgz name: '@rush-temp/server-notification-resources' version: 0.0.0 @@ -25631,12 +25631,13 @@ packages: dev: false file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.3): - resolution: {integrity: sha512-ErBexvPUdBHDpLANxgrqNef832SQXZicJp7U+UDiqr9EsTjKjJ71ZgTYlGtj7unI6XfBEoUlH19cAu7ExV3veA==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-7THPdPOTi6rdVo4B0AUJz4mx3urr/lNWZ88ZzgUjPXTl7T2w2PyplAEoCFvxN8A184+6KM9Wb0trUlnftH72fA==, tarball: file:projects/server-tool.tgz} id: file:projects/server-tool.tgz name: '@rush-temp/server-tool' version: 0.0.0 dependencies: '@types/jest': 29.5.12 + '@types/uuid': 8.3.4 '@types/ws': 8.5.10 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) @@ -25651,6 +25652,7 @@ packages: prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 + uuid: 8.3.2 ws: 8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.3) transitivePeerDependencies: - '@aws-sdk/credential-providers' diff --git a/packages/text/src/nodes/image.ts b/packages/text/src/nodes/image.ts index d6770c423a..ee616d6a47 100644 --- a/packages/text/src/nodes/image.ts +++ b/packages/text/src/nodes/image.ts @@ -94,7 +94,6 @@ export const ImageNode = Node.create({ 'data-type': this.name, 'data-align': node.attrs.align } - const imgAttributes = mergeAttributes( { 'data-type': this.name @@ -114,7 +113,6 @@ export const ImageNode = Node.create({ const container = document.createElement('div') const imgElement = document.createElement('img') container.append(imgElement) - const divAttributes = { class: 'text-editor-image-container', 'data-type': this.name, @@ -149,6 +147,13 @@ export const ImageNode = Node.create({ imgElement.srcset = val.srcset }) } + } else { + if (imgAttributes.srcset != null) { + imgElement.srcset = imgAttributes.srcset + } + if (imgAttributes.src != null) { + imgElement.src = imgAttributes.src + } } return { diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 50e1774b3a..84e5704816 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -46,11 +46,9 @@ import core, { type Branding } from '@hcengineering/core' import { consoleModelLogger, MigrateOperation, ModelLogger } from '@hcengineering/model' -import { getModelVersion } from '@hcengineering/model-all' import platform, { getMetadata, PlatformError, Severity, Status, translate } from '@hcengineering/platform' -import { cloneWorkspace } from '@hcengineering/server-backup' import { decodeToken, generateToken } from '@hcengineering/server-token' -import toolPlugin, { connect, getStorageAdapter, initModel, upgradeModel } from '@hcengineering/server-tool' +import toolPlugin, { connect, initializeWorkspace, initModel, upgradeModel } from '@hcengineering/server-tool' import { pbkdf2Sync, randomBytes } from 'crypto' import { Binary, Db, Filter, ObjectId, type MongoClient } from 'mongodb' import fetch from 'node-fetch' @@ -940,69 +938,19 @@ export async function createWorkspace ( childLogger.error(msg, data) } } - let model: Tx[] = [] + const model: Tx[] = [] try { - const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) const wsId = getWorkspaceId(workspaceInfo.workspace, productId) - // We should not try to clone INIT_WS into INIT_WS during it's creation. - let initWSInfo: Workspace | undefined - if (initWS !== undefined) { - initWSInfo = (await getWorkspaceById(db, productId, initWS)) ?? undefined - } - if (initWS !== undefined && initWSInfo !== undefined && initWS !== workspaceInfo.workspace) { - // Just any valid model for transactor to be able to function - await childLogger.with('init-model', {}, async (ctx) => { - await initModel(ctx, getTransactor(), wsId, txes, [], ctxModellogger, async (value) => { - await updateInfo({ createProgress: Math.round((Math.min(value, 100) / 100) * 20) }) - }) + await childLogger.withLog('init-workspace', {}, async (ctx) => { + await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger, async (value) => { + await updateInfo({ createProgress: 10 + Math.round((Math.min(value, 100) / 100) * 20) }) }) + }) - await updateInfo({ createProgress: 20 }) - - // Clone init workspace. - await cloneWorkspace( - childLogger, - getTransactor(), - getWorkspaceId(initWS, productId), - getWorkspaceId(workspaceInfo.workspace, productId), - true, - async (value) => { - await updateInfo({ createProgress: 20 + Math.round((Math.min(value, 100) / 100) * 70) }) - }, - getStorageAdapter() - ) - const modelVersion = getModelVersion() - await updateInfo({ createProgress: 90 }) - - // Skip tx update if version of init workspace are proper one. - const skipTxUpdate = - versionToString(modelVersion) === versionToString(initWSInfo.version ?? { major: 0, minor: 0, patch: 0 }) - model = await childLogger.withLog( - 'upgrade-model', - {}, - async (ctx) => - await upgradeModel( - ctx, - getTransactor(), - wsId, - txes, - migrationOperation, - ctxModellogger, - skipTxUpdate, - async (value) => { - await updateInfo({ createProgress: Math.round(90 + (Math.min(value, 100) / 100) * 10) }) - } - ) - ) - await updateInfo({ createProgress: 99 }) - } else { - await childLogger.withLog('init-workspace', {}, async (ctx) => { - await initModel(ctx, getTransactor(), wsId, txes, migrationOperation, ctxModellogger, async (value) => { - await updateInfo({ createProgress: Math.round(Math.min(value, 100)) }) - }) - }) - } + await initializeWorkspace(ctx, branding, getTransactor(), wsId, ctxModellogger, async (value) => { + await updateInfo({ createProgress: 30 + Math.round((Math.min(value, 100) / 100) * 65) }) + }) } catch (err: any) { Analytics.handleError(err) return { workspaceInfo, err, client: null as any } @@ -1124,7 +1072,7 @@ export const createUserWorkspace = notifyHandler, async (workspace, model) => { const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) - const shouldUpdateAccount = initWS !== undefined && (await getWorkspaceById(db, productId, initWS)) !== null + const shouldUpdateAccount = initWS !== undefined const client = await connect( getTransactor(), getWorkspaceId(workspace.workspace, productId), diff --git a/server/tool/package.json b/server/tool/package.json index ea85b0a050..407d5d5972 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -32,6 +32,7 @@ "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", "typescript": "^5.3.3", + "@types/uuid": "^8.3.1", "@types/ws": "^8.5.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", @@ -46,6 +47,8 @@ "@hcengineering/client": "^0.6.18", "ws": "^8.16.0", "@hcengineering/model": "^0.6.11", + "@hcengineering/rank": "^0.6.4", + "uuid": "^8.3.2", "@hcengineering/server-token": "^0.6.11", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server": "^0.6.4", diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 66e41a2168..01b0fd4536 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -16,6 +16,7 @@ import contact from '@hcengineering/contact' import core, { BackupClient, + Branding, Client as CoreClient, DOMAIN_MIGRATION, DOMAIN_MODEL, @@ -37,9 +38,11 @@ import { DomainIndexHelperImpl, StorageAdapter, StorageConfiguration } from '@hc import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { Db, Document } from 'mongodb' import { connect } from './connect' +import { createWorkspaceData, InitScript } from './initializer' import toolPlugin from './plugin' import { MigrateClientImpl } from './upgrade' +import { getMetadata } from '@hcengineering/platform' import fs from 'fs' import path from 'path' @@ -47,6 +50,9 @@ export * from './connect' export * from './plugin' export { toolPlugin as default } +export const CONFIG_DB = '%config' +const scriptsCol = 'initScripts' + export class FileModelLogger implements ModelLogger { handle: fs.WriteStream constructor (readonly file: string) { @@ -181,6 +187,61 @@ export async function initModel ( } } +/** + * @public + */ +export async function initializeWorkspace ( + ctx: MeasureContext, + branding: Branding | null, + transactorUrl: string, + workspaceId: WorkspaceId, + logger: ModelLogger = consoleModelLogger, + progress: (value: number) => Promise +): Promise { + const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) + if (initWS === undefined) return + + const { mongodbUri } = prepareTools([]) + + const _client = getMongoClient(mongodbUri) + const client = await _client.getClient() + let connection: (CoreClient & BackupClient) | undefined + const storageConfig: StorageConfiguration = storageConfigFromEnv() + const storageAdapter = buildStorageFromConfig(storageConfig, mongodbUri) + try { + const db = client.db(CONFIG_DB) + const scripts = await db.collection(scriptsCol).find({}).toArray() + let script: InitScript | undefined + if (initWS !== undefined) { + script = scripts.find((it) => it.name === initWS) + } + if (script === undefined) { + script = scripts.find((it) => it.default) + } + if (script === undefined) { + return + } + try { + connection = (await connect(transactorUrl, workspaceId, undefined, { + model: 'upgrade', + admin: 'true' + })) as unknown as CoreClient & BackupClient + await createWorkspaceData(ctx, connection, storageAdapter, workspaceId, script, logger, progress) + } catch (e: any) { + logger.error('error', { error: e }) + throw e + } + } catch (err: any) { + ctx.error('Failed to create workspace', { error: err }) + throw err + } finally { + await storageAdapter.close() + await connection?.sendForceClose() + await connection?.close() + _client.close() + } +} + export function getStorageAdapter (): StorageAdapter { const { mongodbUri } = prepareTools([]) diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts new file mode 100644 index 0000000000..8f4aff69aa --- /dev/null +++ b/server/tool/src/initializer.ts @@ -0,0 +1,224 @@ +import core, { + AttachedDoc, + Class, + Client, + Data, + Doc, + MeasureContext, + Mixin, + Ref, + Space, + TxOperations, + WorkspaceId +} from '@hcengineering/core' +import { ModelLogger } from '@hcengineering/model' +import { makeRank } from '@hcengineering/rank' +import { AggregatorStorageAdapter } from '@hcengineering/server-core' +import { v4 as uuid } from 'uuid' + +const fieldRegexp = /\${\S+?}/ + +export interface InitScript { + name: string + lang?: string + default: boolean + steps: InitStep[] +} + +export type InitStep = CreateStep | MixinStep | UpdateStep | FindStep | UploadStep + +export interface CreateStep { + type: 'create' + _class: Ref> + data: Props + resultVariable?: string +} + +export interface MixinStep { + type: 'mixin' + _class: Ref> + mixin: Ref> + data: Props +} + +export interface UpdateStep { + type: 'update' + _class: Ref> + data: Props +} + +export interface FindStep { + type: 'find' + _class: Ref> + query: Partial + resultVariable?: string +} + +export interface UploadStep { + type: 'upload' + fromUrl: string + contentType: string + size?: number + resultVariable?: string +} + +export type Props = Data & Partial & { space: Ref } + +const nextRank = '#nextRank' +const now = '#now' + +export async function createWorkspaceData ( + ctx: MeasureContext, + connection: Client, + storageAdapter: AggregatorStorageAdapter, + workspaceId: WorkspaceId, + script: InitScript, + logger: ModelLogger, + progress: (value: number) => Promise +): Promise { + const client = new TxOperations(connection, core.account.System) + const vars: Record = {} + for (let index = 0; index < script.steps.length; index++) { + const step = script.steps[index] + if (step.type === 'create') { + await processCreate(client, step, vars) + } else if (step.type === 'update') { + await processUpdate(client, step, vars) + } else if (step.type === 'mixin') { + await processMixin(client, step, vars) + } else if (step.type === 'find') { + await processFind(client, step, vars) + } else if (step.type === 'upload') { + await processUpload(ctx, storageAdapter, workspaceId, step, vars, logger) + } + + await progress(Math.round(((index + 1) * 100) / script.steps.length)) + } +} + +async function processUpload ( + ctx: MeasureContext, + storageAdapter: AggregatorStorageAdapter, + workspaceId: WorkspaceId, + step: UploadStep, + vars: Record, + logger: ModelLogger +): Promise { + try { + const id = uuid() + const resp = await fetch(step.fromUrl) + const buffer = Buffer.from(await resp.arrayBuffer()) + await storageAdapter.put(ctx, workspaceId, id, buffer, step.contentType, step.size) + if (step.resultVariable !== undefined) { + vars[step.resultVariable] = id + } + } catch (error) { + logger.error('Upload failed', error) + } +} + +async function processFind ( + client: TxOperations, + step: FindStep, + vars: Record +): Promise { + const query = fillProps(step.query, vars) + const res = await client.findOne(step._class, { ...(query as any) }) + if (res === undefined) { + throw new Error(`Document not found: ${JSON.stringify(query)}`) + } + if (step.resultVariable !== undefined) { + vars[step.resultVariable] = res + } +} + +async function processMixin ( + client: TxOperations, + step: MixinStep, + vars: Record +): Promise { + const data = fillProps(step.data, vars) + const { _id, space, ...props } = data + if (_id === undefined || space === undefined) { + throw new Error('Mixin step must have _id and space') + } + await client.createMixin(_id, step._class, space, step.mixin, props) +} + +async function processUpdate ( + client: TxOperations, + step: UpdateStep, + vars: Record +): Promise { + const data = fillProps(step.data, vars) + const { _id, space, ...props } = data + if (_id === undefined || space === undefined) { + throw new Error('Update step must have _id and space') + } + await client.updateDoc(step._class, space, _id as Ref, props) +} + +async function processCreate ( + client: TxOperations, + step: CreateStep, + vars: Record +): Promise { + const data = fillProps(step.data, vars) + const res = await create(client, step._class, data) + if (step.resultVariable !== undefined) { + vars[step.resultVariable] = res + } +} + +async function create (client: TxOperations, _class: Ref>, data: Props): Promise> { + const hierarchy = client.getHierarchy() + if (hierarchy.isDerived(_class, core.class.AttachedDoc)) { + const { space, attachedTo, attachedToClass, collection, ...props } = data as unknown as Props + if (attachedTo === undefined || space === undefined || attachedToClass === undefined || collection === undefined) { + throw new Error('Add collection step must have attachedTo, attachedToClass, collection and space') + } + return (await client.addCollection( + _class, + space, + attachedTo, + attachedToClass, + collection, + props + )) as unknown as Ref + } else { + const { space, ...props } = data + if (space === undefined) { + throw new Error('Create step must have space') + } + return await client.createDoc(_class, space, props as Data) + } +} + +function fillProps | Props> (data: P, vars: Record): P { + for (const key in data) { + let value = (data as any)[key] + if (typeof value === 'object') { + ;(data as any)[key] = fillProps(value, vars) + } else if (typeof value === 'string') { + if (value === nextRank) { + const rank = makeRank(vars[nextRank], undefined) + ;(data as any)[key] = rank + vars[nextRank] = rank + } else if (value === now) { + ;(data as any)[key] = new Date().getTime() + } else { + while (true) { + const matched = fieldRegexp.exec(value) + if (matched === null) break + const result = vars[matched[0]] + if (result !== undefined && typeof result === 'string') { + value = value.replaceAll(matched[0], result) + fieldRegexp.lastIndex = 0 + } + } + ;(data as any)[key] = value + } + } + } + return data +}