From db94cab1e0e35dc1a8803458a02ad0d5170989b6 Mon Sep 17 00:00:00 2001 From: Alexey Zinoviev Date: Mon, 2 Sep 2024 20:12:04 +0400 Subject: [PATCH] UBERF-7684: Workspace service (#6460) Signed-off-by: Alexey Zinoviev --- .vscode/launch.json | 34 +- common/config/rush/pnpm-lock.yaml | 87 ++ dev/docker-compose.yaml | 26 + dev/tool/package.json | 1 + dev/tool/src/index.ts | 128 ++- packages/core/src/classes.ts | 6 +- packages/core/src/utils.ts | 7 + .../src/components/SelectWorkspace.svelte | 4 +- plugins/login/src/index.ts | 10 +- .../src/components/WorkbenchApp.svelte | 2 +- plugins/workbench-resources/src/connect.ts | 10 +- pods/account/src/__start.ts | 10 +- pods/authProviders/src/github.ts | 4 +- pods/authProviders/src/google.ts | 4 +- pods/authProviders/src/token.ts | 4 +- pods/authProviders/src/utils.ts | 7 - pods/workspace/.eslintrc.js | 7 + pods/workspace/.npmignore | 4 + pods/workspace/Dockerfile | 15 + pods/workspace/build.sh | 20 + pods/workspace/config/rig.json | 5 + pods/workspace/jest.config.js | 7 + pods/workspace/package.json | 61 ++ pods/workspace/src/__start.ts | 48 + pods/workspace/tsconfig.json | 10 + qms-tests/docker-compose.yaml | 17 + rush.json | 10 + .../notification-resources/src/utils.ts | 6 +- server/account-service/src/index.ts | 45 +- .../src/__tests__/account.test_skip.ts | 3 +- server/account/src/index.ts | 2 - server/account/src/operations.ts | 858 ++++++++++-------- server/account/src/service.ts | 197 ---- server/client/src/account.ts | 157 ++++ server/client/src/blob.ts | 225 +++++ server/client/src/client.ts | 72 ++ server/client/src/index.ts | 343 +------ server/tool/src/index.ts | 28 +- server/workspace-service/.eslintrc.js | 7 + server/workspace-service/.npmignore | 4 + server/workspace-service/build.sh | 20 + server/workspace-service/config/rig.json | 5 + server/workspace-service/jest.config.js | 7 + server/workspace-service/package.json | 59 ++ server/workspace-service/src/index.ts | 137 +++ server/workspace-service/src/service.ts | 258 ++++++ server/workspace-service/src/ws-operations.ts | 259 ++++++ server/workspace-service/tsconfig.json | 10 + server/ws/src/server.ts | 8 +- services/ai-bot/pod-ai-bot/src/controller.ts | 2 +- .../pod-analytics-collector/src/collector.ts | 2 +- tests/docker-compose.yaml | 17 + 52 files changed, 2199 insertions(+), 1080 deletions(-) create mode 100644 pods/workspace/.eslintrc.js create mode 100644 pods/workspace/.npmignore create mode 100644 pods/workspace/Dockerfile create mode 100755 pods/workspace/build.sh create mode 100644 pods/workspace/config/rig.json create mode 100644 pods/workspace/jest.config.js create mode 100644 pods/workspace/package.json create mode 100644 pods/workspace/src/__start.ts create mode 100644 pods/workspace/tsconfig.json delete mode 100644 server/account/src/service.ts create mode 100644 server/client/src/account.ts create mode 100644 server/client/src/blob.ts create mode 100644 server/client/src/client.ts create mode 100644 server/workspace-service/.eslintrc.js create mode 100644 server/workspace-service/.npmignore create mode 100755 server/workspace-service/build.sh create mode 100644 server/workspace-service/config/rig.json create mode 100644 server/workspace-service/jest.config.js create mode 100644 server/workspace-service/package.json create mode 100644 server/workspace-service/src/index.ts create mode 100644 server/workspace-service/src/service.ts create mode 100644 server/workspace-service/src/ws-operations.ts create mode 100644 server/workspace-service/tsconfig.json diff --git a/.vscode/launch.json b/.vscode/launch.json index da37746870..84d2a272c8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -48,7 +48,7 @@ "ACCOUNTS_URL": "http://localhost:3000", // "SERVER_PROVIDER":"uweb" "SERVER_PROVIDER":"ws", - "MODEL_VERSION": "", + "MODEL_VERSION": "v0.6.287", "ELASTIC_INDEX_NAME": "local_storage_index", "UPLOAD_URL": "/files", @@ -75,17 +75,47 @@ "TRANSACTOR_URL": "ws://localhost:3333", "ACCOUNT_PORT": "3000", "FRONT_URL": "http://localhost:8080", - "outputCapture": "std", "SES_URL": "", "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost" + // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml", + // "INIT_WORKSPACE": "onboarding", }, + "runtimeVersion": "20", "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "sourceMaps": true, + "outputCapture": "std", "cwd": "${workspaceRoot}/pods/account", "protocol": "inspector" }, + { + "name": "Debug Workspace", + "type": "node", + "request": "launch", + "args": ["src/__start.ts"], + "env": { + "MONGO_URL": "mongodb://localhost:27017", + "SERVER_SECRET": "secret", + "TRANSACTOR_URL": "ws://localhost:3333", + "ACCOUNTS_URL": "http://localhost:3000", + "FRONT_URL": "http://localhost:8080", + "SES_URL": "", + "MINIO_ACCESS_KEY": "minioadmin", + "MINIO_SECRET_KEY": "minioadmin", + "MINIO_ENDPOINT": "localhost", + "MODEL_VERSION": "v0.6.286", + // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml", + // "INIT_WORKSPACE": "onboarding", + "NOTIFY_INBOX_ONLY": "true" + }, + "runtimeVersion": "20", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "sourceMaps": true, + "outputCapture": "std", + "cwd": "${workspaceRoot}/pods/workspace", + "protocol": "inspector" + }, { "name": "Debug Front", "type": "node", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index ba45f4b3f5..868cbd724d 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -626,6 +626,9 @@ dependencies: '@rush-temp/pod-telegram-bot': specifier: file:./projects/pod-telegram-bot.tgz version: file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4) + '@rush-temp/pod-workspace': + specifier: file:./projects/pod-workspace.tgz + version: file:projects/pod-workspace.tgz '@rush-temp/preference': specifier: file:./projects/preference.tgz version: file:projects/preference.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -1064,6 +1067,9 @@ dependencies: '@rush-temp/workbench-resources': specifier: file:./projects/workbench-resources.tgz version: file:projects/workbench-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) + '@rush-temp/workspace-service': + specifier: file:./projects/workspace-service.tgz + version: file:projects/workspace-service.tgz '@sentry/node': specifier: ^7.103.0 version: 7.118.0 @@ -30132,6 +30138,46 @@ packages: - utf-8-validate dev: false + file:projects/pod-workspace.tgz: + resolution: {integrity: sha512-3hJxKzpd+0/zfr++zNy57willy+3axei4d1ehwmhxivlQh0/iKsV2RvKCMD7q74b/grO+gewyFY7XsVzBlkvrg==, tarball: file:projects/pod-workspace.tgz} + name: '@rush-temp/pod-workspace' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) + cross-env: 7.0.3 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + mongodb: 6.8.0 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) + typescript: 5.3.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@babel/core' + - '@jest/types' + - '@mongodb-js/zstd' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - gcp-metadata + - kerberos + - mongodb-client-encryption + - node-notifier + - snappy + - socks + - supports-color + dev: false + file:projects/preference-assets.tgz(esbuild@0.20.1)(ts-node@10.9.2): resolution: {integrity: sha512-VlBSKBg3XmuMLtxNAS703aS+dhhb5a7H5Ns2nzhhv7w3KlAqtwp6cQ5VLxceNuRaPbTtI+2K+YkjFb2S1ld5VQ==, tarball: file:projects/preference-assets.tgz} id: file:projects/preference-assets.tgz @@ -35221,3 +35267,44 @@ packages: - supports-color - ts-node dev: false + + file:projects/workspace-service.tgz: + resolution: {integrity: sha512-WUCtvidfvVcahSFbmbtTZeGvedNNsG4RERSfnG+MWuDnDfyFAYnpBVest9gyO2/jH4cZ/AxeE1tgZKWPqCpSeg==, tarball: file:projects/workspace-service.tgz} + name: '@rush-temp/workspace-service' + version: 0.0.0 + dependencies: + '@koa/cors': 5.0.0 + '@types/jest': 29.5.12 + '@types/koa': 2.15.0 + '@types/koa-bodyparser': 4.3.12 + '@types/koa-router': 7.4.8 + '@types/koa__cors': 5.0.0 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) + cross-env: 7.0.3 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + koa: 2.15.3 + koa-bodyparser: 4.4.1 + koa-router: 12.0.1 + mongodb: 6.8.0 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) + typescript: 5.3.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + dev: false diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 9b045a3d1a..37deb7113e 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -76,6 +76,32 @@ services: # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped + workspace: + image: hardcoreeng/workspace + # deploy: + # mode: replicated + # replicas: 3 + links: + - mongodb + - minio + volumes: + - ./branding.json:/var/cfg/branding.json + environment: + # - WS_OPERATION=create + - SERVER_SECRET=secret + - MONGO_URL=mongodb://mongodb:27017?compressors=snappy + - TRANSACTOR_URL=ws://transactor:3333;ws://localhost:3333 + - SES_URL= + - STORAGE_CONFIG=${STORAGE_CONFIG} + - FRONT_URL=http://localhost:8087 + - RESERVED_DB_NAMES=telegram,gmail,github + - MODEL_ENABLED=* + - ACCOUNTS_URL=http://account:3000 + - BRANDING_PATH=/var/cfg/branding.json + - NOTIFY_INBOX_ONLY=true + # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml + # - INIT_WORKSPACE=onboarding + restart: unless-stopped collaborator: image: hardcoreeng/collaborator extra_hosts: diff --git a/dev/tool/package.json b/dev/tool/package.json index 79dd4519fa..ca8e811c1c 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -55,6 +55,7 @@ "dependencies": { "@elastic/elasticsearch": "^7.14.0", "@hcengineering/account": "^0.6.0", + "@hcengineering/workspace-service": "^0.6.0", "@hcengineering/attachment": "^0.6.14", "@hcengineering/calendar": "^0.6.24", "@hcengineering/chunter": "^0.6.20", diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index e8ec0c284f..bcfba61ec7 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -19,7 +19,6 @@ import accountPlugin, { assignWorkspace, confirmEmail, createAcc, - createWorkspace, dropAccount, dropWorkspace, dropWorkspaceFull, @@ -32,10 +31,11 @@ import accountPlugin, { replacePassword, setAccountAdmin, setRole, - UpgradeWorker, - upgradeWorkspace, + updateWorkspace, + createWorkspace as createWorkspaceRecord, type Workspace } from '@hcengineering/account' +import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import { setMetadata } from '@hcengineering/platform' import { backup, @@ -48,7 +48,8 @@ import { } from '@hcengineering/server-backup' import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' -import toolPlugin from '@hcengineering/server-tool' +import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' +import path from 'path' import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage' import { program, type Command } from 'commander' @@ -327,19 +328,28 @@ export function devTool ( .action(async (workspace, cmd: { email: string, workspaceName: string, init?: string, branding?: string }) => { const { mongodbUri, txes, version, migrateOperations } = prepareTools() await withDatabase(mongodbUri, async (db) => { - await createWorkspace( - toolCtx, - version, - txes, - migrateOperations, - db, - cmd.init !== undefined || cmd.branding !== undefined - ? { initWorkspace: cmd.init, key: cmd.branding ?? 'huly' } - : null, - cmd.email, - cmd.workspaceName, - workspace - ) + const measureCtx = new MeasureMetricsContext('create-workspace', {}) + const brandingObj = + cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null + const wsInfo = await createWorkspaceRecord(measureCtx, db, brandingObj, cmd.email, cmd.workspaceName, workspace) + + // update the record so it's not taken by one of the workers for the next 60 seconds + await updateWorkspace(db, wsInfo, { + mode: 'creating', + progress: 0, + lastProcessingTime: Date.now() + 1000 * 60 + }) + + await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations) + + await updateWorkspace(db, wsInfo, { + mode: 'active', + progress: 100, + disabled: false, + version + }) + + console.log('create-workspace done') }) }) @@ -387,30 +397,36 @@ export function devTool ( throw new Error(`workspace ${workspace} not found`) } - const measureCtx = new MeasureMetricsContext('upgrade', {}) + const measureCtx = new MeasureMetricsContext('upgrade-workspace', {}) await upgradeWorkspace( measureCtx, version, txes, migrateOperations, - db, - info.workspaceUrl ?? info.workspace, + info, consoleModelLogger, + async () => {}, cmd.force, - cmd.indexes + cmd.indexes, + true ) - console.log(metricsToString(measureCtx.metrics, 'upgrade', 60), {}) - console.log('upgrade done') + + await updateWorkspace(db, info, { + mode: 'active', + progress: 100, + version + }) + + console.log(metricsToString(measureCtx.metrics, 'upgrade', 60)) + console.log('upgrade-workspace done') }) }) program .command('upgrade') .description('upgrade') - .option('-p|--parallel ', 'Parallel upgrade', '0') .option('-l|--logs ', 'Default logs folder', './logs') - .option('-r|--retry ', 'Number of apply retries', '0') .option('-i|--ignore [ignore]', 'Ignore workspaces', '') .option( '-c|--console', @@ -418,29 +434,45 @@ export function devTool ( false ) .option('-f|--force [force]', 'Force update', false) - .action( - async (cmd: { - parallel: string - logs: string - retry: string - force: boolean - console: boolean - ignore: string - }) => { - const { mongodbUri, version, txes, migrateOperations } = prepareTools() - await withDatabase(mongodbUri, async (db, client) => { - const worker = new UpgradeWorker(db, client, version, txes, migrateOperations) - await worker.upgradeAll(toolCtx, { - errorHandler: async (ws, err) => {}, - force: cmd.force, - console: cmd.console, - logs: cmd.logs, - parallel: parseInt(cmd.parallel ?? '1'), - ignore: cmd.ignore - }) - }) - } - ) + .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string }) => { + const { mongodbUri, version, txes, migrateOperations } = prepareTools() + await withDatabase(mongodbUri, async (db, client) => { + const workspaces = (await listWorkspacesRaw(db)).filter((ws) => !cmd.ignore.includes(ws.workspace)) + workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + const measureCtx = new MeasureMetricsContext('upgrade', {}) + + for (const ws of workspaces) { + try { + const logger = cmd.console + ? consoleModelLogger + : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) + + await upgradeWorkspace( + measureCtx, + version, + txes, + migrateOperations, + ws, + logger, + async () => {}, + cmd.force, + false, + true + ) + + await updateWorkspace(db, ws, { + mode: 'active', + progress: 100, + version + }) + } catch (err: any) { + console.error(err) + } + } + + console.log('upgrade done') + }) + }) program .command('list-unused-workspaces') diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index e847294dfe..9d2f4f4413 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -652,6 +652,8 @@ export interface DomainIndexConfiguration extends Doc { skip?: string[] } +export type WorkspaceMode = 'pending-creation' | 'creating' | 'upgrading' | 'deleting' | 'active' + export interface BaseWorkspaceInfo { workspace: string // An uniq workspace name, Database names disabled?: boolean @@ -665,8 +667,8 @@ export interface BaseWorkspaceInfo { createdBy: string - creating?: boolean - createProgress?: number // Some progress + mode: WorkspaceMode + progress?: number // Some progress endpoint: string } diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index c8fc702086..bd8dcecbc1 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -47,6 +47,7 @@ import { TxOperations } from './operations' import { isPredicate } from './predicate' import { DocumentQuery, FindResult } from './storage' import { DOMAIN_TX } from './tx' +import { Branding, BrandingMap } from './server' function toHex (value: number, chars: number): string { const result = value.toString(16) @@ -807,3 +808,9 @@ export function isOwnerOrMaintainer (): boolean { export function hasAccountRole (acc: Account, targerRole: AccountRole): boolean { return roleOrder[acc.role] >= roleOrder[targerRole] } + +export function getBranding (brandings: BrandingMap, key: string | undefined): Branding | null { + if (key === undefined) return null + + return Object.values(brandings).find((branding) => branding.key === key) ?? null +} diff --git a/plugins/login-resources/src/components/SelectWorkspace.svelte b/plugins/login-resources/src/components/SelectWorkspace.svelte index a0bd2ee07d..5f6df4479c 100644 --- a/plugins/login-resources/src/components/SelectWorkspace.svelte +++ b/plugins/login-resources/src/components/SelectWorkspace.svelte @@ -125,8 +125,8 @@
{wsName} - {#if workspace.creating === true} - ({workspace.createProgress}%) + {#if workspace.mode === 'creating'} + ({workspace.progress}%) {/if} {#if isAdmin && wsName !== workspace.workspace} diff --git a/plugins/login/src/index.ts b/plugins/login/src/index.ts index 05924f9114..c4288de8cd 100644 --- a/plugins/login/src/index.ts +++ b/plugins/login/src/index.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { AccountRole, Doc, Ref, Timestamp } from '@hcengineering/core' +import { AccountRole, Doc, Ref, Timestamp, WorkspaceMode } from '@hcengineering/core' import type { Asset, IntlString, Metadata, Plugin, Resource, Status } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import type { AnyComponent } from '@hcengineering/ui' @@ -31,8 +31,8 @@ export interface Workspace { workspaceName?: string // A company name workspaceId: string // A unique identifier for the workspace - creating?: boolean - createProgress?: number + mode?: WorkspaceMode + progress?: number lastVisit: number } @@ -43,8 +43,8 @@ export interface Workspace { export interface WorkspaceLoginInfo extends LoginInfo { workspace: string workspaceId: string - creating?: boolean - createProgress?: number + mode?: WorkspaceMode + progress?: number } /** diff --git a/plugins/workbench-resources/src/components/WorkbenchApp.svelte b/plugins/workbench-resources/src/components/WorkbenchApp.svelte index 43dbfb090b..1f2bede397 100644 --- a/plugins/workbench-resources/src/components/WorkbenchApp.svelte +++ b/plugins/workbench-resources/src/components/WorkbenchApp.svelte @@ -57,7 +57,7 @@ {#key $location.path[1]} {#await connect(getMetadata(workbenchRes.metadata.PlatformTitle) ?? 'Platform')} - {#if ($workspaceCreating ?? -1) > 0} + {#if ($workspaceCreating ?? -1) >= 0}