UBERF-4569 Move collaborator to platform (#4177)

Signed-off-by: Alexander Onnikov <alexander.onnikov@xored.com>

.vscode/launch.json

@@ -43,6 +43,7 @@
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "SERVER_SECRET": "secret",
+        "COLLABORATOR_URL": "ws://localhost:3078",
         "REKONI_URL": "http://localhost:4004",
         "FRONT_URL": "http://localhost:8080",
         // "SERVER_PROVIDER":"uweb"
@@ -80,6 +81,23 @@
       "cwd": "${workspaceRoot}/pods/account",
       "protocol": "inspector"
     },
+    {
+      "name": "Debug Collaborator",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/__start.ts"],
+      "env": {
+        "SERVER_SECRET": "secret",
+        "TRANSACTOR_URL": "ws://localhost:3333",
+        "MINIO_ACCESS_KEY": "minioadmin",
+        "MINIO_SECRET_KEY": "minioadmin",
+        "MINIO_ENDPOINT": "localhost"
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "sourceMaps": true,
+      "cwd": "${workspaceRoot}/pods/collaborator",
+      "protocol": "inspector"
+    },
     {
       "type": "node",
       "request": "launch",

common/config/rush/pnpm-lock.yaml

@@ -14,6 +14,9 @@ dependencies:
   '@hocuspocus/provider':
     specifier: ^2.5.0
     version: 2.6.1(bufferutil@4.0.7)(yjs@13.6.8)
+  '@hocuspocus/server':
+    specifier: ^2.5.0
+    version: 2.8.1(bufferutil@4.0.7)(yjs@13.6.8)
   '@koa/cors':
     specifier: ^3.1.0
     version: 3.4.3
@@ -86,6 +89,9 @@ dependencies:
   '@rush-temp/client-resources':
     specifier: file:./projects/client-resources.tgz
     version: file:projects/client-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)
+  '@rush-temp/collaborator':
+    specifier: file:./projects/collaborator.tgz
+    version: file:projects/collaborator.tgz(bufferutil@4.0.7)(svelte@4.2.5)
   '@rush-temp/contact':
     specifier: file:./projects/contact.tgz
     version: file:projects/contact.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)
@@ -3616,6 +3622,12 @@ packages:
       lib0: 0.2.86
     dev: false
 
+  /@hocuspocus/common@2.8.1:
+    resolution: {integrity: sha512-pNXBvmiNmF13X/t/VTNfYMyEnWkLJ9e1gTyc7vFozBTfN9EgX7rexT4INradC288wkQ5Ha1gZb908Q4XBucm9g==}
+    dependencies:
+      lib0: 0.2.88
+    dev: false
+
   /@hocuspocus/provider@2.6.1(bufferutil@4.0.7)(yjs@13.6.8):
     resolution: {integrity: sha512-ue9U8isfCO4nE8ddmZ5AmoLHNrUEldxcDJ8eBxaRFCtc56V1GEk93M4r69Bvu/YvOcsJaURpXrSaN5GyQNJVQw==}
     peerDependencies:
@@ -3632,6 +3644,24 @@ packages:
       - utf-8-validate
     dev: false
 
+  /@hocuspocus/server@2.8.1(bufferutil@4.0.7)(yjs@13.6.8):
+    resolution: {integrity: sha512-3BJFcShMQxB8+A1uPSdKTeL+UZvVg+XS6vbdcyC6yCy3ZScLGjCP+IPAgiNwa5xvcUEr64yp8LCroM+gZHyxbQ==}
+    peerDependencies:
+      y-protocols: ^1.0.6
+      yjs: ^13.6.8
+    dependencies:
+      '@hocuspocus/common': 2.8.1
+      async-lock: 1.4.0
+      kleur: 4.1.5
+      lib0: 0.2.86
+      uuid: 9.0.1
+      ws: 8.14.2(bufferutil@4.0.7)
+      yjs: 13.6.8
+    transitivePeerDependencies:
+      - bufferutil
+      - utf-8-validate
+    dev: false
+
   /@humanwhocodes/config-array@0.11.11:
     resolution: {integrity: sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==}
     engines: {node: '>=10.10.0'}
@@ -8070,6 +8100,10 @@ packages:
     requiresBuild: true
     dev: false
 
+  /async-lock@1.4.0:
+    resolution: {integrity: sha512-coglx5yIWuetakm3/1dsX9hxCNox22h7+V80RQOu2XUUMidtArxKoZoOtHUPuR84SycKTXzgGzAUR5hJxujyJQ==}
+    dev: false
+
   /async-value-promise@1.1.1:
     resolution: {integrity: sha512-c2RFDKjJle1rHa0YxN9Ysu97/QBu3Wa+NOejJxsX+1qVDJrkD3JL/GN1B3gaILAEXJXbu/4Z1lcoCHFESe/APA==}
     dependencies:
@@ -12508,6 +12542,11 @@ packages:
     engines: {node: '>=6'}
     dev: false
 
+  /kleur@4.1.5:
+    resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==}
+    engines: {node: '>=6'}
+    dev: false
+
   /known-css-properties@0.29.0:
     resolution: {integrity: sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==}
     dev: false
@@ -12733,6 +12772,14 @@ packages:
       isomorphic.js: 0.2.5
     dev: false
 
+  /lib0@0.2.88:
+    resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==}
+    engines: {node: '>=16'}
+    hasBin: true
+    dependencies:
+      isomorphic.js: 0.2.5
+    dev: false
+
   /libphonenumber-js@1.10.47:
     resolution: {integrity: sha512-b4t7VQDV29xx/ni+58yl9KWPGjnDLDXCeCTLrD4V8vDpObXZRZBrg7uX/HWZ7YXiJKqdBDGgc+barUUTNB6Slw==}
     dev: false
@@ -18072,6 +18119,56 @@ packages:
       - ts-node
     dev: false
 
+  file:projects/collaborator.tgz(bufferutil@4.0.7)(svelte@4.2.5):
+    resolution: {integrity: sha512-Aqxt81MeApDyPNFAK2c+YaKTe5kbBZdHrL/7UgrXClC3hqN7tVepxeWK77TaSz/Sk9/1+dOYphDBMBYK5jOxGQ==, tarball: file:projects/collaborator.tgz}
+    id: file:projects/collaborator.tgz
+    name: '@rush-temp/collaborator'
+    version: 0.0.0
+    dependencies:
+      '@hocuspocus/server': 2.8.1(bufferutil@4.0.7)(yjs@13.6.8)
+      '@types/body-parser': 1.19.3
+      '@types/compression': 1.7.3
+      '@types/cors': 2.8.14
+      '@types/express': 4.17.18
+      '@types/jest': 29.5.5
+      '@types/node': 16.11.68
+      '@types/ws': 8.5.6
+      '@typescript-eslint/eslint-plugin': 6.11.0(@typescript-eslint/parser@6.11.0)(eslint@8.54.0)(typescript@5.2.2)
+      '@typescript-eslint/parser': 6.11.0(eslint@8.54.0)(typescript@5.2.2)
+      body-parser: 1.19.2
+      compression: 1.7.4
+      cors: 2.8.5
+      cross-env: 7.0.3
+      esbuild: 0.16.17
+      eslint: 8.54.0
+      eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.11.0)(eslint-plugin-import@2.28.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.54.0)(typescript@5.2.2)
+      eslint-plugin-import: 2.28.1(eslint@8.54.0)
+      eslint-plugin-n: 15.7.0(eslint@8.54.0)
+      eslint-plugin-promise: 6.1.1(eslint@8.54.0)
+      express: 4.18.2
+      jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1)
+      prettier: 3.1.0
+      prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5)
+      ts-jest: 29.1.1(esbuild@0.16.17)(jest@29.7.0)(typescript@5.2.2)
+      ts-node: 10.9.1(@types/node@16.11.68)(typescript@5.2.2)
+      typescript: 5.2.2
+      ws: 8.14.2(bufferutil@4.0.7)
+      yjs: 13.6.8
+    transitivePeerDependencies:
+      - '@babel/core'
+      - '@jest/types'
+      - '@swc/core'
+      - '@swc/wasm'
+      - babel-jest
+      - babel-plugin-macros
+      - bufferutil
+      - node-notifier
+      - supports-color
+      - svelte
+      - utf-8-validate
+      - y-protocols
+    dev: false
+
   file:projects/contact-assets.tgz(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)(typescript@5.2.2):
     resolution: {integrity: sha512-d3+KY+ge6P6JfemCKJyPIqf/RGxiVbQw8aek5bVoqb8SFuQuEDc6CvXPSf6ibOQHUJBznOkm0wfmXmpWE4meMA==, tarball: file:projects/contact-assets.tgz}
     id: file:projects/contact-assets.tgz

dev/docker-compose.yaml

@@ -69,6 +69,7 @@ services:
       - mongodb
       - minio
       - elastic
+      - collaborator
       - transactor
     ports:
       - 8087:8080
@@ -83,12 +84,28 @@ services:
       - UPLOAD_URL=/files
       - TRANSACTOR_URL=ws://localhost:3333
       - ELASTIC_URL=http://elastic:9200
+      - COLLABORATOR_URL=ws://localhost:3078
       - MINIO_ENDPOINT=minio
       - MINIO_ACCESS_KEY=minioadmin
       - MINIO_SECRET_KEY=minioadmin
       - TITLE=DevPlatform
       - DEFAULT_LANGUAGE=ru
     restart: unless-stopped
+  collaborator:
+    image: hardcoreeng/collaborator
+    links:
+      - minio
+      - transactor
+    ports:
+      - 3078:3078
+    environment:
+      - COLLABORATOR_PORT=3078
+      - SECRET=secret
+      - TRANSACTOR_URL=ws://localhost:3333
+      - MINIO_ENDPOINT=minio
+      - MINIO_ACCESS_KEY=minioadmin
+      - MINIO_SECRET_KEY=minioadmin
+    restart: unless-stopped
   # tracker-front:
   #   image: hardcoreeng/tracker-front
   #   links:

dev/prod/.env

@@ -7,3 +7,4 @@ CALENDAR_URL=http://localhost:8095
 FRONT_URL=http://localhost:8080
 
 REKONI_URL=http://localhost:4004
+COLLABORATOR_URL=ws://locahost:3078

dev/prod/config.json

@@ -1,5 +1,6 @@
 {
   "ACCOUNTS_URL":"http://localhost:3000",
   "UPLOAD_URL":"/files",
+  "COLLABORATOR_URL": "ws://localhost:3078",
   "REKONI_URL": "http://localhost:4004"
 }

dev/prod/public/config-dev.json

@@ -5,5 +5,6 @@
   "TELEGRAM_URL": "https://telegram.hc.engineering",
   "GMAIL_URL": "https://gmail.hc.engineering",
   "CALENDAR_URL": "https://calendar.hc.engineering",
-  "REKONI_URL": "https://rekoni.hc.engineering"
+  "REKONI_URL": "https://rekoni.hc.engineering",
+  "COLLABORATOR_URL": "wss://collaborator.hc.engineering"
 }

dev/prod/public/config.json

@@ -5,5 +5,6 @@
   "TELEGRAM_URL": "http://localhost:8086",
   "GMAIL_URL": "http://localhost:8088",
   "CALENDAR_URL": "http://localhost:8095",
-  "REKONI_URL": "http://localhost:4004"
+  "REKONI_URL": "http://localhost:4004",
+  "COLLABORATOR_URL": "ws://localhost:3078"
 }

dev/prod/src/platform.ts

@@ -73,7 +73,7 @@ import '@hcengineering/workbench-assets'
 
 import { coreId } from '@hcengineering/core'
 import presentation, { presentationId } from '@hcengineering/presentation'
-import { textEditorId } from '@hcengineering/text-editor'
+import textEditor, { textEditorId } from '@hcengineering/text-editor'
 
 import { setMetadata } from '@hcengineering/platform'
 import { setDefaultLanguage } from '@hcengineering/theme'
@@ -88,6 +88,7 @@ interface Config {
   TELEGRAM_URL: string
   GMAIL_URL: string
   CALENDAR_URL: string
+  COLLABORATOR_URL: string
   TITLE?: string
   LANGUAGES?: string
   DEFAULT_LANGUAGE?: string
@@ -149,6 +150,8 @@ export async function configurePlatform() {
 
   setMetadata(rekoni.metadata.RekoniUrl, config.REKONI_URL)
 
+  setMetadata(textEditor.metadata.CollaboratorUrl, config.COLLABORATOR_URL ?? 'ws://locahost:3078')
+
   setMetadata(uiPlugin.metadata.DefaultApplication, login.component.LoginApp)
 
   const languages = config.LANGUAGES ? (config.LANGUAGES as string).split(',').map((l) => l.trim()) : ['en', 'ru']

packages/text-editor/src/plugin.ts

@@ -1,6 +1,6 @@
 //
 // Copyright © 2020, 2021 Anticrm Platform Contributors.
-// Copyright © 2021 Hardcore Engineering Inc.
+// Copyright © 2021, 2023 Hardcore Engineering Inc.
 //
 // Licensed under the Eclipse Public License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License. You may
@@ -15,7 +15,7 @@
 //
 
 import { type Class, type Ref } from '@hcengineering/core'
-import { type IntlString, type Plugin, plugin } from '@hcengineering/platform'
+import { type IntlString, type Metadata, type Plugin, plugin } from '@hcengineering/platform'
 import { type RefInputActionItem } from './types'
 
 /**
@@ -27,6 +27,9 @@ export default plugin(textEditorId, {
   class: {
     RefInputActionItem: '' as Ref<Class<RefInputActionItem>>
   },
+  metadata: {
+    CollaboratorUrl: '' as Metadata<string>
+  },
   string: {
     TableOfContents: '' as IntlString,
     Suggested: '' as IntlString,

pods/collaborator/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}

pods/collaborator/.npmignore

@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/

pods/collaborator/Dockerfile

@@ -0,0 +1,9 @@
+
+FROM node:20-alpine
+
+WORKDIR /usr/src/app
+
+COPY bundle.js ./
+
+EXPOSE 3078
+CMD [ "node", "bundle.js" ]

pods/collaborator/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build
+rushx docker:push

pods/collaborator/config/rig.json

@@ -0,0 +1,4 @@
+{  
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+  "rigPackageName": "@hcengineering/platform-rig"
+}

pods/collaborator/jest.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}

pods/collaborator/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@hcengineering/collaborator",
+  "version": "0.6.0",
+  "main": "lib/index.js",
+  "author": "Hardcore Engineering Inc.",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "tsc",
+    "build:watch": "tsc",
+    "lint:fix": "eslint --fix src",
+    "bundle": "esbuild src/__start.ts --bundle  --platform=node > bundle.js",
+    "docker:build": "docker build -t hardcoreeng/collaborator .",
+    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging",
+    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator",
+    "run-local": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts",
+    "run-bundle": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin node ./bundle.js",
+    "lint": "eslint src",
+    "format": "format src",
+    "test": "jest --passWithNoTests --silent"
+  },
+  "devDependencies": {
+    "cross-env": "~7.0.3",
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@types/node": "~16.11.12",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "esbuild": "^0.16.14",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "ts-node": "^10.8.0",
+    "typescript": "^5.2.2",
+    "@types/body-parser": "~1.19.2",
+    "@types/express": "^4.17.13",
+    "@types/cors": "^2.8.12",
+    "@types/compression": "~1.7.2",
+    "@types/ws": "^8.5.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5",
+    "prettier-plugin-svelte": "^3.1.0"
+  },
+  "dependencies": {
+    "@hcengineering/core": "^0.6.28",
+    "@hcengineering/account": "^0.6.0",
+    "@hcengineering/platform": "^0.6.9",
+    "@hcengineering/server-tool": "^0.6.0",
+    "@hcengineering/server-token": "^0.6.7",
+    "@hcengineering/server-core": "^0.6.1",
+    "@hcengineering/attachment": "^0.6.9",
+    "@hcengineering/client": "^0.6.14",
+    "@hcengineering/client-resources": "^0.6.23",
+    "@hcengineering/minio": "^0.6.0",
+    "yjs": "^13.5.52",
+    "@hocuspocus/server": "^2.5.0",
+    "express": "^4.17.1",
+    "body-parser": "~1.19.1",
+    "cors": "^2.8.5",
+    "compression": "~1.7.4",
+    "ws": "^8.10.0"
+  }
+}

pods/collaborator/src/__start.ts

@@ -0,0 +1,50 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { setMetadata } from '@hcengineering/platform'
+import serverToken from '@hcengineering/server-token'
+import { metricsContext } from './metrics'
+import { start } from './server'
+
+import { MinioService } from '@hcengineering/minio'
+import config from './config'
+
+setMetadata(serverToken.metadata.Secret, config.Secret)
+
+let minioPort = 9000
+let minioEndpoint = config.MinioEndpoint
+const sp = minioEndpoint.split(':')
+if (sp.length > 1) {
+  minioEndpoint = sp[0]
+  minioPort = parseInt(sp[1])
+}
+
+const minio = new MinioService({
+  endPoint: minioEndpoint,
+  port: minioPort,
+  useSSL: false,
+  accessKey: config.MinioAccessKey,
+  secretKey: config.MinioSecretKey
+})
+
+const server = start(metricsContext, config, minio)
+
+const close = (): void => {
+  server()
+}
+process.on('SIGINT', close)
+process.on('SIGTERM', close)
+process.on('exit', close)

pods/collaborator/src/config.ts

@@ -0,0 +1,76 @@
+//
+// Copyright © 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+/**
+ * @public
+ */
+export interface Config {
+  ServiceID: string
+  Secret: string
+
+  Interval: number
+
+  Port: number
+
+  TransactorUrl: string
+
+  MinioEndpoint: string
+  MinioAccessKey: string
+  MinioSecretKey: string
+}
+
+const envMap: { [key in keyof Config]: string } = {
+  ServiceID: 'SERVICE_ID',
+  Secret: 'SECRET',
+  Interval: 'INTERVAL',
+  Port: 'COLLABORATOR_PORT',
+  TransactorUrl: 'TRANSACTOR_URL',
+  MinioEndpoint: 'MINIO_ENDPOINT',
+  MinioAccessKey: 'MINIO_ACCESS_KEY',
+  MinioSecretKey: 'MINIO_SECRET_KEY'
+}
+
+const required: Array<keyof Config> = [
+  'Secret',
+  'ServiceID',
+  'Port',
+  'TransactorUrl',
+  'MinioEndpoint',
+  'MinioAccessKey',
+  'MinioSecretKey'
+]
+
+const config: Config = (() => {
+  const params: Partial<Config> = {
+    Secret: process.env[envMap.Secret],
+    ServiceID: process.env[envMap.ServiceID] ?? 'collaborator-service',
+    Interval: parseInt(process.env[envMap.Interval] ?? '30000'),
+    Port: parseInt(process.env[envMap.Port] ?? '3078'),
+    TransactorUrl: process.env[envMap.TransactorUrl],
+    MinioEndpoint: process.env[envMap.MinioEndpoint],
+    MinioAccessKey: process.env[envMap.MinioAccessKey],
+    MinioSecretKey: process.env[envMap.MinioSecretKey]
+  }
+
+  const missingEnv = required.filter((key) => params[key] === undefined).map((key) => envMap[key])
+
+  if (missingEnv.length > 0) {
+    throw Error(`Missing env variables: ${missingEnv.join(', ')}`)
+  }
+
+  return params as Config
+})()
+
+export default config

pods/collaborator/src/context.ts

@@ -0,0 +1,34 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Token, decodeToken } from '@hcengineering/server-token'
+import { onAuthenticatePayload } from '@hocuspocus/server'
+
+export interface Context {
+  token: Token
+  initialContentId: string
+}
+
+export function buildContext (data: onAuthenticatePayload): Context {
+  const token = decodeToken(data.token)
+  const initialContentId = data.requestParameters.get('initialContentId') as string
+
+  const context: Context = {
+    token,
+    initialContentId: initialContentId ?? ''
+  }
+
+  return context
+}

pods/collaborator/src/extensions/action.ts

@@ -0,0 +1,139 @@
+import { Connection, Document, Extension, Hocuspocus, onConfigurePayload, onStatelessPayload } from '@hocuspocus/server'
+import * as Y from 'yjs'
+
+import { Context } from '../context'
+import { Action, ActionStatus, ActionStatusResponse, DocumentCopyAction, DocumentFieldCopyAction } from '../types'
+
+export class ActionsExtension implements Extension {
+  instance!: Hocuspocus
+
+  async onConfigure ({ instance }: onConfigurePayload): Promise<void> {
+    this.instance = instance
+  }
+
+  async onStateless (data: onStatelessPayload): Promise<any> {
+    try {
+      const action = JSON.parse(data.payload) as Action
+      const context = data.connection.context as Context
+      const { connection } = data
+
+      switch (action.action) {
+        case 'document.copy':
+          await this.onCopyDocument(context, action)
+          this.sendActionStatus(connection, action, 'completed')
+          return
+        case 'document.field.copy':
+          await this.onCopyDocumentField(context, action)
+          this.sendActionStatus(connection, action, 'completed')
+          return
+        default:
+          console.error('unsupported action type', action)
+      }
+    } catch (err: any) {
+      console.error('failed to process stateless message', err)
+    }
+  }
+
+  sendActionStatus (connection: Connection, action: Action, status: ActionStatus): void {
+    const payload: ActionStatusResponse = { action, status }
+    connection.sendStateless(JSON.stringify(payload))
+  }
+
+  async onCopyDocument (context: Context, action: DocumentCopyAction): Promise<void> {
+    const instance = this.instance
+
+    const { sourceId, targetId } = action.params
+    console.info(`copy document content ${sourceId} -> ${targetId}`)
+
+    const _context: Context = {
+      token: context.token,
+      initialContentId: ''
+    }
+
+    let source: Document | null = null
+    let target: Document | null = null
+
+    const sourceConnection = await instance.openDirectConnection(sourceId, _context)
+    const targetConnection = await instance.openDirectConnection(targetId, _context)
+
+    try {
+      source = sourceConnection.document
+      target = targetConnection.document
+
+      if (source !== null && target !== null) {
+        const updates = Y.encodeStateAsUpdate(source)
+
+        // make an empty transaction to force source document save
+        // without that force document unload won't save the doc
+        await sourceConnection.transact(() => {})
+
+        await targetConnection.transact((target) => {
+          Y.applyUpdate(target, updates)
+        })
+      } else {
+        console.warn('empty ydoc document', sourceId, targetId)
+      }
+    } finally {
+      await targetConnection.disconnect()
+      await sourceConnection.disconnect()
+    }
+
+    // Hocuspocus does not unload document when direct conneciton is used
+    // so we have to do it manually
+    // https://github.com/ueberdosis/hocuspocus/issues/709
+
+    if (source !== null && source.getConnectionsCount() === 0) {
+      instance.unloadDocument(source)
+    }
+
+    if (target !== null && target.getConnectionsCount() === 0) {
+      instance.unloadDocument(target)
+    }
+  }
+
+  async onCopyDocumentField (context: Context, action: DocumentFieldCopyAction): Promise<void> {
+    const instance = this.instance
+
+    const { documentId, srcFieldId, dstFieldId } = action.params
+    console.info(`copy document ${documentId} field content ${srcFieldId} -> ${dstFieldId}`)
+
+    if (srcFieldId == null || srcFieldId === '' || dstFieldId == null || dstFieldId === '') {
+      console.error('empty srcFieldId or dstFieldId', srcFieldId, dstFieldId)
+      return
+    }
+
+    const _context: Context = {
+      token: context.token,
+      initialContentId: ''
+    }
+
+    let doc: Document | null = null
+
+    const docConnection = await instance.openDirectConnection(documentId, _context)
+
+    try {
+      doc = docConnection.document
+
+      await docConnection.transact((doc) => {
+        const srcField = doc.getXmlFragment(srcFieldId)
+        const dstField = doc.getXmlFragment(dstFieldId)
+
+        // similar to XmlFragment's clone method
+        dstField.insert(
+          0,
+          srcField.toArray().map((item) => (item instanceof Y.AbstractType ? item.clone() : item)) as any
+        )
+      })
+    } finally {
+      await docConnection.disconnect()
+    }
+
+    // Hocuspocus does not unload document when direct conneciton is used
+    // so we have to do it manually
+    // https://github.com/ueberdosis/hocuspocus/issues/709
+
+    if (doc !== null && doc.getConnectionsCount() === 0) {
+      instance.unloadDocument(doc)
+    }
+  }
+}

pods/collaborator/src/extensions/request.ts

@@ -0,0 +1,38 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { Extension, onRequestPayload } from '@hocuspocus/server'
+import { MeasureContext } from '@hcengineering/core'
+import { RequestListener } from 'http'
+
+export interface RequestConfiguration {
+  ctx: MeasureContext
+  handler: RequestListener
+}
+
+export class RequestExtension implements Extension {
+  private readonly configuration: RequestConfiguration
+
+  constructor (configuration: RequestConfiguration) {
+    this.configuration = configuration
+  }
+
+  async onRequest (data: onRequestPayload): Promise<void> {
+    this.configuration.ctx.measure('request', 1)
+
+    const { request, response } = data
+    this.configuration.handler(request, response)
+  }
+}

pods/collaborator/src/extensions/storage.ts

@@ -0,0 +1,133 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import attachment, { Attachment } from '@hcengineering/attachment'
+import client from '@hcengineering/client'
+import clientResources from '@hcengineering/client-resources'
+import core, { Client, MeasureContext, Ref, TxOperations } from '@hcengineering/core'
+import { MinioService } from '@hcengineering/minio'
+import { setMetadata } from '@hcengineering/platform'
+import { Token, generateToken } from '@hcengineering/server-token'
+import { Extension, onLoadDocumentPayload, onStoreDocumentPayload } from '@hocuspocus/server'
+import { applyUpdate, encodeStateAsUpdate } from 'yjs'
+import config from '../config'
+import { Context } from '../context'
+
+// eslint-disable-next-line
+const WebSocket = require('ws')
+
+async function connect (transactorUrl: string, token: Token): Promise<Client> {
+  const encodedToken = generateToken(token.email, token.workspace)
+  // We need to override default factory with 'ws' one.
+  setMetadata(client.metadata.ClientSocketFactory, (url) => {
+    return new WebSocket(url, {
+      headers: {
+        'User-Agent': config.ServiceID
+      }
+    })
+  })
+  return await (await clientResources()).function.GetClient(encodedToken, transactorUrl)
+}
+
+export interface StorageConfiguration {
+  ctx: MeasureContext
+  minio: MinioService
+  transactorUrl: string
+}
+
+export class StorageExtension implements Extension {
+  private readonly configuration: StorageConfiguration
+
+  constructor (configuration: StorageConfiguration) {
+    this.configuration = configuration
+  }
+
+  async getMinioDocument (documentId: string, token: Token): Promise<Buffer | undefined> {
+    const buffer = await this.configuration.minio.read(token.workspace, documentId)
+    return Buffer.concat(buffer)
+  }
+
+  async onLoadDocument (data: onLoadDocumentPayload): Promise<any> {
+    console.log('load document', data.documentName)
+
+    const documentId = data.documentName
+    const { token, initialContentId } = data.context as Context
+
+    await this.configuration.ctx.with('load-document', {}, async () => {
+      let minioDocument: Buffer | undefined
+      try {
+        minioDocument = await this.getMinioDocument(documentId, token)
+      } catch (err: any) {
+        if (initialContentId !== undefined && initialContentId.length > 0) {
+          minioDocument = await this.getMinioDocument(initialContentId, token)
+        }
+      }
+
+      if (minioDocument !== undefined && minioDocument.length > 0) {
+        try {
+          const uint8arr = new Uint8Array(minioDocument)
+          applyUpdate(data.document, uint8arr)
+        } catch (err) {
+          console.error(err)
+        }
+      }
+    })
+
+    return data.document
+  }
+
+  async onStoreDocument (data: onStoreDocumentPayload): Promise<void> {
+    console.log('store document', data.documentName)
+
+    const documentId = data.documentName
+    const { token } = data.context as Context
+
+    await this.configuration.ctx.with('store-document', {}, async (ctx) => {
+      const updates = encodeStateAsUpdate(data.document)
+      const buffer = Buffer.from(updates.buffer)
+
+      // persist document to Minio
+      await ctx.with('minio', {}, async () => {
+        const metaData = { 'content-type': 'application/ydoc' }
+        await this.configuration.minio.put(token.workspace, documentId, buffer, buffer.length, metaData)
+      })
+
+      // notify platform about changes
+      await ctx.with('platform', {}, async () => {
+        try {
+          const connection = await connect(this.configuration.transactorUrl, token)
+
+          // token belongs to the first user opened the document, this is not accurate, but
+          // since the document is collaborative, we need to choose some account to update the doc
+          const account = await connection.findOne(core.class.Account, { email: token.email })
+          const accountId = account?._id ?? core.account.System
+
+          const client = new TxOperations(connection, accountId, true)
+          const current = await client.findOne(attachment.class.Attachment, { _id: documentId as Ref<Attachment> })
+          if (current !== undefined) {
+            console.debug('platform notification for document', documentId)
+            await client.update(current, { lastModified: Date.now(), size: buffer.length })
+          } else {
+            console.debug('platform attachment document not found', documentId)
+          }
+
+          await connection.close()
+        } catch (err: any) {
+          console.debug('failed to notify platform', documentId, err)
+        }
+      })
+    })
+  }
+}

pods/collaborator/src/index.ts

@@ -0,0 +1,18 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021, 2022 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export type { Config } from './config'
+export { start } from './server'

pods/collaborator/src/metrics.ts

@@ -0,0 +1,36 @@
+import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { writeFile } from 'fs/promises'
+
+const metricsFile = process.env.METRICS_FILE
+const metricsConsole = (process.env.METRICS_CONSOLE ?? 'false') === 'true'
+
+const METRICS_UPDATE_INTERVAL = !metricsConsole ? 1000 : 30000
+
+const metrics = newMetrics()
+export const metricsContext = new MeasureMetricsContext('System', {}, metrics)
+
+if (metricsFile !== undefined || metricsConsole) {
+  console.info('storing measurements into local file', metricsFile)
+  let oldMetricsValue = ''
+
+  const intTimer = setInterval(() => {
+    const val = metricsToString(metrics, 'System', 140)
+    if (val !== oldMetricsValue) {
+      oldMetricsValue = val
+      if (metricsFile !== undefined) {
+        writeFile(metricsFile, val).catch((err) => {
+          console.error(err)
+        })
+      }
+      if (metricsConsole) {
+        console.info('METRICS:\n', val)
+      }
+    }
+  }, METRICS_UPDATE_INTERVAL)
+
+  const closeTimer = (): void => {
+    clearInterval(intTimer)
+  }
+  process.on('SIGINT', closeTimer)
+  process.on('SIGTERM', closeTimer)
+}

pods/collaborator/src/server.ts

@@ -0,0 +1,144 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { MeasureContext } from '@hcengineering/core'
+import { MinioService } from '@hcengineering/minio'
+import { Hocuspocus, onAuthenticatePayload } from '@hocuspocus/server'
+import bp from 'body-parser'
+import compression from 'compression'
+import cors from 'cors'
+import express from 'express'
+import { IncomingMessage, createServer } from 'http'
+import { WebSocket, WebSocketServer } from 'ws'
+
+import { Config } from './config'
+import { ActionsExtension } from './extensions/action'
+import { StorageExtension } from './extensions/storage'
+import { Context, buildContext } from './context'
+
+const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0'
+
+/**
+ * @public
+ */
+export function start (ctx: MeasureContext, config: Config, minio: MinioService): () => void {
+  const port = config.Port
+  console.log(`starting server on :${port} ...`)
+
+  const app = express()
+  app.use(cors())
+  app.use(bp.json())
+  app.use(
+    compression({
+      filter: (req, res) => {
+        if (req.headers['x-no-compression'] != null) {
+          // don't compress responses with this request header
+          return false
+        }
+
+        // fallback to standard filter function
+        return compression.filter(req, res)
+      },
+      level: 6
+    })
+  )
+
+  const hocuspocus = new Hocuspocus({
+    address: '0.0.0.0',
+    port,
+
+    /**
+     * Defines in which interval the server sends a ping, and closes the connection when no pong is sent back.
+     */
+    timeout: 30000,
+    /**
+     * Debounces the call of the `onStoreDocument` hook for the given amount of time in ms.
+     * Otherwise every single update would be persisted.
+     */
+    debounce: 10000,
+    /**
+     * Makes sure to call `onStoreDocument` at least in the given amount of time (ms).
+     */
+    maxDebounce: 30000,
+    /**
+     * options to pass to the ydoc document
+     */
+    yDocOptions: {
+      gc: gcEnabled,
+      gcFilter: () => true
+    },
+    /**
+     * If set to false, respects the debounce time of `onStoreDocument` before unloading a document.
+     * Otherwise, the document will be unloaded immediately.
+     *
+     * This prevents a client from DOSing the server by repeatedly connecting and disconnecting when
+     * your onStoreDocument is rate-limited.
+     */
+    unloadImmediately: false,
+
+    extensions: [
+      new ActionsExtension(),
+      new StorageExtension({
+        ctx: ctx.newChild('minio', {}),
+        minio,
+        transactorUrl: config.TransactorUrl
+      })
+    ],
+
+    async onAuthenticate (data: onAuthenticatePayload): Promise<Context> {
+      ctx.measure('authenticate', 1)
+
+      return buildContext(data)
+    }
+  })
+
+  const wss = new WebSocketServer({
+    noServer: true,
+    perMessageDeflate: {
+      zlibDeflateOptions: {
+        // See zlib defaults.
+        chunkSize: 1024,
+        memLevel: 7,
+        level: 3
+      },
+      zlibInflateOptions: {
+        chunkSize: 10 * 1024
+      },
+      // Below options specified as default values.
+      concurrencyLimit: 10, // Limits zlib concurrency for perf.
+      threshold: 1024 // Size (in bytes) below which messages
+      // should not be compressed if context takeover is disabled.
+    }
+  })
+
+  wss.on('connection', (incoming: WebSocket, request: IncomingMessage) => {
+    hocuspocus.handleConnection(incoming, request)
+  })
+
+  const server = createServer(app)
+
+  server.on('upgrade', (request, socket, head) => {
+    wss.handleUpgrade(request, socket, head, (ws) => {
+      wss.emit('connection', ws, request)
+    })
+  })
+
+  server.listen(port)
+  console.log(`started server on :${port}`)
+
+  return () => {
+    server.close()
+  }
+}

pods/collaborator/src/types.ts

@@ -0,0 +1,40 @@
+//
+// Copyright © 2023 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+export type Action = DocumentCopyAction | DocumentFieldCopyAction
+
+export interface DocumentCopyAction {
+  action: 'document.copy'
+  params: {
+    sourceId: string
+    targetId: string
+  }
+}
+
+export interface DocumentFieldCopyAction {
+  action: 'document.field.copy'
+  params: {
+    documentId: string
+    srcFieldId: string
+    dstFieldId: string
+  }
+}
+
+export type ActionStatus = 'completed' | 'failed'
+
+export interface ActionStatusResponse {
+  action: Action
+  status: ActionStatus
+}

pods/collaborator/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./lib",
+    "tsBuildInfoFile": ".build/build.tsbuildinfo"
+  }
+}

pods/front/run.sh

@@ -4,6 +4,7 @@ export ACCOUNTS_URL=http://localhost:3333
 export UPLOAD_URL=http://localhost:3333/files
 export TRANSACTOR_URL=ws://localhost:3333
 export ELASTIC_URL=http://elastic:9200
+export COLLABORATOR_URL=ws://localhost:3078
 export MINIO_ENDPOINT=minio
 export MINIO_ACCESS_KEY=minioadmin
 export MINIO_SECRET_KEY=minioadmin 

rush.json

@@ -796,6 +796,11 @@
       "projectFolder": "pods/account",
       "shouldPublish": false
     },
+    {
+      "packageName": "@hcengineering/collaborator",
+      "projectFolder": "pods/collaborator",
+      "shouldPublish": false
+    },
     {
       "packageName": "@hcengineering/panel",
       "projectFolder": "packages/panel",

server/front/src/index.ts

@@ -140,6 +140,7 @@ export function start (
     telegramUrl: string
     gmailUrl: string
     calendarUrl: string
+    collaboratorUrl: string
     title?: string
     languages: string
     defaultLanguage: string
@@ -180,6 +181,7 @@ export function start (
       TELEGRAM_URL: config.telegramUrl,
       GMAIL_URL: config.gmailUrl,
       CALENDAR_URL: config.calendarUrl,
+      COLLABORATOR_URL: config.collaboratorUrl,
       TITLE: config.title,
       LANGUAGES: config.languages,
       DEFAULT_LANGUAGE: config.defaultLanguage,

server/front/src/starter.ts

@@ -98,6 +98,12 @@ export function startFront (extraConfig?: Record<string, string>): void {
     process.exit(1)
   }
 
+  const collaboratorUrl = process.env.COLLABORATOR_URL
+  if (collaboratorUrl === undefined) {
+    console.error('please provide collaborator url')
+    process.exit(1)
+  }
+
   const modelVersion = process.env.MODEL_VERSION
   if (modelVersion === undefined) {
     console.error('please provide model version requirement')
@@ -125,6 +131,7 @@ export function startFront (extraConfig?: Record<string, string>): void {
     telegramUrl,
     rekoniUrl,
     calendarUrl,
+    collaboratorUrl,
     title,
     languages,
     defaultLanguage

tests/docker-compose.yaml

@@ -58,6 +58,7 @@ services:
       - mongodb
       - minio
       - elastic
+      - collaborator
       - transactor
     ports:
       - 8083:8083
@@ -70,8 +71,9 @@ services:
       - ELASTIC_URL=http://elastic:9200
       - GMAIL_URL=http://localhost:8088
       - CALENDAR_URL=http://localhost:8095
-      - TELEGRAM_URL=http://localhost:8086
       - REKONI_URL=http://rekoni:4005
+      - TELEGRAM_URL=http://localhost:8086
+      - COLLABORATOR_URL=ws://localhost:3079
       - MINIO_ENDPOINT=minio
       - MINIO_ACCESS_KEY=minioadmin
       - MINIO_SECRET_KEY=minioadmin
@@ -97,6 +99,21 @@ services:
       - MINIO_SECRET_KEY=minioadmin  
       - REKONI_URL=http://rekoni:4005
       - FRONT_URL=http://localhost:8083
+  collaborator:
+    image: hardcoreeng/collaborator
+    links:
+      - minio
+      - transactor
+    ports:
+      - 3079:3078
+    environment:
+      - COLLABORATOR_PORT=3078
+      - SECRET=secret
+      - TRANSACTOR_URL=ws://localhost:3334
+      - MINIO_ENDPOINT=minio
+      - MINIO_ACCESS_KEY=minioadmin
+      - MINIO_SECRET_KEY=minioadmin
+    restart: unless-stopped
   rekoni:
     image: hardcoreeng/rekoni-service
     restart: on-failure