UBERF-4905 Collaborator Client (#4385)

Signed-off-by: Alexander Onnikov <alexander.onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2024-01-19 15:02:41 +07:00 committed by GitHub
parent 0d28919499
commit 96bd320f59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 583 additions and 187 deletions

View File

@ -95,6 +95,9 @@ dependencies:
'@rush-temp/collaborator':
specifier: file:./projects/collaborator.tgz
version: file:projects/collaborator.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5)
'@rush-temp/collaborator-client':
specifier: file:./projects/collaborator-client.tgz
version: file:projects/collaborator-client.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5)
'@rush-temp/contact':
specifier: file:./projects/contact.tgz
version: file:projects/contact.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1)
@ -17426,7 +17429,7 @@ packages:
dev: false
file:projects/activity-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(ts-node@10.9.1):
resolution: {integrity: sha512-S3azsw6dQwDQHF9dH26Ffp/Yje78ZWyL2Gv1xw9FRphd7bwsG8cz3/KgKHVjJ52Bg8lsuSAQZsUbUJqmn+U+QQ==, tarball: file:projects/activity-resources.tgz}
resolution: {integrity: sha512-PPeM8e2xFvmz4UBGUsVS7eoCxrzyepeOYWpM5iSaHM8qCV+FVIvVz5Bo2hdFrfuJFqqkWXQIGdG5KR9vjZ7RoQ==, tarball: file:projects/activity-resources.tgz}
id: file:projects/activity-resources.tgz
name: '@rush-temp/activity-resources'
version: 0.0.0
@ -17472,7 +17475,7 @@ packages:
dev: false
file:projects/activity.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-DwKYrRwF0ueeZMJ/FS/LEg+r/HhwqApV4K8cCmy0snR8ANFai7ySCs+4kswontH7boYtqqYXPc0Mh3GbJt9PKw==, tarball: file:projects/activity.tgz}
resolution: {integrity: sha512-X2rvNu6LpZQBVDB4fWC+0uz9LgMiD2zUIafX+97zhwblerUnNFdDsNS3HduaQ7E7Lxd2yPR6g5f2/483hyWw3g==, tarball: file:projects/activity.tgz}
id: file:projects/activity.tgz
name: '@rush-temp/activity'
version: 0.0.0
@ -18067,7 +18070,7 @@ packages:
dev: false
file:projects/chunter.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-smJiIU6iEoGU0Oz30Mor08eJ2/ZIlXH04X79HJqzD9q1rIBvq4KLR+iNv9j4TjExVNpehJYuTq2zqNBgBGSM9w==, tarball: file:projects/chunter.tgz}
resolution: {integrity: sha512-Rp37zqz6vL3bsizaFVha0dWFkLglKm5jrmnIhXWd2oxwmHwjrfCwdZDvMNQ9v17HEumNtsneKPel1EdqTXqvkw==, tarball: file:projects/chunter.tgz}
id: file:projects/chunter.tgz
name: '@rush-temp/chunter'
version: 0.0.0
@ -18163,8 +18166,68 @@ packages:
- ts-node
dev: false
file:projects/collaborator-client.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5):
resolution: {integrity: sha512-Y2u0DCP4rKKaMr/xXErbICh/BhNLqPCEf4G/IQO2zffTM0uzMB0Iep4nitc2JQ/jbrpcH17mvIgKULdjZzUJng==, tarball: file:projects/collaborator-client.tgz}
id: file:projects/collaborator-client.tgz
name: '@rush-temp/collaborator-client'
version: 0.0.0
dependencies:
'@hocuspocus/server': 2.9.0(bufferutil@4.0.7)(yjs@13.6.8)
'@hocuspocus/transformer': 2.9.0(@tiptap/pm@2.1.12)(y-prosemirror@1.2.1)(yjs@13.6.8)
'@tiptap/core': 2.1.12(@tiptap/pm@2.1.12)
'@tiptap/html': 2.1.12(@tiptap/core@2.1.12)(@tiptap/pm@2.1.12)
'@types/body-parser': 1.19.3
'@types/compression': 1.7.3
'@types/cors': 2.8.14
'@types/express': 4.17.18
'@types/jest': 29.5.5
'@types/node': 16.11.68
'@types/ws': 8.5.6
'@typescript-eslint/eslint-plugin': 6.11.0(@typescript-eslint/parser@6.11.0)(eslint@8.54.0)(typescript@5.2.2)
'@typescript-eslint/parser': 6.11.0(eslint@8.54.0)(typescript@5.2.2)
body-parser: 1.19.2
compression: 1.7.4
cors: 2.8.5
cross-env: 7.0.3
esbuild: 0.16.17
eslint: 8.54.0
eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.11.0)(eslint-plugin-import@2.28.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.54.0)(typescript@5.2.2)
eslint-plugin-import: 2.28.1(eslint@8.54.0)
eslint-plugin-n: 15.7.0(eslint@8.54.0)
eslint-plugin-promise: 6.1.1(eslint@8.54.0)
express: 4.18.2
jest: 29.7.0(@types/node@16.11.68)(ts-node@10.9.1)
mongodb: 4.17.1
prettier: 3.1.0
prettier-plugin-svelte: 3.1.0(prettier@3.1.0)(svelte@4.2.5)
ts-jest: 29.1.1(esbuild@0.16.17)(jest@29.7.0)(typescript@5.2.2)
ts-node: 10.9.1(@types/node@16.11.68)(typescript@5.2.2)
typescript: 5.2.2
ws: 8.14.2(bufferutil@4.0.7)
y-prosemirror: 1.2.1(prosemirror-model@1.19.3)(yjs@13.6.8)
yjs: 13.6.8
transitivePeerDependencies:
- '@babel/core'
- '@jest/types'
- '@swc/core'
- '@swc/wasm'
- '@tiptap/pm'
- aws-crt
- babel-jest
- babel-plugin-macros
- bufferutil
- node-notifier
- prosemirror-model
- prosemirror-state
- prosemirror-view
- supports-color
- svelte
- utf-8-validate
- y-protocols
dev: false
file:projects/collaborator.tgz(@tiptap/pm@2.1.12)(bufferutil@4.0.7)(prosemirror-model@1.19.3)(svelte@4.2.5):
resolution: {integrity: sha512-DF7F2NpWmslkwkGxwZ8nDLPvzRuK3nDEibOqhvTVyrikrfoaa6pwuSVfWP4W5fAUEBNGLssmHPn+ycCJjImFgQ==, tarball: file:projects/collaborator.tgz}
resolution: {integrity: sha512-3i/AaU2M+nAeTR6IvfoGu0Wv586kiA3g2lBd+yIujeJ8hD1ZZxaF6rbfJ7YD8E/BwOkBuFUoz5XVYKSTrszH4w==, tarball: file:projects/collaborator.tgz}
id: file:projects/collaborator.tgz
name: '@rush-temp/collaborator'
version: 0.0.0
@ -19458,7 +19521,7 @@ packages:
dev: false
file:projects/model-activity.tgz(svelte@4.2.5)(typescript@5.2.2):
resolution: {integrity: sha512-Fx/yJE88VjyIbfkipf4pXcHNPE7mis0NAl7OuOnSpHfDwrIwmroHlqiK39MFkhdaGNG7jXq0mS5IS6E3/Gp7Qg==, tarball: file:projects/model-activity.tgz}
resolution: {integrity: sha512-lvXe/UT6nwihndaVq8nfQJM1SGxOluGKjPwi5yg15go2BNr7M5wFs5sd0zL1Z9YmoTGV8ULxQ9hYaDeTqprAgA==, tarball: file:projects/model-activity.tgz}
id: file:projects/model-activity.tgz
name: '@rush-temp/model-activity'
version: 0.0.0
@ -19588,7 +19651,7 @@ packages:
dev: false
file:projects/model-chunter.tgz(svelte@4.2.5)(typescript@5.2.2):
resolution: {integrity: sha512-dKEGvo+qbnjpaBnAkCoQp5V+DRidHKAnUkaxpM87AVMf86KgbMXyauA9I5MMsiYxntETGbYKvtm+t86WHUKvsw==, tarball: file:projects/model-chunter.tgz}
resolution: {integrity: sha512-rHs6KE5C97g/93AQIbZTXKCziqe4cexWIJeY1LgGvb8hpfSdz6es02D2z6CkMfpFB9TN7CiEgam1rAqtSNryCg==, tarball: file:projects/model-chunter.tgz}
id: file:projects/model-chunter.tgz
name: '@rush-temp/model-chunter'
version: 0.0.0
@ -19843,7 +19906,7 @@ packages:
dev: false
file:projects/model-server-activity.tgz(svelte@4.2.5)(typescript@5.2.2):
resolution: {integrity: sha512-n6L6M7urgihLXLCFZWoh8LlVt+8dQHiGaehmAG4KJ1X3pz8xmSnZAjUAeLdXZvuRHFcfTPyHv0YzDBJ+npXxHg==, tarball: file:projects/model-server-activity.tgz}
resolution: {integrity: sha512-W5yodpuH2avXeIt+UOhKjsmZ858Kl5t2+IwbZMfJSevRVtdYQyZB/Q5D+TtXLvG9GP56e1v/MmqKp8Bq9RRsCQ==, tarball: file:projects/model-server-activity.tgz}
id: file:projects/model-server-activity.tgz
name: '@rush-temp/model-server-activity'
version: 0.0.0
@ -20595,7 +20658,7 @@ packages:
dev: false
file:projects/notification-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(postcss-load-config@4.0.1)(postcss@8.4.31)(ts-node@10.9.1):
resolution: {integrity: sha512-cW6xnhzBK/hf8sY/2gINDxY3q0LmLiL/qZ7PWwMjlZ4+RucWVlulv1L/odUE5BvrM24h3QDdblmcMosifajYCA==, tarball: file:projects/notification-resources.tgz}
resolution: {integrity: sha512-ie7dK5/1NvHN74rkaw5PwPNWjse1JjWZZ4jSsKoZupVoTrnxn7rJwFn0+y8pjOKFzQYWRY0DuOyPyTrPgzvMgw==, tarball: file:projects/notification-resources.tgz}
id: file:projects/notification-resources.tgz
name: '@rush-temp/notification-resources'
version: 0.0.0
@ -21505,7 +21568,7 @@ packages:
dev: false
file:projects/server-activity-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-NJwDYiOdiOtH/MMY/pcUduYzJIBgDHdVwmquqQ82r0Dg39c/Zz8de7At1VrAnvnfpbSSKFZLsBC8GtfGwdIHRA==, tarball: file:projects/server-activity-resources.tgz}
resolution: {integrity: sha512-sVs96kA6Pj0YZVwa71+MONgs5lbj6xj02J5uPyHbOyLILH7F6sdHJm4DYgHDhau70CMiaTSDSBMYqRFqbVdZ5g==, tarball: file:projects/server-activity-resources.tgz}
id: file:projects/server-activity-resources.tgz
name: '@rush-temp/server-activity-resources'
version: 0.0.0
@ -21731,7 +21794,7 @@ packages:
dev: false
file:projects/server-chunter-resources.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-ibW+MFJLyMOmVQAHiLX5fBdMBfh39Mmn5Dy9oVTHQPRnIxv27Z+372EPzgc45msNiiYBTK0N2J6orXFQPUxEag==, tarball: file:projects/server-chunter-resources.tgz}
resolution: {integrity: sha512-urgNqn+l0IGYW7E4UK3ke8wljzFVnUGYO+Q2w0E7WYL1aR7nXvIi8l5xJ9U6e81gPsR2kkxBrtzyy9cL/lbHpQ==, tarball: file:projects/server-chunter-resources.tgz}
id: file:projects/server-chunter-resources.tgz
name: '@rush-temp/server-chunter-resources'
version: 0.0.0
@ -23368,7 +23431,7 @@ packages:
dev: false
file:projects/task.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-qtjMbTtDdi6Zz2GIi6HXzFOVPYjrC35i3m/3E6cJcbFG7LzRv/QU9guhZl5M3zuSmymP0m6wxtJ3C4CiboCMJQ==, tarball: file:projects/task.tgz}
resolution: {integrity: sha512-IH/xqCjSTuVcbWA+pbE/dcZeChW7N4+jJiVCbwGDB605yipH/IDYSz1fsf+evXbkBm8lSHBnrzQf2ovVWk+Iig==, tarball: file:projects/task.tgz}
id: file:projects/task.tgz
name: '@rush-temp/task'
version: 0.0.0
@ -24278,7 +24341,7 @@ packages:
dev: false
file:projects/workbench.tgz(@types/node@16.11.68)(esbuild@0.16.17)(svelte@4.2.5)(ts-node@10.9.1):
resolution: {integrity: sha512-yqJzXkQsuNm3Cg9d7I8VqnsNjupgB/I2TGkkU52GH/GJ1BYWqyAtlZJoYnXcFN0J3cXY6MlQKilLRyg72ApTdQ==, tarball: file:projects/workbench.tgz}
resolution: {integrity: sha512-9k1vLPeqdczcg7wTZoViMiujvREnQmsuQKEnV29FJsX525KIXHzf1sotd+HjrlwmTFKXTlpcdU9aJv2Cl0bmnQ==, tarball: file:projects/workbench.tgz}
id: file:projects/workbench.tgz
name: '@rush-temp/workbench'
version: 0.0.0

View File

@ -137,6 +137,7 @@ export async function configurePlatform() {
console.log('loading configuration', config)
setMetadata(login.metadata.AccountsUrl, config.ACCOUNTS_URL)
setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
if (config.MODEL_VERSION != null) {
console.log('Minimal Model version requirement', config.MODEL_VERSION)

View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

View File

@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}

View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}

View File

@ -0,0 +1,38 @@
{
"name": "@hcengineering/collaborator-client",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "tsc",
"build:watch": "tsc",
"lint:fix": "eslint --fix src",
"lint": "eslint src",
"format": "format src",
"test": "jest --passWithNoTests --silent"
},
"devDependencies": {
"cross-env": "~7.0.3",
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~16.11.12",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"esbuild": "^0.16.14",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"ts-node": "^10.8.0",
"typescript": "^5.2.2",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.28"
}
}

View File

@ -0,0 +1,63 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, Doc, Markup, Ref, concatLink } from '@hcengineering/core'
/**
* @public
*/
export interface CollaboratorClient {
get: (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string) => Promise<Markup>
update: (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string, value: Markup) => Promise<void>
}
/**
* @public
*/
export function getClient (token: string, collaboratorUrl: string): CollaboratorClient {
return new CollaboratorClientImpl(token, collaboratorUrl)
}
class CollaboratorClientImpl implements CollaboratorClient {
constructor (
private readonly token: string,
private readonly collaboratorUrl: string
) {}
async get (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): Promise<Markup> {
const url = concatLink(this.collaboratorUrl, `/api/content/${classId}/${docId}/${attribute}`)
const res = await fetch(url, {
method: 'GET',
headers: {
Authorization: 'Bearer ' + this.token,
Accept: 'application/json'
}
})
const json = await res.json()
return json.html ?? '<p></p>'
}
async update (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string, value: Markup): Promise<void> {
const url = concatLink(this.collaboratorUrl, `/api/content/${classId}/${docId}/${attribute}`)
await fetch(url, {
method: 'PUT',
headers: {
Authorization: 'Bearer ' + this.token,
'Content-Type': 'application/json'
},
body: JSON.stringify({ html: value })
})
}
}

View File

@ -0,0 +1,16 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export { type CollaboratorClient, getClient } from './client'

View File

@ -0,0 +1,9 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

View File

@ -43,6 +43,7 @@
"@hcengineering/view": "^0.6.9",
"svelte": "^4.2.5",
"@hcengineering/client": "^0.6.14",
"@hcengineering/collaborator-client": "^0.6.0",
"fast-equals": "^2.0.3"
}
}

View File

@ -0,0 +1,52 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type CollaboratorClient, getClient } from '@hcengineering/collaborator-client'
import type { Class, Doc, Markup, Ref } from '@hcengineering/core'
import { getMetadata } from '@hcengineering/platform'
import presentation from './plugin'
/**
* @public
*/
export function getCollaboratorClient (): CollaboratorClient {
const token = getMetadata(presentation.metadata.Token) ?? ''
const collaboratorURL = getMetadata(presentation.metadata.CollaboratorUrl) ?? ''
return getClient(token, collaboratorURL)
}
/**
* @public
*/
export async function getMarkup (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): Promise<Markup> {
const client = getCollaboratorClient()
return await client.get(classId, docId, attribute)
}
/**
* @public
*/
export async function updateMarkup (
classId: Ref<Class<Doc>>,
docId: Ref<Doc>,
attribute: string,
value: Markup
): Promise<void> {
const client = getCollaboratorClient()
await client.update(classId, docId, attribute, value)
}

View File

@ -49,6 +49,7 @@ export * from './types'
export * from './utils'
export * from './drafts'
export { presentationId }
export * from './collaborator'
export * from './configuration'
export * from './context'
export * from './pipeline'

View File

@ -75,6 +75,7 @@ export default plugin(presentationId, {
RequiredVersion: '' as Metadata<string>,
Draft: '' as Metadata<Record<string, any>>,
UploadURL: '' as Metadata<string>,
CollaboratorUrl: '' as Metadata<string>,
Token: '' as Metadata<string>,
FrontUrl: '' as Asset
}

View File

@ -25,13 +25,15 @@ interface EVENTS {
async function fetchContent (doc: YDoc, name: string): Promise<void> {
const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin
const res = await fetch(concatLink(frontUrl, `/files?file=${name}`))
try {
const res = await fetch(concatLink(frontUrl, `/files?file=${name}`))
if (res.ok) {
const blob = await res.blob()
const buffer = await blob.arrayBuffer()
applyUpdate(doc, new Uint8Array(buffer))
}
if (res.ok) {
const blob = await res.blob()
const buffer = await blob.arrayBuffer()
applyUpdate(doc, new Uint8Array(buffer))
}
} catch {}
}
export class MinioProvider extends Observable<EVENTS> {

View File

@ -491,6 +491,11 @@
"projectFolder": "packages/ui",
"shouldPublish": true
},
{
"packageName": "@hcengineering/collaborator-client",
"projectFolder": "packages/collaborator-client",
"shouldPublish": true
},
{
"packageName": "@hcengineering/prod",
"projectFolder": "dev/prod",

View File

@ -13,23 +13,28 @@
// limitations under the License.
//
import { generateId } from '@hcengineering/core'
import { Token, decodeToken } from '@hcengineering/server-token'
import { WorkspaceId, generateId } from '@hcengineering/core'
import { decodeToken } from '@hcengineering/server-token'
import { onAuthenticatePayload } from '@hocuspocus/server'
import { ClientFactory, Controller, getClientFactory } from './platform'
export interface Context {
connectionId: string
token: string
decodedToken: Token
workspaceId: WorkspaceId
clientFactory: ClientFactory
initialContentId: string
targetContentId: string
}
export type withContext<T> = Omit<T, 'context'> & {
interface WithContext {
context: any
}
export type withContext<T extends WithContext> = Omit<T, 'context'> & {
context: Context
}
export function buildContext (data: onAuthenticatePayload): Context {
export function buildContext (data: onAuthenticatePayload, controller: Controller): Context {
const connectionId = generateId()
const decodedToken = decodeToken(data.token)
const initialContentId = data.requestParameters.get('initialContentId') as string
@ -37,8 +42,8 @@ export function buildContext (data: onAuthenticatePayload): Context {
const context: Context = {
connectionId,
decodedToken,
token: data.token,
workspaceId: decodedToken.workspace,
clientFactory: getClientFactory(decodedToken, controller),
initialContentId: initialContentId ?? '',
targetContentId: targetContentId ?? ''
}

View File

@ -99,45 +99,25 @@ export class ActionsExtension implements Extension {
const _context: Context = { ...context, initialContentId: '', targetContentId: '' }
let source: Document | null = null
let target: Document | null = null
const sourceConnection = await instance.openDirectConnection(sourceId, _context)
const targetConnection = await instance.openDirectConnection(targetId, _context)
try {
source = sourceConnection.document
target = targetConnection.document
let updates = new Uint8Array()
if (source !== null && target !== null) {
const updates = Y.encodeStateAsUpdate(source)
await sourceConnection.transact((source) => {
updates = Y.encodeStateAsUpdate(source)
})
// make an empty transaction to force source document save
// without that force document unload won't save the doc
await sourceConnection.transact(() => {})
await targetConnection.transact((target) => {
Y.applyUpdate(target, updates)
})
} else {
console.warn('empty ydoc document', sourceId, targetId)
}
await targetConnection.transact((target) => {
// TODO this does not work properly for existing documents
// we need to replace content, not only apply updates
Y.applyUpdate(target, updates)
})
} finally {
await targetConnection.disconnect()
await sourceConnection.disconnect()
}
// Hocuspocus does not unload document when direct conneciton is used
// so we have to do it manually
// https://github.com/ueberdosis/hocuspocus/issues/709
if (source !== null && source.getConnectionsCount() === 0) {
instance.unloadDocument(source)
}
if (target !== null && target.getConnectionsCount() === 0) {
instance.unloadDocument(target)
}
}
async onCopyDocumentField (context: Context, action: DocumentFieldCopyAction): Promise<void> {
@ -153,13 +133,9 @@ export class ActionsExtension implements Extension {
const _context: Context = { ...context, initialContentId: '', targetContentId: '' }
let doc: Document | null = null
const docConnection = await instance.openDirectConnection(documentId, _context)
try {
doc = docConnection.document
await docConnection.transact((doc) => {
const srcField = doc.getXmlFragment(srcFieldId)
const dstField = doc.getXmlFragment(dstFieldId)
@ -173,13 +149,5 @@ export class ActionsExtension implements Extension {
} finally {
await docConnection.disconnect()
}
// Hocuspocus does not unload document when direct conneciton is used
// so we have to do it manually
// https://github.com/ueberdosis/hocuspocus/issues/709
if (doc !== null && doc.getConnectionsCount() === 0) {
instance.unloadDocument(doc)
}
}
}

View File

@ -1,38 +0,0 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Extension, onRequestPayload } from '@hocuspocus/server'
import { MeasureContext } from '@hcengineering/core'
import { RequestListener } from 'http'
export interface RequestConfiguration {
ctx: MeasureContext
handler: RequestListener
}
export class RequestExtension implements Extension {
private readonly configuration: RequestConfiguration
constructor (configuration: RequestConfiguration) {
this.configuration = configuration
}
async onRequest (data: onRequestPayload): Promise<void> {
this.configuration.ctx.measure('request', 1)
const { request, response } = data
this.configuration.handler(request, response)
}
}

View File

@ -15,15 +15,15 @@
import client from '@hcengineering/client'
import clientResources from '@hcengineering/client-resources'
import core, { Client, TxOperations } from '@hcengineering/core'
import core, { Client, Tx, TxOperations, WorkspaceId, systemAccountEmail, toWorkspaceString } from '@hcengineering/core'
import { setMetadata } from '@hcengineering/platform'
import { Token } from '@hcengineering/server-token'
import { Token, generateToken } from '@hcengineering/server-token'
import config from './config'
// eslint-disable-next-line
const WebSocket = require('ws')
export async function connect (transactorUrl: string, token: string): Promise<Client> {
async function connect (token: string): Promise<Client> {
// We need to override default factory with 'ws' one.
setMetadata(client.metadata.ClientSocketFactory, (url) => {
return new WebSocket(url, {
@ -32,12 +32,90 @@ export async function connect (transactorUrl: string, token: string): Promise<Cl
}
})
})
return await (await clientResources()).function.GetClient(token, transactorUrl)
return await (await clientResources()).function.GetClient(token, config.TransactorUrl)
}
export async function getTxOperations (client: Client, token: Token, isDerived: boolean = false): Promise<TxOperations> {
async function getTxOperations (client: Client, token: Token, isDerived: boolean = false): Promise<TxOperations> {
const account = await client.findOne(core.class.Account, { email: token.email })
const accountId = account?._id ?? core.account.System
return new TxOperations(client, accountId, isDerived)
}
/**
* @public
*/
export interface ClientFactoryParams {
derived: boolean
}
/**
* @public
*/
export type ClientFactory = (params: ClientFactoryParams) => Promise<TxOperations>
/**
* @public
*/
export function getClientFactory (token: Token, controller: Controller): ClientFactory {
return async ({ derived }: ClientFactoryParams) => {
const workspaceClient = await controller.get(token.workspace)
return await getTxOperations(workspaceClient.client, token, derived)
}
}
/**
* @public
*/
export class Controller {
private readonly workspaces: Map<string, WorkspaceClient> = new Map<string, WorkspaceClient>()
async get (workspaceId: WorkspaceId): Promise<WorkspaceClient> {
const workspace = toWorkspaceString(workspaceId)
let client = this.workspaces.get(workspace)
if (client === undefined) {
client = await WorkspaceClient.create(workspaceId)
this.workspaces.set(workspace, client)
}
return client
}
async close (): Promise<void> {
for (const workspace of this.workspaces.values()) {
await workspace.close()
}
this.workspaces.clear()
}
}
/**
* @public
*/
export class WorkspaceClient {
private readonly txHandlers: ((tx: Tx) => Promise<void>)[] = []
private constructor (
readonly workspace: WorkspaceId,
readonly client: Client
) {
this.client.notify = (tx) => {
void this.txHandler(tx)
}
}
static async create (workspace: WorkspaceId): Promise<WorkspaceClient> {
const token = generateToken(systemAccountEmail, workspace)
const client = await connect(token)
return new WorkspaceClient(workspace, client)
}
async close (): Promise<void> {
await this.client.close()
}
private async txHandler (tx: Tx): Promise<void> {
this.txHandlers.map((handler) => handler(tx))
}
}

View File

@ -13,10 +13,11 @@
// limitations under the License.
//
import { MeasureContext } from '@hcengineering/core'
import { MeasureContext, generateId } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Token, decodeToken } from '@hcengineering/server-token'
import { ServerKit } from '@hcengineering/text'
import { Hocuspocus, onAuthenticatePayload } from '@hocuspocus/server'
import { Hocuspocus, onAuthenticatePayload, onDestroyPayload } from '@hocuspocus/server'
import bp from 'body-parser'
import compression from 'compression'
import cors from 'cors'
@ -24,15 +25,17 @@ import express from 'express'
import { IncomingMessage, createServer } from 'http'
import { MongoClient } from 'mongodb'
import { WebSocket, WebSocketServer } from 'ws'
import { applyUpdate, encodeStateAsUpdate } from 'yjs'
import { Config } from './config'
import { ActionsExtension } from './extensions/action'
import { Context, buildContext } from './context'
import { ActionsExtension } from './extensions/action'
import { HtmlTransformer } from './transformers/html'
import { StorageExtension } from './extensions/storage'
import { Controller, getClientFactory } from './platform'
import { MinioStorageAdapter } from './storage/minio'
import { MongodbStorageAdapter } from './storage/mongodb'
import { PlatformStorageAdapter } from './storage/platform'
import { StorageExtension } from './extensions/storage'
import { RouterStorageAdapter } from './storage/router'
const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0'
@ -83,6 +86,10 @@ export async function start (
const extensionsCtx = ctx.newChild('extensions', {})
const storageCtx = ctx.newChild('storage', {})
const controller = new Controller()
const transformer = new HtmlTransformer(extensions)
const hocuspocus = new Hocuspocus({
address: '0.0.0.0',
port,
@ -119,23 +126,15 @@ export async function start (
extensions: [
new ActionsExtension({
ctx: extensionsCtx.newChild('actions', {}),
transformer: new HtmlTransformer(extensions)
transformer
}),
new StorageExtension({
ctx: extensionsCtx.newChild('storage', {}),
adapter: new RouterStorageAdapter(
{
minio: new MinioStorageAdapter(storageCtx.newChild('minio', {}), minio, config.TransactorUrl),
mongodb: new MongodbStorageAdapter(
storageCtx.newChild('mongodb', {}),
mongo,
new HtmlTransformer(extensions)
),
platform: new PlatformStorageAdapter(
storageCtx.newChild('platform', {}),
config.TransactorUrl,
new HtmlTransformer(extensions)
)
minio: new MinioStorageAdapter(storageCtx.newChild('minio', {}), minio),
mongodb: new MongodbStorageAdapter(storageCtx.newChild('mongodb', {}), mongo, transformer),
platform: new PlatformStorageAdapter(storageCtx.newChild('platform', {}), transformer)
},
'minio'
)
@ -144,11 +143,142 @@ export async function start (
async onAuthenticate (data: onAuthenticatePayload): Promise<Context> {
ctx.measure('authenticate', 1)
return buildContext(data, controller)
},
return buildContext(data)
async onDestroy (data: onDestroyPayload): Promise<void> {
await controller.close()
}
})
const restCtx = ctx.newChild('REST', {})
const getContext = (token: Token, documentId: string): Context => {
return {
connectionId: generateId(),
workspaceId: token.workspace,
clientFactory: getClientFactory(token, controller),
initialContentId: documentId,
targetContentId: ''
}
}
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/api/content/:classId/:docId/:attribute', async (req, res) => {
console.log('handle request', req.method, req.url)
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const decodedToken = decodeToken(token)
const docId = req.params.docId
const attribute = req.params.attribute
if (docId === undefined || docId === '') {
res.status(400).send({ err: "'docId' is missing" })
return
}
if (attribute === undefined || attribute === '') {
res.status(400).send({ err: "'attribute' is missing" })
return
}
const documentId = `minio://${docId}%${attribute}`
const context = getContext(decodedToken, documentId)
await restCtx.with(`${req.method} /content`, {}, async (ctx) => {
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
const html = await ctx.with('transform', {}, async () => {
let content = ''
await connection.transact((document) => {
content = transformer.fromYdoc(document, attribute)
})
return content
})
res.writeHead(200, { 'Content-Type': 'application/json' })
const json = JSON.stringify({ html })
res.end(json)
} catch (err: any) {
res.status(500).send({ message: err.message })
} finally {
await connection.disconnect()
}
})
res.end()
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.put('/api/content/:classId/:docId/:attribute', async (req, res) => {
console.log('handle request', req.method, req.url)
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const decodedToken = decodeToken(token)
const docId = req.params.docId
const attribute = req.params.attribute
if (docId === undefined || docId === '') {
res.status(400).send({ err: "'docId' is missing" })
return
}
if (attribute === undefined || attribute === '') {
res.status(400).send({ err: "'attribute' is missing" })
return
}
const documentId = `minio://${docId}%${attribute}`
const data = req.body.html ?? '<p></p>'
const context = getContext(decodedToken, documentId)
await restCtx.with(`${req.method} /content`, {}, async (ctx) => {
const update = await ctx.with('transform', {}, () => {
const ydoc = transformer.toYdoc(data, attribute)
return encodeStateAsUpdate(ydoc)
})
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
await ctx.with('update', {}, async () => {
await connection.transact((document) => {
const fragment = document.getXmlFragment(attribute)
document.transact((tr) => {
fragment.delete(0, fragment.length)
applyUpdate(document, update)
})
})
})
} finally {
await connection.disconnect()
}
})
res.status(200).end()
})
const wss = new WebSocketServer({
noServer: true,
perMessageDeflate: {
@ -174,7 +304,7 @@ export async function start (
const server = createServer(app)
server.on('upgrade', (request, socket, head) => {
server.on('upgrade', (request: IncomingMessage, socket: any, head: Buffer) => {
wss.handleUpgrade(request, socket, head, (ws) => {
wss.emit('connection', ws, request)
})

View File

@ -21,7 +21,6 @@ import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'
import { Context } from '../context'
import { StorageAdapter } from './adapter'
import { connect, getTxOperations } from '../platform'
function maybePlatformDocumentId (documentId: string): boolean {
return !documentId.includes('%')
@ -30,19 +29,16 @@ function maybePlatformDocumentId (documentId: string): boolean {
export class MinioStorageAdapter implements StorageAdapter {
constructor (
private readonly ctx: MeasureContext,
private readonly minio: MinioService,
private readonly transactorUrl: string
private readonly minio: MinioService
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const {
decodedToken: { workspace }
} = context
const { workspaceId } = context
return await this.ctx.with('load-document', {}, async (ctx) => {
const minioDocument = await ctx.with('query', {}, async () => {
try {
const buffer = await this.minio.read(workspace, documentId)
const buffer = await this.minio.read(workspaceId, documentId)
return Buffer.concat(buffer)
} catch {
return undefined
@ -69,7 +65,7 @@ export class MinioStorageAdapter implements StorageAdapter {
}
async saveDocument (documentId: string, document: YDoc, context: Context): Promise<void> {
const { decodedToken, token } = context
const { clientFactory, workspaceId } = context
await this.ctx.with('save-document', {}, async (ctx) => {
const buffer = await ctx.with('transform', {}, () => {
@ -79,7 +75,7 @@ export class MinioStorageAdapter implements StorageAdapter {
await ctx.with('update', {}, async () => {
const metadata = { 'content-type': 'application/ydoc' }
await this.minio.put(decodedToken.workspace, documentId, buffer, buffer.length, metadata)
await this.minio.put(workspaceId, documentId, buffer, buffer.length, metadata)
})
// minio file is usually an attachment document
@ -91,24 +87,18 @@ export class MinioStorageAdapter implements StorageAdapter {
}
await ctx.with('platform', {}, async () => {
const connection = await ctx.with('connect', {}, async () => {
return await connect(this.transactorUrl, token)
const client = await ctx.with('connect', {}, async () => {
return await clientFactory({ derived: true })
})
try {
const client = await getTxOperations(connection, decodedToken, true)
const current = await ctx.with('query', {}, async () => {
return await client.findOne(attachment.class.Attachment, { _id: documentId as Ref<Attachment> })
})
const current = await ctx.with('query', {}, async () => {
return await client.findOne(attachment.class.Attachment, { _id: documentId as Ref<Attachment> })
if (current !== undefined) {
await ctx.with('update', {}, async () => {
await client.update(current, { lastModified: Date.now(), size: buffer.length })
})
if (current !== undefined) {
await ctx.with('update', {}, async () => {
await client.update(current, { lastModified: Date.now(), size: buffer.length })
})
}
} finally {
await connection.close()
}
})
})

View File

@ -49,7 +49,7 @@ export class MongodbStorageAdapter implements StorageAdapter {
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { decodedToken } = context
const { workspaceId } = context
const { objectId, objectDomain, objectAttr } = parseDocumentId(documentId)
if (!isValidDocumentId({ objectId, objectDomain, objectAttr })) {
@ -59,7 +59,7 @@ export class MongodbStorageAdapter implements StorageAdapter {
return await this.ctx.with('load-document', {}, async (ctx) => {
const doc = await ctx.with('query', {}, async () => {
const db = this.mongodb.db(toWorkspaceString(decodedToken.workspace))
const db = this.mongodb.db(toWorkspaceString(workspaceId))
return await db.collection(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } })
})

View File

@ -18,7 +18,6 @@ import { Transformer } from '@hocuspocus/transformer'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { connect, getTxOperations } from '../platform'
import { StorageAdapter } from './adapter'
@ -44,12 +43,11 @@ function isValidDocumentId (documentId: PlatformDocumentId): boolean {
export class PlatformStorageAdapter implements StorageAdapter {
constructor (
private readonly ctx: MeasureContext,
private readonly transactorUrl: string,
private readonly transformer: Transformer
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { token } = context
const { clientFactory } = context
const { objectId, objectClass, objectAttr } = parseDocumentId(documentId)
if (!isValidDocumentId({ objectId, objectClass, objectAttr })) {
@ -61,18 +59,14 @@ export class PlatformStorageAdapter implements StorageAdapter {
let content = ''
const client = await ctx.with('connect', {}, async () => {
return await connect(this.transactorUrl, token)
return await clientFactory({ derived: false })
})
try {
const doc = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId }, { projection: { [objectAttr]: 1 } })
})
if (doc !== undefined && objectAttr in doc) {
content = (doc as any)[objectAttr] as string
}
} finally {
await client.close()
const doc = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId }, { projection: { [objectAttr]: 1 } })
})
if (doc !== undefined && objectAttr in doc) {
content = (doc as any)[objectAttr] as string
}
return await ctx.with('transform', {}, () => {
@ -82,7 +76,7 @@ export class PlatformStorageAdapter implements StorageAdapter {
}
async saveDocument (documentId: string, document: YDoc, context: Context): Promise<void> {
const { decodedToken, token } = context
const { clientFactory } = context
const { objectId, objectClass, objectAttr } = parseDocumentId(documentId)
if (!isValidDocumentId({ objectId, objectClass, objectAttr })) {
@ -91,28 +85,23 @@ export class PlatformStorageAdapter implements StorageAdapter {
}
await this.ctx.with('save-document', {}, async (ctx) => {
const connection = await ctx.with('connect', {}, async () => {
return await connect(this.transactorUrl, token)
const client = await ctx.with('connect', {}, async () => {
return await clientFactory({ derived: false })
})
const client = await getTxOperations(connection, decodedToken)
try {
const current = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId })
const current = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId })
})
if (current !== undefined) {
const content = await ctx.with('transform', {}, () => {
return this.transformer.fromYdoc(document, objectAttr)
})
await ctx.with('update', {}, async () => {
if ((current as any)[objectAttr] !== content) {
await client.update(current, { [objectAttr]: content })
}
})
if (current !== undefined) {
const content = await ctx.with('transform', {}, () => {
return this.transformer.fromYdoc(document, objectAttr)
})
await ctx.with('update', {}, async () => {
if ((current as any)[objectAttr] !== content) {
await client.update(current, { [objectAttr]: content })
}
})
}
} finally {
await connection.close()
}
})
}

View File

@ -110,7 +110,7 @@ services:
environment:
- COLLABORATOR_PORT=3078
- SECRET=secret
- TRANSACTOR_URL=ws://localhost:3334
- TRANSACTOR_URL=ws://transactor:3334
- UPLOAD_URL=/files
- MONGO_URL=mongodb://mongodb:27018
- MINIO_ENDPOINT=minio