hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 02:51:54 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-5233 YDoc versioning (#4645)

Browse Source 
Signed-off-by: Alexander Onnikov <alexander.onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2024-02-22 18:39:50 +07:00 committed by GitHub
parent d6909b4605
commit 83e7723f16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
103 changed files with 3293 additions and 655 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.vscode/launch.json vendored
View File

@ -44,6 +44,7 @@
"MINIO_SECRET_KEY": "minioadmin",
"SERVER_SECRET": "secret",
"COLLABORATOR_URL": "ws://localhost:3078",
"COLLABORATOR_API_URL": "http://localhost:3078",
"REKONI_URL": "http://localhost:4004",
"FRONT_URL": "http://localhost:8080",
"ACCOUNTS_URL": "http://localhost:3000",

541
common/config/rush/pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

1
dev/docker-compose.yaml
View File

@ -84,6 +84,7 @@ services:
- TRANSACTOR_URL=ws://localhost:3333
- ELASTIC_URL=http://elastic:9200
- COLLABORATOR_URL=ws://localhost:3078
- COLLABORATOR_API_URL=http://localhost:3078
- MINIO_ENDPOINT=minio
- MINIO_ACCESS_KEY=minioadmin
- MINIO_SECRET_KEY=minioadmin

1
dev/prod/.env
View File

@ -8,3 +8,4 @@ FRONT_URL=http://localhost:8080
REKONI_URL=http://localhost:4004
COLLABORATOR_URL=ws://locahost:3078
COLLABORATOR_API_URL=http://locahost:3078

1
dev/prod/config.json
View File

@ -2,5 +2,6 @@
"ACCOUNTS_URL":"http://localhost:3000",
"UPLOAD_URL":"/files",
"COLLABORATOR_URL": "ws://localhost:3078",
"COLLABORATOR_API_URL": "http://localhost:3078",
"REKONI_URL": "http://localhost:4004"
}

2
dev/prod/package.json
View File

@ -104,6 +104,8 @@
"@hcengineering/inventory-resources": "^0.6.0",
"@hcengineering/server-attachment": "^0.6.1",
"@hcengineering/server-attachment-resources": "^0.6.0",
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server-contact": "^0.6.1",
"@hcengineering/server-contact-resources": "^0.6.0",
"@hcengineering/server-notification": "^0.6.1",

3
dev/prod/public/config-dev.json
View File

@ -6,5 +6,6 @@
"GMAIL_URL": "https://gmail.hc.engineering",
"CALENDAR_URL": "https://calendar.hc.engineering",
"REKONI_URL": "https://rekoni.hc.engineering",
"COLLABORATOR_URL": "wss://collaborator.hc.engineering"
"COLLABORATOR_URL": "wss://collaborator.hc.engineering",
"COLLABORATOR_API_URL": "https://collaborator.hc.engineering"
}

1
dev/prod/public/config.json
View File

@ -7,5 +7,6 @@
"CALENDAR_URL": "http://localhost:8095",
"REKONI_URL": "http://localhost:4004",
"COLLABORATOR_URL": "ws://localhost:3078",
"COLLABORATOR_API_URL": "http://localhost:3078",
"LAST_NAME_FIRST": "true"
}

2
dev/prod/src/platform.ts
View File

@ -89,6 +89,7 @@ interface Config {
GMAIL_URL: string
CALENDAR_URL: string
COLLABORATOR_URL: string
COLLABORATOR_API_URL: string
TITLE?: string
LANGUAGES?: string
DEFAULT_LANGUAGE?: string
@ -139,6 +140,7 @@ export async function configurePlatform() {
setMetadata(login.metadata.AccountsUrl, config.ACCOUNTS_URL)
setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
setMetadata(presentation.metadata.CollaboratorApiUrl, config.COLLABORATOR_API_URL)
if (config.MODEL_VERSION != null) {
console.log('Minimal Model version requirement', config.MODEL_VERSION)

2
dev/tool/package.json
View File

@ -76,6 +76,8 @@
"@hcengineering/rekoni": "^0.6.0",
"@hcengineering/server-attachment": "^0.6.1",
"@hcengineering/server-attachment-resources": "^0.6.0",
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server-backup": "^0.6.0",
"@hcengineering/server-calendar": "^0.6.0",
"@hcengineering/server-calendar-resources": "^0.6.0",

2
dev/tool/src/__start.ts
View File

@ -24,6 +24,7 @@ import { devTool } from '.'
import { addLocation } from '@hcengineering/platform'
import { serverActivityId } from '@hcengineering/server-activity'
import { serverAttachmentId } from '@hcengineering/server-attachment'
import { serverCollaborationId } from '@hcengineering/server-collaboration'
import { serverCalendarId } from '@hcengineering/server-calendar'
import { serverChunterId } from '@hcengineering/server-chunter'
import { serverContactId } from '@hcengineering/server-contact'
@ -43,6 +44,7 @@ import { serverViewId } from '@hcengineering/server-view'
addLocation(serverActivityId, () => import('@hcengineering/server-activity-resources'))
addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources'))
addLocation(serverCollaborationId, () => import('@hcengineering/server-collaboration-resources'))
addLocation(serverContactId, () => import('@hcengineering/server-contact-resources'))
addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources'))
addLocation(serverChunterId, () => import('@hcengineering/server-chunter-resources'))

1
models/all/package.json
View File

@ -41,6 +41,7 @@
"@hcengineering/model-telegram": "^0.6.0",
"@hcengineering/model-server-core": "^0.6.0",
"@hcengineering/model-server-attachment": "^0.6.0",
"@hcengineering/model-server-collaboration": "^0.6.0",
"@hcengineering/model-server-contact": "^0.6.0",
"@hcengineering/model-server-notification": "^0.6.0",
"@hcengineering/model-server-setting": "^0.6.0",

5
models/all/src/index.ts
View File

@ -35,6 +35,10 @@ import recruit, { recruitId, createModel as recruitModel } from '@hcengineering/
import { requestId, createModel as requestModel } from '@hcengineering/model-request'
import { serverActivityId, createModel as serverActivityModel } from '@hcengineering/model-server-activity'
import { serverAttachmentId, createModel as serverAttachmentModel } from '@hcengineering/model-server-attachment'
import {
serverCollaborationId,
createModel as serverCollaborationModel
} from '@hcengineering/model-server-collaboration'
import { serverCalendarId, createModel as serverCalendarModel } from '@hcengineering/model-server-calendar'
import { serverChunterId, createModel as serverChunterModel } from '@hcengineering/model-server-chunter'
import { serverContactId, createModel as serverContactModel } from '@hcengineering/model-server-contact'
@ -274,6 +278,7 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[
[serverCoreModel, serverCoreId],
[serverAttachmentModel, serverAttachmentId],
[serverCollaborationModel, serverCollaborationId],
[serverContactModel, serverContactId],
[serveSettingModel, serverSettingId],
[serverChunterModel, serverChunterId],

8
models/core/src/core.ts
View File

@ -355,3 +355,11 @@ export class TIndexConfiguration<T extends Doc = Doc> extends TClass implements
indexes!: FieldIndex<T>[]
searchDisabled!: boolean
}
@UX(core.string.CollaborativeDoc)
@Model(core.class.TypeCollaborativeDoc, core.class.Type)
export class TTypeCollaborativeDoc extends TType {}
@UX(core.string.CollaborativeDocVersion)
@Model(core.class.TypeCollaborativeDocVersion, core.class.Type)
export class TTypeCollaborativeDocVersion extends TType {}

4
models/core/src/index.ts
View File

@ -52,6 +52,8 @@ import {
TTypeAny,
TTypeAttachment,
TTypeBoolean,
TTypeCollaborativeDoc,
TTypeCollaborativeDocVersion,
TTypeCollaborativeMarkup,
TTypeDate,
TTypeHyperlink,
@ -110,6 +112,8 @@ export function createModel (builder: Builder): void {
TType,
TEnumOf,
TTypeMarkup,
TTypeCollaborativeDoc,
TTypeCollaborativeDocVersion,
TTypeCollaborativeMarkup,
TArrOf,
TRefTo,

7
models/server-collaboration/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
models/server-collaboration/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

5
models/server-collaboration/config/rig.json Normal file
View File

@ -0,0 +1,5 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "model"
}

36
models/server-collaboration/package.json Normal file
View File

@ -0,0 +1,36 @@
{
"name": "@hcengineering/model-server-collaboration",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src"
},
"template": "@hcengineering/model-package",
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.28",
"@hcengineering/model": "^0.6.7",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-core": "^0.6.1"
}
}

28
models/server-collaboration/src/index.ts Normal file
View File

@ -0,0 +1,28 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Builder } from '@hcengineering/model'
import core from '@hcengineering/core'
import serverCollaboration from '@hcengineering/server-collaboration'
import serverCore from '@hcengineering/server-core'
export { serverCollaborationId } from '@hcengineering/server-collaboration'
export function createModel (builder: Builder): void {
builder.createDoc(serverCore.class.Trigger, core.space.Model, {
trigger: serverCollaboration.trigger.OnDelete
})
}

9
models/server-collaboration/tsconfig.json Normal file
View File

@ -0,0 +1,9 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

8
models/view/src/index.ts
View File

@ -512,6 +512,14 @@ export function createModel (builder: Builder): void {
editor: view.component.CollaborativeHTMLEditor
})
builder.mixin(core.class.TypeCollaborativeDoc, core.class.Class, view.mixin.InlineAttributEditor, {
editor: view.component.CollaborativeDocEditor
})
builder.mixin(core.class.TypeCollaborativeDocVersion, core.class.Class, view.mixin.InlineAttributEditor, {
editor: view.component.CollaborativeDocEditor
})
classPresenter(builder, core.class.TypeBoolean, view.component.BooleanPresenter, view.component.BooleanEditor)
classPresenter(
builder,

1
models/view/src/plugin.ts
View File

@ -70,6 +70,7 @@ export default mergeIds(viewId, view, {
EnumArrayEditor: '' as AnyComponent,
HTMLEditor: '' as AnyComponent,
CollaborativeHTMLEditor: '' as AnyComponent,
CollaborativeDocEditor: '' as AnyComponent,
MarkupEditor: '' as AnyComponent,
MarkupEditorPopup: '' as AnyComponent,
ListView: '' as AnyComponent,

219
packages/collaborator-client/src/client.ts
View File

@ -13,20 +13,111 @@
// limitations under the License.
//
import { Class, Doc, Hierarchy, Markup, Ref, WorkspaceId, concatLink } from '@hcengineering/core'
import { minioDocumentId, mongodbDocumentId } from './utils'
import {
Account,
Class,
CollaborativeDoc,
Doc,
Hierarchy,
Markup,
Ref,
Timestamp,
WorkspaceId,
concatLink,
toCollaborativeDocVersion
} from '@hcengineering/core'
import { DocumentURI, collaborativeDocumentUri, mongodbDocumentUri } from './uri'
/**
* @public
*/
export interface CollaboratorClient {
get: (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string) => Promise<Markup>
update: (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string, value: Markup) => Promise<void>
/** @public */
export interface GetContentRequest {
documentId: DocumentURI
field: string
}
/**
* @public
*/
/** @public */
export interface GetContentResponse {
html: string
}
/** @public */
export interface UpdateContentRequest {
documentId: DocumentURI
field: string
html: string
}
/** @public */
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface UpdateContentResponse {}
/** @public */
export interface CopyContentRequest {
documentId: DocumentURI
sourceField: string
targetField: string
}
/** @public */
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface CopyContentResponse {}
/** @public */
export interface BranchDocumentRequest {
sourceDocumentId: DocumentURI
targetDocumentId: DocumentURI
}
/** @public */
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface BranchDocumentResponse {}
/** @public */
export interface RemoveDocumentRequest {
documentId: DocumentURI
collaborativeDoc: CollaborativeDoc
}
/** @public */
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface RemoveDocumentResponse {}
/** @public */
export interface TakeSnapshotRequest {
documentId: DocumentURI
collaborativeDoc: CollaborativeDoc
createdBy: string
snapshotName: string
}
/** @public */
export interface TakeSnapshotResponse {
versionId: string
name: string
createdBy: string
createdOn: Timestamp
}
/** @public */
export interface CollaborativeDocSnapshotParams {
snapshotName: string
createdBy: Ref<Account>
}
/** @public */
export interface CollaboratorClient {
// field operations
getContent: (collaborativeDoc: CollaborativeDoc, field: string) => Promise<Markup>
updateContent: (collaborativeDoc: CollaborativeDoc, field: string, value: Markup) => Promise<void>
copyContent: (collaborativeDoc: CollaborativeDoc, sourceField: string, targetField: string) => Promise<void>
// document operations
branch: (source: CollaborativeDoc, target: CollaborativeDoc) => Promise<void>
remove: (collaborativeDoc: CollaborativeDoc) => Promise<void>
snapshot: (collaborativeDoc: CollaborativeDoc, params: CollaborativeDocSnapshotParams) => Promise<CollaborativeDoc>
}
/** @public */
export function getClient (
hierarchy: Hierarchy,
workspaceId: WorkspaceId,
@ -44,51 +135,85 @@ class CollaboratorClientImpl implements CollaboratorClient {
private readonly collaboratorUrl: string
) {}
initialContentId (workspace: string, classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): string {
initialContentId (workspace: string, classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): DocumentURI {
const domain = this.hierarchy.getDomain(classId)
return mongodbDocumentId(workspace, domain, docId, attribute)
return mongodbDocumentUri(workspace, domain, docId, attribute)
}
async get (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): Promise<Markup> {
const workspace = this.workspace.name
const documentId = encodeURIComponent(minioDocumentId(workspace, docId, attribute))
const initialContentId = encodeURIComponent(this.initialContentId(workspace, classId, docId, attribute))
attribute = encodeURIComponent(attribute)
const url = concatLink(
this.collaboratorUrl,
`/api/content/${documentId}/${attribute}?initialContentId=${initialContentId}`
)
private async rpc (method: string, payload: any): Promise<any> {
const url = concatLink(this.collaboratorUrl, '/rpc')
const res = await fetch(url, {
method: 'GET',
headers: {
Authorization: 'Bearer ' + this.token,
Accept: 'application/json'
}
})
const json = await res.json()
return json.html ?? '<p></p>'
}
async update (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string, value: Markup): Promise<void> {
const workspace = this.workspace.name
const documentId = encodeURIComponent(minioDocumentId(workspace, docId, attribute))
const initialContentId = encodeURIComponent(this.initialContentId(workspace, classId, docId, attribute))
attribute = encodeURIComponent(attribute)
const url = concatLink(
this.collaboratorUrl,
`/api/content/${documentId}/${attribute}?initialContentId=${initialContentId}`
)
await fetch(url, {
method: 'PUT',
method: 'POST',
headers: {
Authorization: 'Bearer ' + this.token,
'Content-Type': 'application/json'
},
body: JSON.stringify({ html: value })
body: JSON.stringify({ method, payload })
})
const result = await res.json()
if (result.error != null) {
throw new Error(result.error)
}
return result
}
async getContent (collaborativeDoc: CollaborativeDoc, field: string): Promise<Markup> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: GetContentRequest = { documentId, field }
const res = (await this.rpc('getContent', payload)) as GetContentResponse
return res.html ?? ''
}
async updateContent (collaborativeDoc: CollaborativeDoc, field: string, value: Markup): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: UpdateContentRequest = { documentId, field, html: value }
await this.rpc('updateContent', payload)
}
async copyContent (collaborativeDoc: CollaborativeDoc, sourceField: string, targetField: string): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: CopyContentRequest = { documentId, sourceField, targetField }
await this.rpc('copyContent', payload)
}
async branch (source: CollaborativeDoc, target: CollaborativeDoc): Promise<void> {
const workspace = this.workspace.name
const sourceDocumentId = collaborativeDocumentUri(workspace, source)
const targetDocumentId = collaborativeDocumentUri(workspace, target)
const payload: BranchDocumentRequest = { sourceDocumentId, targetDocumentId }
await this.rpc('branchDocument', payload)
}
async remove (collaborativeDoc: CollaborativeDoc): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: RemoveDocumentRequest = { documentId, collaborativeDoc }
await this.rpc('removeDocument', payload)
}
async snapshot (
collaborativeDoc: CollaborativeDoc,
params: CollaborativeDocSnapshotParams
): Promise<CollaborativeDoc> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: TakeSnapshotRequest = { documentId, collaborativeDoc, ...params }
const res = (await this.rpc('takeSnapshot', payload)) as TakeSnapshotResponse
return toCollaborativeDocVersion(collaborativeDoc, res.versionId)
}
}

3
packages/collaborator-client/src/index.ts
View File

@ -13,5 +13,6 @@
// limitations under the License.
//
export { type CollaboratorClient, getClient } from './client'
export * from './client'
export * from './utils'
export * from './uri'

41
packages/collaborator-client/src/uri.ts Normal file
View File

@ -0,0 +1,41 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, CollaborativeDoc, Doc, Domain, Ref, parseCollaborativeDoc } from '@hcengineering/core'
export type DocumentURI = string & { __documentUri: true }
export function collaborativeDocumentUri (workspaceUrl: string, docId: CollaborativeDoc): DocumentURI {
const { documentId, versionId } = parseCollaborativeDoc(docId)
return `minio://${workspaceUrl}/${documentId}/${versionId}` as DocumentURI
}
export function platformDocumentUri (
workspaceUrl: string,
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `platform://${workspaceUrl}/${objectClass}/${objectId}/${objectAttr}` as DocumentURI
}
export function mongodbDocumentUri (
workspaceUrl: string,
domain: Domain,
docId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `mongodb://${workspaceUrl}/${domain}/${docId}/${objectAttr}` as DocumentURI
}

63
packages/core/src/__tests__/collaboration.test.ts Normal file
View File

@ -0,0 +1,63 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
CollaborativeDoc,
formatCollaborativeDoc,
formatCollaborativeDocVersion,
parseCollaborativeDoc
} from '../collaboration'
describe('collaborative-doc', () => {
describe('parseCollaborativeDoc', () => {
it('parses collaborative doc id', async () => {
expect(parseCollaborativeDoc('minioDocumentId:HEAD:0' as CollaborativeDoc)).toEqual({
documentId: 'minioDocumentId',
versionId: 'HEAD',
revisionId: '0'
})
})
it('parses collaborative doc version id', async () => {
expect(parseCollaborativeDoc('minioDocumentId:main' as CollaborativeDoc)).toEqual({
documentId: 'minioDocumentId',
versionId: 'main',
revisionId: 'main'
})
})
})
describe('formatCollaborativeDoc', () => {
it('returns valid collaborative doc id', async () => {
expect(
formatCollaborativeDoc({
documentId: 'minioDocumentId',
versionId: 'HEAD',
revisionId: '0'
})
).toEqual('minioDocumentId:HEAD:0')
})
})
describe('formatCollaborativeDocVersion', () => {
it('returns valid collaborative doc id', async () => {
expect(
formatCollaborativeDocVersion({
documentId: 'minioDocumentId',
versionId: 'versionId'
})
).toEqual('minioDocumentId:versionId')
})
})
})

79
packages/core/src/collaboration.ts Normal file
View File

@ -0,0 +1,79 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc, Ref } from './classes'
/**
* Identifier of the collaborative document holding collaborative content.
*
* Format:
* {minioDocumentId}:{versionId}:{revisionId}
* {minioDocumentId}:{versionId}
*
* @public
* */
export type CollaborativeDoc = string & { __collaborativeDocId: true }
/** @public */
export type CollaborativeDocVersion = string | typeof CollaborativeDocVersionHead
/** @public */
export const CollaborativeDocVersionHead = 'HEAD'
/** @public */
export function getCollaborativeDocId (objectId: Ref<Doc>, objectAttr?: string | undefined): string {
return objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}`
}
/** @public */
export function getCollaborativeDoc (documentId: string): CollaborativeDoc {
return formatCollaborativeDoc({
documentId,
versionId: CollaborativeDocVersionHead,
revisionId: '0'
})
}
/** @public */
export interface CollaborativeDocData {
documentId: string
versionId: CollaborativeDocVersion
revisionId: string
}
/** @public */
export function parseCollaborativeDoc (id: CollaborativeDoc): CollaborativeDocData {
const [documentId, versionId, revisionId] = id.split(':')
return { documentId, versionId, revisionId: revisionId ?? versionId }
}
/** @public */
export function formatCollaborativeDoc ({ documentId, versionId, revisionId }: CollaborativeDocData): CollaborativeDoc {
return `${documentId}:${versionId}:${revisionId}` as CollaborativeDoc
}
/** @public */
export function formatCollaborativeDocVersion ({
documentId,
versionId
}: Omit<CollaborativeDocData, 'revisionId'>): CollaborativeDoc {
return `${documentId}:${versionId}` as CollaborativeDoc
}
/** @public */
export function toCollaborativeDocVersion (collaborativeDoc: CollaborativeDoc, versionId: string): CollaborativeDoc {
const { documentId } = parseCollaborativeDoc(collaborativeDoc)
return formatCollaborativeDocVersion({ documentId, versionId })
}

5
packages/core/src/component.ts
View File

@ -48,6 +48,7 @@ import type {
TypeAny,
UserStatus
} from './classes'
import { CollaborativeDoc } from './collaboration'
import { Status, StatusCategory } from './status'
import type {
Tx,
@ -104,6 +105,8 @@ export default plugin(coreId, {
TypeBoolean: '' as Ref<Class<Type<boolean>>>,
TypeTimestamp: '' as Ref<Class<Type<Timestamp>>>,
TypeDate: '' as Ref<Class<Type<Timestamp | Date>>>,
TypeCollaborativeDoc: '' as Ref<Class<Type<CollaborativeDoc>>>,
TypeCollaborativeDocVersion: '' as Ref<Class<Type<CollaborativeDoc>>>,
TypeCollaborativeMarkup: '' as Ref<Class<Type<Markup>>>,
RefTo: '' as Ref<Class<RefTo<Doc>>>,
ArrOf: '' as Ref<Class<ArrOf<Doc>>>,
@ -163,6 +166,8 @@ export default plugin(coreId, {
Record: '' as IntlString,
Markup: '' as IntlString,
Collaborative: '' as IntlString,
CollaborativeDoc: '' as IntlString,
CollaborativeDocVersion: '' as IntlString,
Number: '' as IntlString,
Boolean: '' as IntlString,
Timestamp: '' as IntlString,

1
packages/core/src/index.ts
View File

@ -15,6 +15,7 @@
//
export * from './classes'
export * from './client'
export * from './collaboration'
export { coreId, systemAccountEmail, default } from './component'
export * from './hierarchy'
export * from './measurements'

16
packages/core/src/tx.ts
View File

@ -452,6 +452,22 @@ export abstract class TxProcessor implements WithTx {
return tx
}
static txHasUpdate<T extends Doc>(tx: TxUpdateDoc<T>, attribute: string): boolean {
const ops = tx.operations
if ((ops as any)[attribute] !== undefined) return true
for (const op in ops) {
if (op.startsWith('$')) {
const opValue = (ops as any)[op]
for (const key in opValue) {
if (key === attribute || key.startsWith(attribute + '.')) {
return true
}
}
}
}
return false
}
protected abstract txCreateDoc (tx: TxCreateDoc<Doc>): Promise<TxResult>
protected abstract txUpdateDoc (tx: TxUpdateDoc<Doc>): Promise<TxResult>
protected abstract txRemoveDoc (tx: TxRemoveDoc<Doc>): Promise<TxResult>

3
packages/core/src/utils.ts
View File

@ -184,7 +184,8 @@ export function isFullTextAttribute (attr: AnyAttribute): boolean {
return (
attr.index === IndexKind.FullText ||
attr.type._class === core.class.TypeAttachment ||
attr.type._class === core.class.EnumOf
attr.type._class === core.class.EnumOf ||
attr.type._class === core.class.TypeCollaborativeDoc
)
}

15
packages/model/src/dsl.ts
View File

@ -20,6 +20,7 @@ import core, {
Class,
Classifier,
ClassifierKind,
CollaborativeDoc,
Data,
DateRangeMode,
Doc,
@ -489,3 +490,17 @@ export function Collection<T extends AttachedDoc> (clazz: Ref<Class<T>>, itemLab
export function ArrOf<T extends PropertyType | Ref<Doc>> (type: Type<T>): TypeArrOf<T> {
return { _class: core.class.ArrOf, label: core.string.Array, of: type }
}
/**
* @public
*/
export function TypeCollaborativeDoc (): Type<CollaborativeDoc> {
return { _class: core.class.TypeCollaborativeDoc, label: core.string.CollaborativeDoc }
}
/**
* @public
*/
export function TypeCollaborativeDocVersion (): Type<CollaborativeDoc> {
return { _class: core.class.TypeCollaborativeDocVersion, label: core.string.CollaborativeDocVersion }
}

55
packages/presentation/src/collaborator.ts
View File

@ -14,43 +14,58 @@
//
import { type CollaboratorClient, getClient as getCollaborator } from '@hcengineering/collaborator-client'
import { getWorkspaceId, type Class, type Doc, type Markup, type Ref } from '@hcengineering/core'
import { type CollaborativeDoc, type Markup, getCurrentAccount, getWorkspaceId } from '@hcengineering/core'
import { getMetadata } from '@hcengineering/platform'
import { getCurrentLocation } from '@hcengineering/ui'
import { getClient } from '.'
import presentation from './plugin'
/**
* @public
*/
/** @public */
export function getCollaboratorClient (): CollaboratorClient {
const workspaceId = getWorkspaceId(getCurrentLocation().path[1] ?? '')
const hierarchy = getClient().getHierarchy()
const token = getMetadata(presentation.metadata.Token) ?? ''
const collaboratorURL = getMetadata(presentation.metadata.CollaboratorUrl) ?? ''
const collaboratorURL = getMetadata(presentation.metadata.CollaboratorApiUrl) ?? ''
return getCollaborator(hierarchy, workspaceId, token, collaboratorURL)
}
/**
* @public
*/
export async function getMarkup (classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): Promise<Markup> {
/** @public */
export async function getMarkup (collaborativeDoc: CollaborativeDoc, field: string): Promise<Markup> {
const client = getCollaboratorClient()
return await client.get(classId, docId, attribute)
return await client.getContent(collaborativeDoc, field)
}
/**
* @public
*/
export async function updateMarkup (
classId: Ref<Class<Doc>>,
docId: Ref<Doc>,
attribute: string,
value: Markup
/** @public */
export async function updateMarkup (collaborativeDoc: CollaborativeDoc, field: string, value: Markup): Promise<void> {
const client = getCollaboratorClient()
await client.updateContent(collaborativeDoc, field, value)
}
/** @public */
export async function copyDocumentContent (
collaborativeDoc: CollaborativeDoc,
sourceField: string,
targetField: string
): Promise<void> {
const client = getCollaboratorClient()
await client.update(classId, docId, attribute, value)
await client.copyContent(collaborativeDoc, sourceField, targetField)
}
/** @public */
export async function copyDocument (source: CollaborativeDoc, target: CollaborativeDoc): Promise<void> {
const client = getCollaboratorClient()
await client.branch(source, target)
}
/** @public */
export async function takeSnapshot (
collaborativeDoc: CollaborativeDoc,
snapshotName: string
): Promise<CollaborativeDoc> {
const client = getCollaboratorClient()
const createdBy = getCurrentAccount()._id
return await client.snapshot(collaborativeDoc, { createdBy, snapshotName })
}

1
packages/presentation/src/plugin.ts
View File

@ -78,6 +78,7 @@ export default plugin(presentationId, {
Draft: '' as Metadata<Record<string, any>>,
UploadURL: '' as Metadata<string>,
CollaboratorUrl: '' as Metadata<string>,
CollaboratorApiUrl: '' as Metadata<string>,
Token: '' as Metadata<string>,
FrontUrl: '' as Asset
}

3
packages/presentation/src/utils.ts
View File

@ -409,6 +409,9 @@ export function getAttributePresenterClass (
if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeMarkup)) {
category = 'inplace'
}
if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeDoc)) {
category = 'inplace'
}
if (hierarchy.isDerived(attrClass, core.class.Collection)) {
attrClass = (attribute.type as Collection<AttachedDoc>).of
category = 'collection'

1
packages/text-editor/package.json
View File

@ -44,6 +44,7 @@
"@hcengineering/ui": "^0.6.11",
"@hcengineering/view": "^0.6.9",
"@hcengineering/text": "^0.6.1",
"@hcengineering/collaborator-client": "^0.6.0",
"svelte": "^4.2.5",
"@tiptap/core": "^2.1.12",
"@tiptap/pm": "^2.1.12",

34
packages/text-editor/src/components/CollaborativeAttributeBox.svelte
View File

@ -13,15 +13,16 @@
// limitations under the License.
-->
<script lang="ts">
import { Doc } from '@hcengineering/core'
import core, { CollaborativeDoc, Doc, getCollaborativeDoc, getCollaborativeDocId } from '@hcengineering/core'
import { IntlString } from '@hcengineering/platform'
import { KeyedAttribute } from '@hcengineering/presentation'
import { KeyedAttribute, getAttribute, getClient } from '@hcengineering/presentation'
import { registerFocus } from '@hcengineering/ui'
import CollaborativeTextEditor from './CollaborativeTextEditor.svelte'
import { FocusExtension } from './extension/focus'
import { FileAttachFunction } from './extension/imageExt'
import textEditorPlugin from '../plugin'
import { minioDocumentId, mongodbDocumentId, platformDocumentId } from '../provider/utils'
import { DocumentId } from '../provider/tiptap'
import { collaborativeDocumentId, mongodbDocumentId, platformDocumentId } from '../provider/utils'
import { RefAction, TextNodeAction } from '../types'
export let object: Doc
@ -42,9 +43,30 @@
let editor: CollaborativeTextEditor
$: documentId = minioDocumentId(object._id, key)
$: initialContentId = mongodbDocumentId(object._id, key)
$: targetContentId = platformDocumentId(object._id, key)
$: documentId = getDocumentId(object, key)
$: initialContentId = getInitialContentId(object, key)
$: targetContentId = platformDocumentId(object._class, object._id, key.key)
function getDocumentId (object: Doc, key: KeyedAttribute): DocumentId {
const value = getAttribute(getClient(), object, key)
if (key.attr.type._class === core.class.TypeCollaborativeDoc) {
return collaborativeDocumentId(value as CollaborativeDoc)
} else if (key.attr.type._class === core.class.TypeCollaborativeDocVersion) {
return collaborativeDocumentId(value as CollaborativeDoc)
} else {
// TODO Remove this when we migrate to minio
const collaborativeDocId = getCollaborativeDocId(object._id, key.key)
const collaborativeDoc = getCollaborativeDoc(collaborativeDocId)
return collaborativeDocumentId(collaborativeDoc)
}
}
function getInitialContentId (object: Doc, key: KeyedAttribute): DocumentId | undefined {
// TODO Remove this when we migrate all content to minio
if (key.attr.type._class === core.class.TypeCollaborativeMarkup) {
return mongodbDocumentId(object._id, key)
}
}
// Focusable control with index
let canBlur = true

2
packages/text-editor/src/index.ts
View File

@ -74,7 +74,7 @@ export {
type TiptapCollabProviderConfiguration,
createTiptapCollaborationData
} from './provider/tiptap'
export { minioDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils'
export { collaborativeDocumentId, minioDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils'
export { CollaborationIds } from './types'
export { textEditorId }

5
packages/text-editor/src/provider/tiptap.ts
View File

@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc as Ydoc } from 'yjs'
import { type DocumentURI } from '@hcengineering/collaborator-client'
import { HocuspocusProvider, type HocuspocusProviderConfiguration } from '@hocuspocus/provider'
import { Doc as Ydoc } from 'yjs'
export type DocumentId = string & { __documentId: true }
export type DocumentId = DocumentURI
export type TiptapCollabProviderConfiguration = HocuspocusProviderConfiguration &
Required<Pick<HocuspocusProviderConfiguration, 'token'>> &

39
packages/text-editor/src/provider/utils.ts
View File

@ -13,30 +13,45 @@
// limitations under the License.
//
import type { Doc, Ref } from '@hcengineering/core'
import {
type Class,
type CollaborativeDoc,
type Doc,
type Ref,
getCollaborativeDoc,
getCollaborativeDocId
} from '@hcengineering/core'
import {
type DocumentURI,
collaborativeDocumentUri,
mongodbDocumentUri,
platformDocumentUri
} from '@hcengineering/collaborator-client'
import { type KeyedAttribute, getClient } from '@hcengineering/presentation'
import { getCurrentLocation } from '@hcengineering/ui'
import { type DocumentId } from './tiptap'
function getWorkspace (): string {
return getCurrentLocation().path[1] ?? ''
}
export function minioDocumentId (docId: Ref<Doc>, attr?: KeyedAttribute): DocumentId {
export function collaborativeDocumentId (docId: CollaborativeDoc): DocumentURI {
const workspace = getWorkspace()
return attr !== undefined
? (`minio://${workspace}/${docId}%${attr.key}` as DocumentId)
: (`minio://${workspace}/${docId}` as DocumentId)
return collaborativeDocumentUri(workspace, docId)
}
export function platformDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
const workspace = getWorkspace()
return `platform://${workspace}/${attr.attr.attributeOf}/${docId}/${attr.key}` as DocumentId
// TODO remove this when migrated QMS documents to new model
export function minioDocumentId (docId: Ref<Doc>, attr?: KeyedAttribute): DocumentURI {
const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(docId, attr?.key))
return collaborativeDocumentId(collaborativeDoc)
}
export function mongodbDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentId {
export function platformDocumentId (objectClass: Ref<Class<Doc>>, objectId: Ref<Doc>, objectAttr: string): DocumentURI {
const workspace = getWorkspace()
return platformDocumentUri(workspace, objectClass, objectId, objectAttr)
}
export function mongodbDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentURI {
const workspace = getWorkspace()
const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf)
return `mongodb://${workspace}/${domain}/${docId}/${attr.key}` as DocumentId
return mongodbDocumentUri(workspace, domain, docId, attr.key)
}

4
plugins/attachment-resources/src/components/AttachmentStyleBoxEditor.svelte
View File

@ -107,6 +107,10 @@
export function isFocused (): boolean {
return descriptionBox.isFocused()
}
export function setEditable (editable: boolean): void {
descriptionBox.setEditable(editable)
}
</script>
{#key object?._id}

29
plugins/view-resources/src/components/CollaborativeDocEditor.svelte Normal file
View File

@ -0,0 +1,29 @@
<!--
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
-->
<script lang="ts">
import { Doc } from '@hcengineering/core'
import { KeyedAttribute } from '@hcengineering/presentation'
import { CollaborativeAttributeSectionBox } from '@hcengineering/text-editor'
export let object: Doc
export let key: KeyedAttribute
</script>
{#key object._id}
{#key key.key}
<CollaborativeAttributeSectionBox {object} {key} label={key.attr.label} />
{/key}
{/key}

2
plugins/view-resources/src/index.ts
View File

@ -43,6 +43,7 @@ import StringFilter from './components/filter/StringFilter.svelte'
import StringFilterPresenter from './components/filter/StringFilterPresenter.svelte'
import TimestampFilter from './components/filter/TimestampFilter.svelte'
import ValueFilter from './components/filter/ValueFilter.svelte'
import CollaborativeDocEditor from './components/CollaborativeDocEditor.svelte'
import CollaborativeHTMLEditor from './components/CollaborativeHTMLEditor.svelte'
import HTMLEditor from './components/HTMLEditor.svelte'
import HTMLPresenter from './components/HTMLPresenter.svelte'
@ -246,6 +247,7 @@ export default async (): Promise<Resources> => ({
FilterTypePopup,
ValueSelector,
HTMLEditor,
CollaborativeDocEditor,
CollaborativeHTMLEditor,
ListView,
GrowPresenter,

1
pods/front/run.sh
View File

@ -5,6 +5,7 @@ export UPLOAD_URL=http://localhost:3333/files
export TRANSACTOR_URL=ws://localhost:3333
export ELASTIC_URL=http://elastic:9200
export COLLABORATOR_URL=ws://localhost:3078
export COLLABORATOR_API_URL=http://localhost:3078
export MINIO_ENDPOINT=minio
export MINIO_ACCESS_KEY=minioadmin
export MINIO_SECRET_KEY=minioadmin

2
pods/server/package.json
View File

@ -52,6 +52,8 @@
"@hcengineering/server-ws": "^0.6.11",
"@hcengineering/server-attachment": "^0.6.1",
"@hcengineering/server-attachment-resources": "^0.6.0",
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server": "^0.6.4",
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/elastic": "^0.6.0",

12
pods/server/src/server.ts
View File

@ -47,6 +47,7 @@ import {
type MinioConfig
} from '@hcengineering/server'
import { serverAttachmentId } from '@hcengineering/server-attachment'
import { CollaborativeContentRetrievalStage, serverCollaborationId } from '@hcengineering/server-collaboration'
import { serverCalendarId } from '@hcengineering/server-calendar'
import { serverChunterId } from '@hcengineering/server-chunter'
import { serverContactId } from '@hcengineering/server-contact'
@ -193,6 +194,7 @@ export function start (
}
): () => Promise<void> {
addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources'))
addLocation(serverCollaborationId, () => import('@hcengineering/server-collaboration-resources'))
addLocation(serverContactId, () => import('@hcengineering/server-contact-resources'))
addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources'))
addLocation(serverSettingId, () => import('@hcengineering/server-setting-resources'))
@ -241,6 +243,16 @@ export function start (
// Obtain text content from storage(like minio) and use content adapter to convert files to text content.
stages.push(new ContentRetrievalStage(storageAdapter, workspace, fullText.newChild('content', {}), contentAdapter))
// Obtain collaborative content
stages.push(
new CollaborativeContentRetrievalStage(
storageAdapter,
workspace,
fullText.newChild('collaborative', {}),
contentAdapter
)
)
// // Add any => english language translation
// const retranslateStage = new LibRetranslateStage(fullText.newChild('retranslate', {}), workspace)
// retranslateStage.clearExcept = stages.map(it => it.stageId)

2
products/tracker/package.json
View File

@ -88,6 +88,8 @@
"@hcengineering/image-cropper-resources": "^0.6.0",
"@hcengineering/server-attachment": "^0.6.1",
"@hcengineering/server-attachment-resources": "^0.6.0",
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server-contact": "^0.6.1",
"@hcengineering/server-contact-resources": "^0.6.0",
"@hcengineering/server-notification": "^0.6.0",

20
rush.json
View File

@ -466,6 +466,11 @@
"projectFolder": "packages/analytics",
"shouldPublish": false
},
{
"packageName": "@hcengineering/collaboration",
"projectFolder": "server/collaboration",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-ws",
"projectFolder": "server/ws",
@ -746,6 +751,21 @@
"projectFolder": "server-plugins/attachment-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-collaboration",
"projectFolder": "server-plugins/collaboration",
"shouldPublish": false
},
{
"packageName": "@hcengineering/model-server-collaboration",
"projectFolder": "models/server-collaboration",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-collaboration-resources",
"projectFolder": "server-plugins/collaboration-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/server-contact",
"projectFolder": "server-plugins/contact",

1
server-plugins/chunter-resources/package.json
View File

@ -39,6 +39,7 @@
"@hcengineering/view": "^0.6.9",
"@hcengineering/login": "^0.6.8",
"@hcengineering/workbench": "^0.6.9",
"@hcengineering/collaboration": "^0.6.0",
"@hcengineering/notification": "^0.6.16",
"@hcengineering/server-notification": "^0.6.1",
"@hcengineering/server-notification-resources": "^0.6.0",

179
server-plugins/chunter-resources/src/backlinks.ts
View File

@ -14,10 +14,11 @@
//
import chunter, { Backlink } from '@hcengineering/chunter'
import { loadCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration'
import core, {
AttachedDoc,
Class,
CollaborativeDoc,
Data,
Doc,
Hierarchy,
@ -26,25 +27,127 @@ import core, {
TxCollectionCUD,
TxCUD,
TxFactory,
TxProcessor
TxProcessor,
Type
} from '@hcengineering/core'
import { ServerKit, extractReferences, getHTML, parseHTML } from '@hcengineering/text'
import { TriggerControl } from '@hcengineering/server-core'
import notification from '@hcengineering/notification'
import { ServerKit, extractReferences, getHTML, parseHTML, yDocContentToNodes } from '@hcengineering/text'
import { StorageAdapter, TriggerControl } from '@hcengineering/server-core'
const extensions = [ServerKit]
export function isMarkupType (type: Ref<Class<Type<any>>>): boolean {
return type === core.class.TypeMarkup || type === core.class.TypeCollaborativeMarkup
}
export function isCollaborativeType (type: Ref<Class<Type<any>>>): boolean {
return type === core.class.TypeCollaborativeDoc
}
export async function getCreateBacklinksTxes (
control: TriggerControl,
storage: StorageAdapter,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Promise<Tx[]> {
const attachedDocId = doc._id
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (isMarkupType(attr.type._class)) {
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
} else if (attr.type._class === core.class.TypeCollaborativeDoc) {
const collaborativeDoc = (doc as any)[attr.name] as CollaborativeDoc
try {
const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx)
if (ydoc !== undefined) {
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, yDocToBuffer(ydoc))
backlinks.push(...attrBacklinks)
}
} catch {
// do nothing, the collaborative doc does not sem to exist yet
}
}
}
return getBacklinksTxes(txFactory, backlinks, [])
}
export async function getUpdateBacklinksTxes (
control: TriggerControl,
storage: StorageAdapter,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Promise<Tx[]> {
const attachedDocId = doc._id
// collect attribute backlinks
let hasBacklinkAttrs = false
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (isMarkupType(attr.type._class)) {
hasBacklinkAttrs = true
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
} else if (attr.type._class === core.class.TypeCollaborativeDoc) {
hasBacklinkAttrs = true
try {
const collaborativeDoc = (doc as any)[attr.name] as CollaborativeDoc
const ydoc = await loadCollaborativeDoc(storage, control.workspace, collaborativeDoc, control.ctx)
if (ydoc !== undefined) {
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, yDocToBuffer(ydoc))
backlinks.push(...attrBacklinks)
}
} catch {
// do nothing, the collaborative doc does not sem to exist yet
}
}
}
// There is a chance that backlinks are managed manually
// do not update backlinks if there are no backlink sources in the doc
if (hasBacklinkAttrs) {
const current = await control.findAll(chunter.class.Backlink, {
backlinkId,
backlinkClass,
attachedDocId,
collection: 'backlinks'
})
return getBacklinksTxes(txFactory, backlinks, current)
}
return []
}
export function getBacklinks (
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>,
attachedDocId: Ref<Doc> | undefined,
content: string
content: string | Buffer
): Array<Data<Backlink>> {
const doc = parseHTML(content, extensions)
const result: Array<Data<Backlink>> = []
const references = extractReferences(doc)
const references = []
if (content instanceof Buffer) {
const nodes = yDocContentToNodes(extensions, content)
for (const node of nodes) {
references.push(...extractReferences(node))
}
} else {
const doc = parseHTML(content, extensions)
references.push(...extractReferences(doc))
}
for (const ref of references) {
if (ref.objectId !== attachedDocId && ref.objectId !== backlinkId) {
result.push({
@ -117,66 +220,6 @@ export function getBacklinksTxes (txFactory: TxFactory, backlinks: Data<Backlink
return txes
}
export function getCreateBacklinksTxes (
control: TriggerControl,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Tx[] {
const attachedDocId = doc._id
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (attr.type._class === core.class.TypeMarkup) {
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
}
}
return getBacklinksTxes(txFactory, backlinks, [])
}
export async function getUpdateBacklinksTxes (
control: TriggerControl,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Promise<Tx[]> {
const attachedDocId = doc._id
// collect attribute backlinks
let hasBacklinkAttrs = false
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (attr.type._class === core.class.TypeMarkup) {
hasBacklinkAttrs = true
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
}
}
// There is a chance that backlinks are managed manually
// do not update backlinks if there are no backlink sources in the doc
if (hasBacklinkAttrs) {
const current = await control.findAll(chunter.class.Backlink, {
backlinkId,
backlinkClass,
attachedDocId,
collection: 'backlinks'
})
return getBacklinksTxes(txFactory, backlinks, current)
}
return []
}
export async function getRemoveBacklinksTxes (
control: TriggerControl,
txFactory: TxFactory,

134
server-plugins/chunter-resources/src/index.ts
View File

@ -27,7 +27,6 @@ import core, {
AttachedDoc,
Class,
concatLink,
Data,
Doc,
DocumentQuery,
FindOptions,
@ -41,8 +40,7 @@ import core, {
TxFactory,
TxProcessor,
TxRemoveDoc,
TxUpdateDoc,
Type
TxUpdateDoc
} from '@hcengineering/core'
import notification, { NotificationContent } from '@hcengineering/notification'
import { getMetadata, IntlString } from '@hcengineering/platform'
@ -57,74 +55,15 @@ import { stripTags } from '@hcengineering/text'
import { Person, PersonAccount } from '@hcengineering/contact'
import activity, { ActivityMessage } from '@hcengineering/activity'
import {
getCreateBacklinksTxes,
getRemoveBacklinksTxes,
getUpdateBacklinksTxes,
guessBacklinkTx,
isMarkupType,
isCollaborativeType
} from './backlinks'
import { IsChannelMessage, IsDirectMessage, IsMeMentioned, IsThreadMessage } from './utils'
import { getBacklinks, getBacklinksTxes, getRemoveBacklinksTxes, guessBacklinkTx } from './backlinks'
export { getBacklinksTxes } from './backlinks'
function isMarkupType (type: Ref<Class<Type<any>>>): boolean {
return type === core.class.TypeMarkup || type === core.class.TypeCollaborativeMarkup
}
function getCreateBacklinksTxes (
control: TriggerControl,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Tx[] {
const attachedDocId = doc._id
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (isMarkupType(attr.type._class)) {
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
}
}
return getBacklinksTxes(txFactory, backlinks, [])
}
async function getUpdateBacklinksTxes (
control: TriggerControl,
txFactory: TxFactory,
doc: Doc,
backlinkId: Ref<Doc>,
backlinkClass: Ref<Class<Doc>>
): Promise<Tx[]> {
const attachedDocId = doc._id
// collect attribute backlinks
let hasBacklinkAttrs = false
const backlinks: Data<Backlink>[] = []
const attributes = control.hierarchy.getAllAttributes(doc._class)
for (const attr of attributes.values()) {
if (isMarkupType(attr.type._class)) {
hasBacklinkAttrs = true
const content = (doc as any)[attr.name]?.toString() ?? ''
const attrBacklinks = getBacklinks(backlinkId, backlinkClass, attachedDocId, content)
backlinks.push(...attrBacklinks)
}
}
// There is a chance that backlinks are managed manually
// do not update backlinks if there are no backlink sources in the doc
if (hasBacklinkAttrs) {
const current = await control.findAll(chunter.class.Backlink, {
backlinkId,
backlinkClass,
attachedDocId,
collection: 'backlinks'
})
return getBacklinksTxes(txFactory, backlinks, current)
}
return []
}
/**
* @public
@ -321,15 +260,24 @@ async function BacklinksCreate (tx: Tx, control: TriggerControl): Promise<Tx[]>
if (ctx._class !== core.class.TxCreateDoc) return []
if (control.hierarchy.isDerived(ctx.objectClass, chunter.class.Backlink)) return []
const txFactory = new TxFactory(control.txFactory.account)
control.storageFx(async (adapter) => {
const txFactory = new TxFactory(control.txFactory.account)
const doc = TxProcessor.createDoc2Doc(ctx)
const targetTx = guessBacklinkTx(control.hierarchy, tx as TxCUD<Doc>)
const txes: Tx[] = getCreateBacklinksTxes(control, txFactory, doc, targetTx.objectId, targetTx.objectClass)
const doc = TxProcessor.createDoc2Doc(ctx)
const targetTx = guessBacklinkTx(control.hierarchy, tx as TxCUD<Doc>)
const txes: Tx[] = await getCreateBacklinksTxes(
control,
adapter,
txFactory,
doc,
targetTx.objectId,
targetTx.objectClass
)
if (txes.length !== 0) {
await control.apply(txes, true)
}
if (txes.length !== 0) {
await control.apply(txes, true)
}
})
return []
}
@ -340,9 +288,11 @@ async function BacklinksUpdate (tx: Tx, control: TriggerControl): Promise<Tx[]>
let hasUpdates = false
const attributes = control.hierarchy.getAllAttributes(ctx.objectClass)
for (const attr of attributes.values()) {
if (isMarkupType(attr.type._class) && attr.name in ctx.operations) {
hasUpdates = true
break
if (isMarkupType(attr.type._class) || isCollaborativeType(attr.type._class)) {
if (TxProcessor.txHasUpdate(ctx, attr.name)) {
hasUpdates = true
break
}
}
}
@ -350,15 +300,23 @@ async function BacklinksUpdate (tx: Tx, control: TriggerControl): Promise<Tx[]>
const rawDoc = (await control.findAll(ctx.objectClass, { _id: ctx.objectId }))[0]
if (rawDoc !== undefined) {
const txFactory = new TxFactory(control.txFactory.account)
control.storageFx(async (adapter) => {
const txFactory = new TxFactory(control.txFactory.account)
const doc = TxProcessor.updateDoc2Doc(rawDoc, ctx)
const targetTx = guessBacklinkTx(control.hierarchy, tx as TxCUD<Doc>)
const txes: Tx[] = await getUpdateBacklinksTxes(
control,
adapter,
txFactory,
doc,
targetTx.objectId,
targetTx.objectClass
)
const doc = TxProcessor.updateDoc2Doc(rawDoc, ctx)
const targetTx = guessBacklinkTx(control.hierarchy, tx as TxCUD<Doc>)
const txes: Tx[] = await getUpdateBacklinksTxes(control, txFactory, doc, targetTx.objectId, targetTx.objectClass)
if (txes.length !== 0) {
await control.apply(txes, true)
}
if (txes.length !== 0) {
await control.apply(txes, true)
}
})
}
}

7
server-plugins/collaboration-resources/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
server-plugins/collaboration-resources/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

4
server-plugins/collaboration-resources/config/rig.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}

7
server-plugins/collaboration-resources/jest.config.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}

38
server-plugins/collaboration-resources/package.json Normal file
View File

@ -0,0 +1,38 @@
{
"name": "@hcengineering/server-collaboration-resources",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.28",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/collaboration": "^0.6.0"
}
}

62
server-plugins/collaboration-resources/src/index.ts Normal file
View File

@ -0,0 +1,62 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import type { CollaborativeDoc, Doc, Tx, TxRemoveDoc } from '@hcengineering/core'
import core, { TxProcessor } from '@hcengineering/core'
import { removeCollaborativeDoc } from '@hcengineering/collaboration'
import type { TriggerControl } from '@hcengineering/server-core'
/**
* @public
*/
export async function OnDelete (tx: Tx, { hierarchy, storageFx, removedMap, ctx }: TriggerControl): Promise<Tx[]> {
const rmTx = TxProcessor.extractTx(tx) as TxRemoveDoc<Doc>
if (rmTx._class !== core.class.TxRemoveDoc) {
return []
}
// Obtain document being deleted
const doc = removedMap.get(rmTx.objectId)
// Ids of files to delete from storage
const toDelete: CollaborativeDoc[] = []
const attributes = hierarchy.getAllAttributes(rmTx.objectClass)
for (const attribute of attributes.values()) {
if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
const value = (doc as any)[attribute.name] as CollaborativeDoc
if (value !== undefined) {
toDelete.push(value)
}
}
}
storageFx(async (adapter, bucket) => {
// TODO This is not accurate way to delete collaborative document
// Even though we are deleting it here, the document can be currently in use by someone else
// and when editing session ends, the collborator service will recreate the document again
await removeCollaborativeDoc(adapter, bucket, toDelete, ctx)
})
return []
}
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
export default async () => ({
trigger: {
OnDelete
}
})

9
server-plugins/collaboration-resources/tsconfig.json Normal file
View File

@ -0,0 +1,9 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

7
server-plugins/collaboration/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
server-plugins/collaboration/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

4
server-plugins/collaboration/config/rig.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}

7
server-plugins/collaboration/jest.config.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}

38
server-plugins/collaboration/package.json Normal file
View File

@ -0,0 +1,38 @@
{
"name": "@hcengineering/server-collaboration",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.28",
"@hcengineering/platform": "^0.6.9",
"@hcengineering/server-core": "^0.6.1"
}
}

173
server-plugins/collaboration/src/fulltext.ts Normal file
View File

@ -0,0 +1,173 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import core, {
Class,
CollaborativeDoc,
Doc,
DocIndexState,
DocumentQuery,
DocumentUpdate,
MeasureContext,
Ref,
WorkspaceId,
parseCollaborativeDoc
} from '@hcengineering/core'
import {
ContentTextAdapter,
DbAdapter,
DocUpdateHandler,
FullTextPipeline,
FullTextPipelineStage,
IndexedDoc,
StorageAdapter,
contentStageId,
docKey,
docUpdKey,
fieldStateId,
getFullTextIndexableAttributes
} from '@hcengineering/server-core'
/**
* @public
*/
export class CollaborativeContentRetrievalStage implements FullTextPipelineStage {
require = []
stageId = contentStageId
extra = ['content', 'base64']
digest = '^digest'
enabled = true
// Clear all except following.
clearExcept: string[] = [fieldStateId, contentStageId]
updateFields: DocUpdateHandler[] = []
textLimit = 100 * 1024
stageValue: boolean | string = true
constructor (
readonly storageAdapter: StorageAdapter | undefined,
readonly workspace: WorkspaceId,
readonly metrics: MeasureContext,
private readonly contentAdapter: ContentTextAdapter
) {}
async initialize (ctx: MeasureContext, storage: DbAdapter, pipeline: FullTextPipeline): Promise<void> {
// Just do nothing
}
async search (
_classes: Ref<Class<Doc>>[],
search: DocumentQuery<Doc>,
size?: number,
from?: number
): Promise<{ docs: IndexedDoc[], pass: boolean }> {
return { docs: [], pass: true }
}
async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline): Promise<void> {
for (const doc of toIndex) {
if (pipeline.cancelling) {
return
}
await this.updateContent(doc, pipeline)
}
}
async updateContent (doc: DocIndexState, pipeline: FullTextPipeline): Promise<void> {
const attributes = getFullTextIndexableAttributes(pipeline.hierarchy, doc.objectClass)
// Copy content attributes as well.
const update: DocumentUpdate<DocIndexState> = {}
if (pipeline.cancelling) {
return
}
try {
for (const [, val] of Object.entries(attributes)) {
if (val.type._class === core.class.TypeCollaborativeDoc) {
const collaborativeDoc = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as CollaborativeDoc
if (collaborativeDoc !== undefined && collaborativeDoc !== '') {
const { documentId } = parseCollaborativeDoc(collaborativeDoc)
let docInfo: any | undefined
try {
docInfo = await this.storageAdapter?.stat(this.workspace, documentId)
} catch (err: any) {
// not found.
}
if (docInfo !== undefined) {
const digest = docInfo.etag
const digestKey = docKey(val.name + this.digest, { _class: val.attributeOf })
if (doc.attributes[digestKey] !== digest) {
;(update as any)[docUpdKey(digestKey)] = digest
const contentType = ((docInfo.metaData['content-type'] as string) ?? '').split(';')[0]
const readable = await this.storageAdapter?.get(this.workspace, documentId)
if (readable !== undefined) {
let textContent = await this.metrics.with(
'fetch',
{},
async () => await this.contentAdapter.content(documentId, contentType, readable)
)
readable?.destroy()
textContent = textContent
.split(/ +|\t+|\f+/)
.filter((it) => it)
.join(' ')
.split(/\n\n+/)
.join('\n')
// trim to large content
if (textContent.length > this.textLimit) {
textContent = textContent.slice(0, this.textLimit)
}
textContent = Buffer.from(textContent).toString('base64')
;(update as any)[docUpdKey(val.name, { _class: val.attributeOf, extra: this.extra })] = textContent
}
}
}
}
}
}
} catch (err: any) {
const wasError = (doc as any).error !== undefined
await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify({ message: err.message, err }) })
if (wasError) {
return
}
// Print error only first time, and update it in doc index
console.error(err)
return
}
await pipeline.update(doc._id, true, update)
}
async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise<void> {
// will be handled by field processor
for (const doc of docs) {
await pipeline.update(doc._id, true, {})
}
}
}

34
server-plugins/collaboration/src/index.ts Normal file
View File

@ -0,0 +1,34 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import type { Plugin, Resource } from '@hcengineering/platform'
import { plugin } from '@hcengineering/platform'
import type { TriggerFunc } from '@hcengineering/server-core'
export * from './fulltext'
/**
* @public
*/
export const serverCollaborationId = 'server-collaboration' as Plugin
/**
* @public
*/
export default plugin(serverCollaborationId, {
trigger: {
OnDelete: '' as Resource<TriggerFunc>
}
})

9
server-plugins/collaboration/tsconfig.json Normal file
View File

@ -0,0 +1,9 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

7
server/collaboration/.eslintrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}

4
server/collaboration/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!/lib/**
!CHANGELOG.md
/lib/**/__tests__/

4
server/collaboration/config/rig.json Normal file
View File

@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}

7
server/collaboration/jest.config.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}

39
server/collaboration/package.json Normal file
View File

@ -0,0 +1,39 @@
{
"name": "@hcengineering/collaboration",
"version": "0.6.0",
"main": "lib/index.js",
"author": "Hardcore Engineering Inc.",
"license": "EPL-2.0",
"scripts": {
"build": "compile",
"build:watch": "compile",
"format": "format src",
"test": "jest --passWithNoTests --silent",
"_phase:build": "compile",
"_phase:test": "jest --passWithNoTests --silent",
"_phase:format": "format src"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@types/node": "~20.11.16",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"prettier-plugin-svelte": "^3.1.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.28",
"@hcengineering/minio": "^0.6.0",
"base64-js": "^1.5.1",
"yjs": "^13.5.52"
}
}

132
server/collaboration/src/history/__tests__/branch.test.ts Normal file
View File

@ -0,0 +1,132 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc as YDoc, encodeStateAsUpdate, encodeStateVector } from 'yjs'
import { yDocBranch, yDocBranchWithGC } from '../branch'
describe('branch', () => {
describe('yDocBranch', () => {
it('branches document without gc', async () => {
const source = new YDoc({ gc: false })
applyGarbageCollectableChanges(source)
const target = yDocBranch(source)
expect(target.gc).toBeFalsy()
// ensure target data
const sourceData = source.getArray('data')
const targetData = target.getArray('data')
expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray()))
// ensure target state
const sourceState = encodeStateVector(source)
const targetState = encodeStateVector(target)
expect(targetState).toEqual(sourceState)
// ensure target updates the same as source regardless of gc
const sourceUpdate = encodeStateAsUpdate(source)
const targetUpdate = encodeStateAsUpdate(target)
expect(targetUpdate).toEqual(sourceUpdate)
})
it('branches document state with gc', async () => {
const source = new YDoc({ gc: true })
applyGarbageCollectableChanges(source)
const target = yDocBranch(source)
expect(target.gc).toBeTruthy()
// ensure target data
const sourceData = source.getArray('data')
const targetData = target.getArray('data')
expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray()))
// ensure target state
const sourceState = encodeStateVector(source)
const targetState = encodeStateVector(target)
expect(targetState).toEqual(sourceState)
// ensure target updates the same as source regardless of gc
const sourceUpdate = encodeStateAsUpdate(source)
const targetUpdate = encodeStateAsUpdate(target)
expect(targetUpdate).toEqual(sourceUpdate)
})
})
describe('yDocBranchWithGC', () => {
it('branches document state without gc', async () => {
const source = new YDoc({ gc: false })
applyGarbageCollectableChanges(source)
const target = yDocBranchWithGC(source)
expect(target.gc).toBeFalsy()
// ensure target data
const sourceData = source.getArray('data')
const targetData = target.getArray('data')
expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray()))
// ensure target state
const sourceState = encodeStateVector(source)
const targetState = encodeStateVector(target)
expect(targetState).toEqual(sourceState)
// ensure target updates different because source is not gc-ed
const sourceUpdate = encodeStateAsUpdate(source)
const targetUpdate = encodeStateAsUpdate(target)
expect(targetUpdate).not.toEqual(sourceUpdate)
})
it('branches document state with gc', async () => {
const source = new YDoc({ gc: true })
applyGarbageCollectableChanges(source)
const target = yDocBranchWithGC(source)
expect(target.gc).toBeTruthy()
// ensure target data
const sourceData = source.getArray('data')
const targetData = target.getArray('data')
expect(targetData.toArray()).toEqual(expect.arrayContaining(sourceData.toArray()))
// ensure target state
const sourceState = encodeStateVector(source)
const targetState = encodeStateVector(target)
expect(targetState).toEqual(sourceState)
// ensure target updates the same because source is gc-ed
const sourceUpdate = encodeStateAsUpdate(source)
const targetUpdate = encodeStateAsUpdate(target)
expect(targetUpdate).toEqual(sourceUpdate)
})
})
function applyGarbageCollectableChanges (ydoc: YDoc): void {
const sourceData = ydoc.getArray('data')
sourceData.insert(0, ['a'])
sourceData.insert(1, [1, 2])
sourceData.delete(0, 1)
sourceData.insert(2, [3])
}
})

159
server/collaboration/src/history/__tests__/history.test.ts Normal file
View File

@ -0,0 +1,159 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { generateId } from '@hcengineering/core'
import { Doc as YDoc, encodeStateAsUpdate } from 'yjs'
import { YDocVersion, addVersion, deleteVersion, getVersion, getVersionData, listVersions } from '../history'
const HISTORY = 'history'
const UPDATES = 'updates'
describe('history', () => {
let ydoc: YDoc
beforeEach(() => {
ydoc = new YDoc()
})
it('addVersion should append new version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
const update = encodeStateAsUpdate(ydoc)
addVersion(ydoc, version, update)
const history = ydoc.getArray(HISTORY)
const updates = ydoc.getMap(UPDATES)
expect(history.length).toEqual(1)
expect(updates.size).toEqual(1)
expect(history.get(0)).toEqual(version)
expect(updates.get(versionId)).toBeDefined()
})
it('addVersion should raise an error when a version already exists', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
const update = encodeStateAsUpdate(ydoc)
addVersion(ydoc, version, update)
expect(() => {
addVersion(ydoc, version, update)
}).toThrow()
const history = ydoc.getArray(HISTORY)
const updates = ydoc.getMap(UPDATES)
expect(history.length).toEqual(1)
expect(updates.size).toEqual(1)
expect(history.get(0)).toEqual(version)
expect(updates.get(versionId)).toBeDefined()
})
it('getVersion should get existing version data', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
const update = encodeStateAsUpdate(ydoc)
addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc))
addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc))
addVersion(ydoc, version, update)
const history = ydoc.getArray(HISTORY)
const updates = ydoc.getMap(UPDATES)
expect(history.length).toEqual(3)
expect(updates.size).toEqual(3)
expect(getVersion(ydoc, versionId)).toEqual(version)
})
it('getVersion should return undefined for unknown version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
addVersion(ydoc, version, encodeStateAsUpdate(ydoc))
expect(getVersion(ydoc, generateId())).toBeUndefined()
})
it('listVersions should return existing versions', async () => {
const version1 = yDocVersion(generateId())
const version2 = yDocVersion(generateId())
addVersion(ydoc, version1, encodeStateAsUpdate(ydoc))
addVersion(ydoc, version2, encodeStateAsUpdate(ydoc))
expect(listVersions(ydoc)).toEqual(expect.arrayContaining([version1, version2]))
})
it('listVersions should return empty list when no versions', async () => {
expect(listVersions(ydoc)).toEqual([])
})
it('getVersionData should get existing version data', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
const update = encodeStateAsUpdate(ydoc)
addVersion(ydoc, version, update)
addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc))
addVersion(ydoc, yDocVersion(generateId()), encodeStateAsUpdate(ydoc))
const history = ydoc.getArray(HISTORY)
const updates = ydoc.getMap(UPDATES)
expect(history.length).toEqual(3)
expect(updates.size).toEqual(3)
expect(getVersionData(ydoc, versionId)).toEqual(update)
})
it('getVersionData should return undefined for unknown version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
addVersion(ydoc, version, encodeStateAsUpdate(ydoc))
expect(getVersionData(ydoc, generateId())).toBeUndefined()
})
it('deleteVersion should delete existing version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
addVersion(ydoc, version, encodeStateAsUpdate(ydoc))
deleteVersion(ydoc, versionId)
const history = ydoc.getArray(HISTORY)
const updates = ydoc.getMap(UPDATES)
expect(history.length).toEqual(0)
expect(updates.size).toEqual(0)
expect(getVersion(ydoc, versionId)).toEqual(undefined)
expect(getVersionData(ydoc, versionId)).toEqual(undefined)
})
})
function yDocVersion (versionId: string): YDocVersion {
return {
versionId,
name: versionId,
createdBy: 'unit test',
createdOn: Date.now()
}
}

109
server/collaboration/src/history/__tests__/snapshot.test.ts Normal file
View File

@ -0,0 +1,109 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { generateId } from '@hcengineering/core'
import { Doc as YDoc } from 'yjs'
import { createYdocSnapshot, restoreYdocSnapshot } from '../snapshot'
import { YDocVersion } from '../history'
const HISTORY = 'history'
const UPDATES = 'updates'
describe('snapshot', () => {
let yContent: YDoc
let yHistory: YDoc
beforeEach(() => {
yContent = new YDoc({ gc: false })
yHistory = new YDoc()
})
it('createYdocSnapshot appends new version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
createYdocSnapshot(yContent, yHistory, version)
const history = yHistory.getArray(HISTORY)
const updates = yHistory.getMap(UPDATES)
expect(history.length).toEqual(1)
expect(updates.size).toEqual(1)
expect(history.get(0)).toEqual(version)
expect(updates.get(versionId)).toBeDefined()
})
it('restoreYdocSnapshot restores existing version', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
const data = yContent.getArray('data')
data.insert(0, [1, 2, 3])
expect(data.toArray()).toEqual(expect.arrayContaining([1, 2, 3]))
createYdocSnapshot(yContent, yHistory, version)
data.delete(1, 1)
expect(data.toArray()).toEqual(expect.arrayContaining([1, 3]))
const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId)
// assert the restored doc has not been changed
expect(yRestore).toBeDefined()
expect(yRestore?.getArray('data').toArray()).toEqual(expect.arrayContaining([1, 2, 3]))
// assert the original doc has not been changed
expect(yContent.getArray('data').toArray()).toEqual(expect.arrayContaining([1, 3]))
})
it('restoreYdocSnapshot throws an error when gc is enabled', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
yContent = new YDoc({ gc: true })
createYdocSnapshot(yContent, yHistory, version)
expect(() => restoreYdocSnapshot(yContent, yHistory, versionId)).toThrow()
})
it('restoreYdocSnapshot does not restore version that does not exist', async () => {
const versionId = generateId()
const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId)
expect(yRestore).toBeUndefined()
})
it('restoreYdocSnapshot restored document has gc enabled', async () => {
const versionId = generateId()
const version = yDocVersion(versionId)
createYdocSnapshot(yContent, yHistory, version)
const yRestore = restoreYdocSnapshot(yContent, yHistory, versionId)
// so far we don't care whether gc is enabled or not in the restore
// but we need to ensure we understand that it is enabled
expect(yRestore?.gc).toEqual(true)
})
})
function yDocVersion (versionId: string): YDocVersion {
return {
versionId,
name: versionId,
createdBy: 'unit test',
createdOn: Date.now()
}
}

53
server/collaboration/src/history/branch.ts Normal file
View File

@ -0,0 +1,53 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'
/**
* Branch (copy) document content as is.
*
* If the source document has gc parameter enabled, then garbage
* collection will be performed. The result document will have the same
* gc parameter value as the source document.
*
* @public
* */
export function yDocBranch (source: YDoc): YDoc {
const target = new YDoc({ gc: source.gc })
const update = encodeStateAsUpdate(source)
applyUpdate(target, update)
return target
}
/**
* Branch (copy) document content with garbage collecting while applying update.
*
* Garbage collection will be performed regardless of the gc parameter
* in the source document. The result document will have the same gc
* parameter value as the source document.
*
* @public
* */
export function yDocBranchWithGC (source: YDoc): YDoc {
const target = new YDoc({ gc: source.gc })
const gc = new YDoc({ gc: true })
applyUpdate(gc, encodeStateAsUpdate(source))
applyUpdate(target, encodeStateAsUpdate(gc))
return target
}

111
server/collaboration/src/history/history.ts Normal file
View File

@ -0,0 +1,111 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Timestamp } from '@hcengineering/core'
import { fromByteArray, toByteArray } from 'base64-js'
import { Doc as YDoc } from 'yjs'
import { YArray, YMap } from 'yjs/dist/src/internals'
/**
* This module provides utils for document version storage based on YDoc
*
* At the top level the history document contains two fields:
* 1. history
* An array containing version ids in creation order
* 2. updates
* A map containing version data keyed by version id
*
* {
* "history": [
* { "versionId": "version1", ... },
* { "versionId": "version2", ... },
* ...
* ],
* "updates": {
* "version1": ... version data as ydoc update base64 encoded ...,
* "version2": ... version data as ydoc update base64 encoded ...,
* ...
* }
* }
*/
/** @public */
export interface YDocVersion {
versionId: string
name: string
createdBy: string
createdOn: Timestamp
}
const HISTORY = 'history'
const UPDATES = 'updates'
function getHistory (ydoc: YDoc): YArray<YDocVersion> {
return ydoc.getArray<YDocVersion>(HISTORY)
}
function getUpdates (ydoc: YDoc): YMap<string> {
return ydoc.getMap<string>(UPDATES)
}
/** @public */
export function addVersion (ydoc: YDoc, version: YDocVersion, update: Uint8Array): void {
const history = getHistory(ydoc)
const updates = getUpdates(ydoc)
const { versionId } = version
if (updates.has(versionId)) {
throw Error('history item already exists')
}
ydoc.transact((tr) => {
history.push([version])
updates.set(versionId, fromByteArray(update))
})
}
/** @public */
export function getVersion (ydoc: YDoc, versionId: string): YDocVersion | undefined {
const history = getHistory(ydoc)
return history.toArray().find((p) => p.versionId === versionId)
}
/** @public */
export function listVersions (ydoc: YDoc): YDocVersion[] {
return getHistory(ydoc).toArray()
}
/** @public */
export function getVersionData (ydoc: YDoc, versionId: string): Uint8Array | undefined {
const updates = getUpdates(ydoc)
const update = updates.get(versionId)
return update !== undefined ? toByteArray(update) : undefined
}
/** @public */
export function deleteVersion (ydoc: YDoc, versionId: string): void {
const history = getHistory(ydoc)
const updates = getUpdates(ydoc)
ydoc.transact((tr) => {
const index = history.toArray().findIndex((p) => p.versionId === versionId)
if (index !== -1) {
history.delete(index, 1)
}
updates.delete(versionId)
})
}

36
server/collaboration/src/history/snapshot.ts Normal file
View File

@ -0,0 +1,36 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc as YDoc } from 'yjs'
import * as Y from 'yjs'
import { YDocVersion, addVersion, getVersionData } from './history'
/** @public */
export function createYdocSnapshot (yContent: YDoc, yHistory: YDoc, version: YDocVersion): void {
const snapshot = Y.snapshot(yContent)
const update = Y.encodeSnapshot(snapshot)
addVersion(yHistory, version, update)
}
/** @public */
export function restoreYdocSnapshot (yContent: YDoc, yHistory: YDoc, versionId: string): YDoc | undefined {
const update = getVersionData(yHistory, versionId)
if (update !== undefined) {
const snapshot = Y.decodeSnapshot(update)
return Y.createDocFromSnapshot(yContent, snapshot)
}
}

21
server/collaboration/src/index.ts Normal file
View File

@ -0,0 +1,21 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export * from './history/branch'
export * from './history/history'
export * from './history/snapshot'
export * from './utils/collaborative-doc'
export * from './utils/minio'
export * from './utils/ydoc'

41
server/collaboration/src/uri.ts Normal file
View File

@ -0,0 +1,41 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, CollaborativeDoc, Doc, Domain, Ref, parseCollaborativeDoc } from '@hcengineering/core'
export type DocumentURI = string & { __documentUri: true }
export function collaborativeDocumentUri (workspaceUrl: string, docId: CollaborativeDoc): DocumentURI {
const { documentId, versionId } = parseCollaborativeDoc(docId)
return `minio://${workspaceUrl}/${documentId}/${versionId}` as DocumentURI
}
export function platformDocumentUri (
workspaceUrl: string,
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `platform://${workspaceUrl}/${objectClass}/${objectId}/${objectAttr}` as DocumentURI
}
export function mongodbDocumentUri (
workspaceUrl: string,
domain: Domain,
docId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `mongodb://${workspaceUrl}/${domain}/${docId}/${objectAttr}` as DocumentURI
}

61
server/collaboration/src/utils/__tests__/collaborative-doc.test.ts Normal file
View File

@ -0,0 +1,61 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { formatCollaborativeDoc } from '@hcengineering/core'
import { collaborativeHistoryDocId, isEditableDoc, isEditableDocVersion } from '../collaborative-doc'
describe('collaborative-doc', () => {
describe('collaborativeHistoryDocId', () => {
it('returns valid history doc id', async () => {
expect(collaborativeHistoryDocId('minioDocumentId')).toEqual('minioDocumentId#history')
})
it('returns valid history doc id for history doc id', async () => {
expect(collaborativeHistoryDocId('minioDocumentId#history')).toEqual('minioDocumentId#history')
})
})
describe('isEditableDoc', () => {
it('returns true for HEAD version', async () => {
const doc = formatCollaborativeDoc({
documentId: 'example',
versionId: 'HEAD',
revisionId: '0'
})
expect(isEditableDoc(doc)).toBeTruthy()
})
it('returns false for other versions', async () => {
const doc = formatCollaborativeDoc({
documentId: 'example',
versionId: 'main',
revisionId: '0'
})
expect(isEditableDoc(doc)).toBeFalsy()
})
})
describe('isEditableDocVersion', () => {
it('returns true for HEAD version', async () => {
expect(isEditableDocVersion('HEAD')).toBeTruthy()
})
it('returns false for other versions', async () => {
expect(isEditableDocVersion('')).toBeFalsy()
expect(isEditableDocVersion('main')).toBeFalsy()
expect(isEditableDocVersion('head')).toBeFalsy()
})
})
})

66
server/collaboration/src/utils/__tests__/ydoc.test.ts Normal file
View File

@ -0,0 +1,66 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Doc as YDoc, XmlElement as YXmlElement, XmlText as YXmlText, encodeStateVector } from 'yjs'
import { yDocCopyXmlField, yDocFromBuffer, yDocToBuffer } from '../ydoc'
describe('ydoc', () => {
it('yDocFromBuffer converts ydoc to a buffer', async () => {
const ydoc = new YDoc()
const buffer = yDocToBuffer(ydoc)
expect(buffer).toBeDefined()
})
it('yDocFromBuffer converts buffer to a ydoc', async () => {
const source = new YDoc()
source.getArray('data').insert(0, [1, 2])
const buffer = yDocToBuffer(source)
const target = yDocFromBuffer(buffer, new YDoc())
expect(target).toBeDefined()
expect(encodeStateVector(target)).toEqual(encodeStateVector(source))
})
describe('yDocCopyXmlField', () => {
it('copies into new field', async () => {
const ydoc = new YDoc()
const source = ydoc.getXmlFragment('source')
source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')])
yDocCopyXmlField(ydoc, 'source', 'target')
const target = ydoc.getXmlFragment('target')
expect(target.toJSON()).toEqual(source.toJSON())
})
it('copies into existing field', async () => {
const ydoc = new YDoc()
const source = ydoc.getXmlFragment('source')
const target = ydoc.getXmlFragment('target')
source.insertAfter(null, [new YXmlElement('p'), new YXmlText('foo'), new YXmlElement('p')])
target.insertAfter(null, [new YXmlText('bar')])
expect(target.toJSON()).not.toEqual(source.toJSON())
yDocCopyXmlField(ydoc, 'source', 'target')
expect(target.toJSON()).toEqual(source.toJSON())
})
})
})

197
server/collaboration/src/utils/collaborative-doc.ts Normal file
View File

@ -0,0 +1,197 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
CollaborativeDoc,
CollaborativeDocVersion,
CollaborativeDocVersionHead,
MeasureContext,
WorkspaceId,
formatCollaborativeDoc,
generateId,
parseCollaborativeDoc
} from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Doc as YDoc } from 'yjs'
import { yDocBranch } from '../history/branch'
import { restoreYdocSnapshot } from '../history/snapshot'
import { yDocFromMinio, yDocToMinio } from './minio'
/** @public */
export function collaborativeHistoryDocId (id: string): string {
const suffix = '#history'
return id.endsWith(suffix) ? id : id + suffix
}
/** @public */
export async function loadCollaborativeDoc (
minio: MinioService,
workspace: WorkspaceId,
collaborativeDoc: CollaborativeDoc,
ctx: MeasureContext
): Promise<YDoc | undefined> {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
return await loadCollaborativeDocVersion(minio, workspace, documentId, versionId, ctx)
}
/** @public */
export async function loadCollaborativeDocVersion (
minio: MinioService,
workspace: WorkspaceId,
documentId: string,
versionId: CollaborativeDocVersion,
ctx: MeasureContext
): Promise<YDoc | undefined> {
const historyDocumentId = collaborativeHistoryDocId(documentId)
return await ctx.with('loadCollaborativeDoc', { type: 'content' }, async (ctx) => {
const yContent = await ctx.with('yDocFromMinio', { type: 'content' }, async () => {
return await yDocFromMinio(minio, workspace, documentId, new YDoc({ gc: false }))
})
if (versionId === 'HEAD') {
return yContent
} else {
const yHistory = await ctx.with('yDocFromMinio', { type: 'history' }, async () => {
return await yDocFromMinio(minio, workspace, historyDocumentId, new YDoc({ gc: false }))
})
return await ctx.with('restoreYdocSnapshot', {}, () => {
return restoreYdocSnapshot(yContent, yHistory, versionId)
})
}
})
}
/** @public */
export async function saveCollaborativeDoc (
minio: MinioService,
workspace: WorkspaceId,
collaborativeDoc: CollaborativeDoc,
ydoc: YDoc,
ctx: MeasureContext
): Promise<void> {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
await saveCollaborativeDocVersion(minio, workspace, documentId, versionId, ydoc, ctx)
}
/** @public */
export async function saveCollaborativeDocVersion (
minio: MinioService,
workspace: WorkspaceId,
documentId: string,
versionId: CollaborativeDocVersion,
ydoc: YDoc,
ctx: MeasureContext
): Promise<void> {
await ctx.with('saveCollaborativeDoc', {}, async (ctx) => {
if (versionId === 'HEAD') {
await ctx.with('yDocToMinio', {}, async () => {
await yDocToMinio(minio, workspace, documentId, ydoc)
})
} else {
console.warn('Cannot save non HEAD document version', documentId, versionId)
}
})
}
/** @public */
export async function removeCollaborativeDoc (
minio: MinioService,
workspace: WorkspaceId,
collaborativeDocs: CollaborativeDoc[],
ctx: MeasureContext
): Promise<void> {
await ctx.with('removeollaborativeDoc', {}, async (ctx) => {
const toRemove: string[] = []
for (const collaborativeDoc of collaborativeDocs) {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
if (versionId === CollaborativeDocVersionHead) {
toRemove.push(documentId, collaborativeHistoryDocId(documentId))
} else {
console.warn('Cannot remove non HEAD document version', documentId, versionId)
}
}
if (toRemove.length > 0) {
await ctx.with('remove', {}, async () => {
await minio.remove(workspace, toRemove)
})
}
})
}
/** @public */
export async function copyCollaborativeDoc (
minio: MinioService,
workspace: WorkspaceId,
source: CollaborativeDoc,
target: CollaborativeDoc,
ctx: MeasureContext
): Promise<YDoc | undefined> {
const { documentId: sourceDocumentId, versionId: sourceVersionId } = parseCollaborativeDoc(source)
const { documentId: targetDocumentId, versionId: targetVersionId } = parseCollaborativeDoc(target)
if (sourceDocumentId === targetDocumentId) {
// no need to copy into itself
return
}
await ctx.with('copyCollaborativeDoc', {}, async (ctx) => {
const ySource = await ctx.with('loadCollaborativeDocVersion', {}, async (ctx) => {
return await loadCollaborativeDocVersion(minio, workspace, sourceDocumentId, sourceVersionId, ctx)
})
if (ySource === undefined) {
return
}
const yTarget = await ctx.with('yDocBranch', {}, () => {
return yDocBranch(ySource)
})
await ctx.with('saveCollaborativeDocVersion', {}, async (ctx) => {
await saveCollaborativeDocVersion(minio, workspace, targetDocumentId, targetVersionId, yTarget, ctx)
})
})
}
/** @public */
export function touchCollaborativeDoc (collaborativeDoc: CollaborativeDoc, revisionId?: string): CollaborativeDoc {
revisionId ??= generateId()
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
return formatCollaborativeDoc({ documentId, versionId, revisionId })
}
/** @public */
export function isEditableDoc (id: CollaborativeDoc): boolean {
const data = parseCollaborativeDoc(id)
return isEditableDocVersion(data.versionId)
}
/** @public */
export function isReadonlyDoc (id: CollaborativeDoc): boolean {
return !isEditableDoc(id)
}
/** @public */
export function isEditableDocVersion (version: CollaborativeDocVersion): boolean {
return version === CollaborativeDocVersionHead
}
/** @public */
export function isReadonlyDocVersion (version: CollaborativeDocVersion): boolean {
return !isEditableDocVersion(version)
}

51
server/collaboration/src/utils/minio.ts Normal file
View File

@ -0,0 +1,51 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { WorkspaceId } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Doc as YDoc } from 'yjs'
import { yDocFromBuffer, yDocToBuffer } from './ydoc'
/** @public */
export async function yDocFromMinio (
minio: MinioService,
workspace: WorkspaceId,
minioDocumentId: string,
ydoc?: YDoc
): Promise<YDoc> {
// no need to apply gc because we load existing document
// it is either already gc-ed, or gc not needed and it is disabled
ydoc ??= new YDoc({ gc: false })
try {
const buffer = await minio.read(workspace, minioDocumentId)
return yDocFromBuffer(Buffer.concat(buffer), ydoc)
} catch (err) {
throw new Error('Failed to load ydoc from minio', { cause: err })
}
}
/** @public */
export async function yDocToMinio (
minio: MinioService,
workspace: WorkspaceId,
minioDocumentId: string,
ydoc: YDoc
): Promise<void> {
const buffer = yDocToBuffer(ydoc)
const metadata = { 'content-type': 'application/ydoc' }
await minio.put(workspace, minioDocumentId, buffer, buffer.length, metadata)
}

45
server/collaboration/src/utils/ydoc.ts Normal file
View File

@ -0,0 +1,45 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { AbstractType as YAbstractType, Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'
/** @public */
export function yDocFromBuffer (buffer: Buffer, ydoc: YDoc): YDoc {
try {
const uint8arr = new Uint8Array(buffer)
applyUpdate(ydoc, uint8arr)
return ydoc
} catch (err) {
throw new Error('Failed to apply ydoc update', { cause: err })
}
}
/** @public */
export function yDocToBuffer (ydoc: YDoc): Buffer {
const update = encodeStateAsUpdate(ydoc)
return Buffer.from(update.buffer)
}
/** @public */
export function yDocCopyXmlField (ydoc: YDoc, source: string, target: string): void {
const srcField = ydoc.getXmlFragment(source)
const dstField = ydoc.getXmlFragment(target)
ydoc.transact((tr) => {
// similar to XmlFragment's clone method
dstField.delete(0, dstField.length)
dstField.insert(0, srcField.toArray().map((item) => (item instanceof YAbstractType ? item.clone() : item)) as any)
})
}

9
server/collaboration/tsconfig.json Normal file
View File

@ -0,0 +1,9 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"tsBuildInfoFile": ".build/build.tsbuildinfo"
}
}

3
server/collaborator/package.json
View File

@ -53,10 +53,11 @@
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-token": "^0.6.7",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/attachment": "^0.6.9",
"@hcengineering/client": "^0.6.14",
"@hcengineering/client-resources": "^0.6.23",
"@hcengineering/minio": "^0.6.0",
"@hcengineering/collaboration": "^0.6.0",
"@hcengineering/collaborator-client": "^0.6.0",
"@hcengineering/text": "^0.6.1",
"@hocuspocus/server": "^2.9.0",
"@hocuspocus/transformer": "^2.9.0",

17
server/collaborator/src/rpc/index.ts Normal file
View File

@ -0,0 +1,17 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export * from './methods'
export * from './rpc'

57
server/collaborator/src/rpc/methods/branchDocument.ts Normal file
View File

@ -0,0 +1,57 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { yDocBranchWithGC } from '@hcengineering/collaboration'
import { type BranchDocumentRequest, type BranchDocumentResponse } from '@hcengineering/collaborator-client'
import { MeasureContext } from '@hcengineering/core'
import { applyUpdate, encodeStateAsUpdate } from 'yjs'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function branchDocument (
ctx: MeasureContext,
context: Context,
payload: BranchDocumentRequest,
params: RpcMethodParams
): Promise<BranchDocumentResponse> {
const { sourceDocumentId, targetDocumentId } = payload
const { hocuspocus } = params
const sourceConnection = await ctx.with('connect', { type: 'source' }, async () => {
return await hocuspocus.openDirectConnection(sourceDocumentId, context)
})
const targetConnection = await ctx.with('connect', { type: 'target' }, async () => {
return await hocuspocus.openDirectConnection(targetDocumentId, context)
})
try {
let update = new Uint8Array()
await sourceConnection.transact((document) => {
const copy = yDocBranchWithGC(document)
update = encodeStateAsUpdate(copy)
})
await targetConnection.transact((document) => {
applyUpdate(document, update)
})
} finally {
await sourceConnection.disconnect()
await targetConnection.disconnect()
}
return {}
}

44
server/collaborator/src/rpc/methods/copyContent.ts Normal file
View File

@ -0,0 +1,44 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { yDocCopyXmlField } from '@hcengineering/collaboration'
import { type CopyContentRequest, type CopyContentResponse } from '@hcengineering/collaborator-client'
import { MeasureContext } from '@hcengineering/core'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function copyContent (
ctx: MeasureContext,
context: Context,
payload: CopyContentRequest,
params: RpcMethodParams
): Promise<CopyContentResponse> {
const { documentId, sourceField, targetField } = payload
const { hocuspocus } = params
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
await connection.transact((document) => {
yDocCopyXmlField(document, sourceField, targetField)
})
} finally {
await connection.disconnect()
}
return {}
}

47
server/collaborator/src/rpc/methods/getContent.ts Normal file
View File

@ -0,0 +1,47 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MeasureContext } from '@hcengineering/core'
import { type GetContentRequest, type GetContentResponse } from '@hcengineering/collaborator-client'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function getContent (
ctx: MeasureContext,
context: Context,
payload: GetContentRequest,
params: RpcMethodParams
): Promise<GetContentResponse> {
const { documentId, field } = payload
const { hocuspocus, transformer } = params
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
const html = await ctx.with('transform', {}, async () => {
let content = ''
await connection.transact((document) => {
content = transformer.fromYdoc(document, field)
})
return content
})
return { html }
} finally {
await connection.disconnect()
}
}

31
server/collaborator/src/rpc/methods/index.ts Normal file
View File

@ -0,0 +1,31 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { getContent } from './getContent'
import { copyContent } from './copyContent'
import { updateContent } from './updateContent'
import { branchDocument } from './branchDocument'
import { removeDocument } from './removeDocument'
import { takeSnapshot } from './takeSnapshot'
import { RpcMethod } from '../rpc'
export const methods: Record<string, RpcMethod> = {
getContent,
copyContent,
updateContent,
branchDocument,
removeDocument,
takeSnapshot
}

48
server/collaborator/src/rpc/methods/removeDocument.ts Normal file
View File

@ -0,0 +1,48 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { collaborativeHistoryDocId } from '@hcengineering/collaboration'
import { type RemoveDocumentRequest, type RemoveDocumentResponse } from '@hcengineering/collaborator-client'
import { MeasureContext, parseCollaborativeDoc } from '@hcengineering/core'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function removeDocument (
ctx: MeasureContext,
context: Context,
payload: RemoveDocumentRequest,
params: RpcMethodParams
): Promise<RemoveDocumentResponse> {
const { documentId, collaborativeDoc } = payload
const { hocuspocus, minio } = params
const { workspaceId } = context
const document = hocuspocus.documents.get(documentId)
if (document !== undefined) {
hocuspocus.closeConnections(documentId)
hocuspocus.unloadDocument(document)
}
const { documentId: minioDocumentId } = parseCollaborativeDoc(collaborativeDoc)
const historyDocumentId = collaborativeHistoryDocId(minioDocumentId)
try {
await minio.remove(workspaceId, [minioDocumentId, historyDocumentId])
} catch (err) {
console.error(err)
}
return {}
}

80
server/collaborator/src/rpc/methods/takeSnapshot.ts Normal file
View File

@ -0,0 +1,80 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
YDocVersion,
collaborativeHistoryDocId,
createYdocSnapshot,
yDocFromMinio,
yDocToMinio
} from '@hcengineering/collaboration'
import { type TakeSnapshotRequest, type TakeSnapshotResponse } from '@hcengineering/collaborator-client'
import { CollaborativeDocVersionHead, MeasureContext, generateId, parseCollaborativeDoc } from '@hcengineering/core'
import { Doc as YDoc } from 'yjs'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function takeSnapshot (
ctx: MeasureContext,
context: Context,
payload: TakeSnapshotRequest,
params: RpcMethodParams
): Promise<TakeSnapshotResponse> {
const { collaborativeDoc, documentId, snapshotName, createdBy } = payload
const { hocuspocus, minio } = params
const { workspaceId } = context
const version: YDocVersion = {
versionId: generateId(),
name: snapshotName,
createdBy,
createdOn: Date.now()
}
const { documentId: minioDocumentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
if (versionId !== CollaborativeDocVersionHead) {
throw new Error('invalid document version')
}
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
// load history document directly from minio
const historyDocumentId = collaborativeHistoryDocId(minioDocumentId)
const yHistory = await ctx.with('yDocFromMinio', {}, async () => {
try {
return await yDocFromMinio(minio, workspaceId, historyDocumentId)
} catch {
return new YDoc()
}
})
await ctx.with('createYdocSnapshot', {}, async () => {
await connection.transact((yContent) => {
createYdocSnapshot(yContent, yHistory, version)
})
})
await ctx.with('yDocToMinio', {}, async () => {
await yDocToMinio(minio, workspaceId, historyDocumentId, yHistory)
})
return { ...version }
} finally {
await connection.disconnect()
}
}

55
server/collaborator/src/rpc/methods/updateContent.ts Normal file
View File

@ -0,0 +1,55 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MeasureContext } from '@hcengineering/core'
import { type UpdateContentRequest, type UpdateContentResponse } from '@hcengineering/collaborator-client'
import { applyUpdate, encodeStateAsUpdate } from 'yjs'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
export async function updateContent (
ctx: MeasureContext,
context: Context,
payload: UpdateContentRequest,
params: RpcMethodParams
): Promise<UpdateContentResponse> {
const { documentId, field, html } = payload
const { hocuspocus, transformer } = params
const update = await ctx.with('transform', {}, () => {
const ydoc = transformer.toYdoc(html, field)
return encodeStateAsUpdate(ydoc)
})
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
await ctx.with('update', {}, async () => {
await connection.transact((document) => {
const fragment = document.getXmlFragment(field)
document.transact(() => {
fragment.delete(0, fragment.length)
applyUpdate(document, update)
})
})
})
} finally {
await connection.disconnect()
}
return {}
}

44
server/collaborator/src/rpc/rpc.ts Normal file
View File

@ -0,0 +1,44 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { MeasureContext } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Hocuspocus } from '@hocuspocus/server'
import { Transformer } from '@hocuspocus/transformer'
import { Context } from '../context'
export interface RpcRequest {
method: string
payload: object
}
export interface RpcErrorResponse {
error: string
}
export type RpcResponse = object | RpcErrorResponse
export type RpcMethod = (
ctx: MeasureContext,
context: Context,
payload: any,
params: RpcMethodParams
) => Promise<RpcResponse>
export interface RpcMethodParams {
hocuspocus: Hocuspocus
minio: MinioService
transformer: Transformer
}

158
server/collaborator/src/server.ts
View File

@ -13,6 +13,7 @@
// limitations under the License.
//
import { isReadonlyDocVersion } from '@hcengineering/collaboration'
import { MeasureContext, generateId } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Token, decodeToken } from '@hcengineering/server-token'
@ -25,7 +26,6 @@ import express from 'express'
import { IncomingMessage, createServer } from 'http'
import { MongoClient } from 'mongodb'
import { WebSocket, WebSocketServer } from 'ws'
import { applyUpdate, encodeStateAsUpdate } from 'yjs'
import { getWorkspaceInfo } from './account'
import { Config } from './config'
@ -34,12 +34,11 @@ import { ActionsExtension } from './extensions/action'
import { HtmlTransformer } from './transformers/html'
import { StorageExtension } from './extensions/storage'
import { Controller, getClientFactory } from './platform'
import { MinioStorageAdapter } from './storage/minio'
import { MinioStorageAdapter, parseDocumentId } from './storage/minio'
import { MongodbStorageAdapter } from './storage/mongodb'
import { PlatformStorageAdapter } from './storage/platform'
import { RouterStorageAdapter } from './storage/router'
const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0'
import { RpcErrorResponse, RpcRequest, RpcResponse, methods } from './rpc'
/**
* @public
@ -112,8 +111,10 @@ export async function start (
* options to pass to the ydoc document
*/
yDocOptions: {
gc: gcEnabled,
gcFilter: () => true
// we intentionally disable gc in order to make snapshots working
// see https://github.com/yjs/yjs/blob/v13.5.52/src/utils/Snapshot.js#L162
gc: false,
gcFilter: () => false
},
/**
* If set to false, respects the debounce time of `onStoreDocument` before unloading a document.
@ -144,29 +145,24 @@ export async function start (
async onAuthenticate (data: onAuthenticatePayload): Promise<Context> {
ctx.measure('authenticate', 1)
const context = buildContext(data, controller)
// verify workspace can be accessed with the token
const workspaceInfo = await getWorkspaceInfo(data.token)
// verify document name
let documentName = data.documentName
if (documentName.includes('://')) {
documentName = documentName.split('://', 2)[1]
}
if (documentName.includes('/')) {
const [workspaceUrl] = documentName.split('/', 2)
const { workspaceUrl, versionId } = parseDocumentId(documentName)
// verify workspace url in the document matches the token
if (workspaceInfo.workspace !== workspaceUrl) {
throw new Error('documentName must include workspace')
}
} else {
// verify workspace can be accessed with the token
const workspaceInfo = await getWorkspaceInfo(data.token)
// verify workspace url in the document matches the token
if (workspaceInfo.workspace !== workspaceUrl) {
throw new Error('documentName must include workspace')
}
return context
data.connection.readOnly = isReadonlyDocVersion(versionId)
return buildContext(data, controller)
},
async onDestroy (data: onDestroyPayload): Promise<void> {
@ -174,7 +170,7 @@ export async function start (
}
})
const restCtx = ctx.newChild('REST', {})
const rpcCtx = ctx.newChild('rpc', {})
const getContext = (token: Token, initialContentId?: string): Context => {
return {
@ -187,117 +183,33 @@ export async function start (
}
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.get('/api/content/:documentId/:field', async (req, res) => {
console.log('handle request', req.method, req.url)
app.post('/rpc', async (req, res) => {
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
res.status(403).send({ error: 'Unauthorized' })
return
}
const token = authHeader.split(' ')[1]
const decodedToken = decodeToken(token)
const token = decodeToken(authHeader.split(' ')[1])
const context = getContext(token)
const documentId = req.params.documentId
const field = req.params.field
const initialContentId = req.query.initialContentId as string
if (documentId === undefined || documentId === '') {
res.status(400).send({ err: "'documentId' is missing" })
return
}
if (field === undefined || field === '') {
res.status(400).send({ err: "'field' is missing" })
return
}
const context = getContext(decodedToken, initialContentId)
await restCtx.with(`${req.method} /content`, {}, async (ctx) => {
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
const html = await ctx.with('transform', {}, async () => {
let content = ''
await connection.transact((document) => {
content = transformer.fromYdoc(document, field)
})
return content
})
res.writeHead(200, { 'Content-Type': 'application/json' })
const json = JSON.stringify({ html })
res.end(json)
} catch (err: any) {
res.status(500).send({ message: err.message })
} finally {
await connection.disconnect()
const request = req.body as RpcRequest
const method = methods[request.method]
if (method === undefined) {
const response: RpcErrorResponse = {
error: 'Unknown method'
}
})
res.end()
})
// eslint-disable-next-line @typescript-eslint/no-misused-promises
app.put('/api/content/:documentId/:field', async (req, res) => {
console.log('handle request', req.method, req.url)
const authHeader = req.headers.authorization
if (authHeader === undefined) {
res.status(403).send()
return
}
const token = authHeader.split(' ')[1]
const decodedToken = decodeToken(token)
const documentId = req.params.documentId
const field = req.params.field
const initialContentId = req.query.initialContentId as string
const data = req.body.html ?? '<p></p>'
if (documentId === undefined || documentId === '') {
res.status(400).send({ err: "'documentId' is missing" })
return
}
if (field === undefined || field === '') {
res.status(400).send({ err: "'field' is missing" })
return
}
const context = getContext(decodedToken, initialContentId)
await restCtx.with(`${req.method} /content`, {}, async (ctx) => {
const update = await ctx.with('transform', {}, () => {
const ydoc = transformer.toYdoc(data, field)
return encodeStateAsUpdate(ydoc)
res.status(400).send(response)
} else {
await rpcCtx.with(request.method, {}, async (ctx) => {
try {
const response: RpcResponse = await method(ctx, context, request.payload, { hocuspocus, minio, transformer })
res.status(200).send(response)
} catch (err: any) {
res.status(500).send({ error: err.message })
}
})
const connection = await ctx.with('connect', {}, async () => {
return await hocuspocus.openDirectConnection(documentId, context)
})
try {
await ctx.with('update', {}, async () => {
await connection.transact((document) => {
const fragment = document.getXmlFragment(field)
document.transact((tr) => {
fragment.delete(0, fragment.length)
applyUpdate(document, update)
})
})
})
} finally {
await connection.disconnect()
}
})
res.status(200).end()
}
})
const wss = new WebSocketServer({

94
server/collaborator/src/storage/minio.ts
View File

@ -1,5 +1,5 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
// Copyright © 2023, 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
@ -13,34 +13,32 @@
// limitations under the License.
//
import attachment, { Attachment } from '@hcengineering/attachment'
import { MeasureContext, Ref } from '@hcengineering/core'
import { loadCollaborativeDocVersion, saveCollaborativeDocVersion } from '@hcengineering/collaboration'
import { CollaborativeDocVersion, CollaborativeDocVersionHead, MeasureContext } from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { StorageAdapter } from './adapter'
interface MinioDocumentId {
export interface MinioDocumentId {
workspaceUrl: string
minioDocumentId: string
versionId: CollaborativeDocVersion
}
function parseDocumentId (documentId: string): MinioDocumentId {
const [workspaceUrl, minioDocumentId] = documentId.split('/')
export function parseDocumentId (documentId: string): MinioDocumentId {
const [workspaceUrl, minioDocumentId, versionId] = documentId.split('/')
return {
workspaceUrl: workspaceUrl ?? '',
minioDocumentId: minioDocumentId ?? ''
minioDocumentId: minioDocumentId ?? '',
versionId: versionId ?? CollaborativeDocVersionHead
}
}
function isValidDocumentId (documentId: MinioDocumentId): boolean {
return documentId.minioDocumentId !== '' && documentId.workspaceUrl !== ''
}
function maybePlatformDocumentId (documentId: string): boolean {
return !documentId.includes('%')
return documentId.workspaceUrl !== '' && documentId.minioDocumentId !== '' && documentId.versionId !== ''
}
export class MinioStorageAdapter implements StorageAdapter {
@ -52,86 +50,34 @@ export class MinioStorageAdapter implements StorageAdapter {
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { workspaceId } = context
const { workspaceUrl, minioDocumentId } = parseDocumentId(documentId)
const { workspaceUrl, minioDocumentId, versionId } = parseDocumentId(documentId)
if (!isValidDocumentId({ workspaceUrl, minioDocumentId })) {
if (!isValidDocumentId({ workspaceUrl, minioDocumentId, versionId })) {
console.warn('malformed document id', documentId)
return undefined
}
return await this.ctx.with('load-document', {}, async (ctx) => {
const minioDocument = await ctx.with('query', {}, async () => {
try {
const buffer = await this.minio.read(workspaceId, minioDocumentId)
return Buffer.concat(buffer)
} catch {
return undefined
}
})
if (minioDocument === undefined) {
try {
return await loadCollaborativeDocVersion(this.minio, workspaceId, minioDocumentId, versionId, ctx)
} catch {
return undefined
}
const ydoc = new YDoc()
await ctx.with('transform', {}, () => {
try {
const uint8arr = new Uint8Array(minioDocument)
applyUpdate(ydoc, uint8arr)
} catch (err) {
console.error(err)
}
})
return ydoc
})
}
async saveDocument (documentId: string, document: YDoc, context: Context): Promise<void> {
const { clientFactory, workspaceId } = context
const { workspaceId } = context
const { workspaceUrl, minioDocumentId } = parseDocumentId(documentId)
const { workspaceUrl, minioDocumentId, versionId } = parseDocumentId(documentId)
if (!isValidDocumentId({ workspaceUrl, minioDocumentId })) {
if (!isValidDocumentId({ workspaceUrl, minioDocumentId, versionId })) {
console.warn('malformed document id', documentId)
return undefined
}
await this.ctx.with('save-document', {}, async (ctx) => {
const buffer = await ctx.with('transform', {}, () => {
const updates = encodeStateAsUpdate(document)
return Buffer.from(updates.buffer)
})
await ctx.with('update', {}, async () => {
const metadata = { 'content-type': 'application/ydoc' }
await this.minio.put(workspaceId, minioDocumentId, buffer, buffer.length, metadata)
})
// minio file is usually an attachment document
// we need to touch an attachment from here to notify platform about changes
if (!maybePlatformDocumentId(minioDocumentId)) {
// documentId is not a platform document id, we can skip platform notification
return
}
await ctx.with('platform', {}, async () => {
const client = await ctx.with('connect', {}, async () => {
return await clientFactory({ derived: true })
})
const current = await ctx.with('query', {}, async () => {
return await client.findOne(attachment.class.Attachment, { _id: minioDocumentId as Ref<Attachment> })
})
if (current !== undefined) {
await ctx.with('update', {}, async () => {
await client.update(current, { lastModified: Date.now(), size: buffer.length })
})
}
})
await saveCollaborativeDocVersion(this.minio, workspaceId, minioDocumentId, versionId, document, ctx)
})
}
}

42
server/collaborator/src/storage/platform.ts
View File

@ -13,7 +13,8 @@
// limitations under the License.
//
import { Class, Doc, MeasureContext, Ref } from '@hcengineering/core'
import { touchCollaborativeDoc } from '@hcengineering/collaboration'
import core, { Class, CollaborativeDoc, Doc, MeasureContext, Ref } from '@hcengineering/core'
import { Transformer } from '@hocuspocus/transformer'
import { Doc as YDoc } from 'yjs'
@ -52,6 +53,8 @@ export class PlatformStorageAdapter implements StorageAdapter {
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
console.warn('loading documents from the platform not supported', documentId)
const { clientFactory } = context
const { workspaceUrl, objectId, objectClass, objectAttr } = parseDocumentId(documentId)
@ -67,6 +70,18 @@ export class PlatformStorageAdapter implements StorageAdapter {
return await clientFactory({ derived: false })
})
const hierarchy = client.getHierarchy()
const attribute = hierarchy.findAttribute(objectClass, objectAttr)
if (attribute === undefined) {
console.warn('invalid attribute', objectAttr)
return undefined
}
if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) {
console.warn('unsupported attribute type', attribute?.type._class)
return undefined
}
const doc = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId }, { projection: { [objectAttr]: 1 } })
})
@ -94,18 +109,35 @@ export class PlatformStorageAdapter implements StorageAdapter {
return await clientFactory({ derived: false })
})
const attribute = client.getHierarchy().findAttribute(objectClass, objectAttr)
if (attribute === undefined) {
console.warn('attribute not found', objectClass, objectAttr)
return
}
const current = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId })
})
if (current !== undefined) {
if (current === undefined) {
return
}
const hierarchy = client.getHierarchy()
if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc
const newCollaborativeDoc = touchCollaborativeDoc(collaborativeDoc)
await ctx.with('update', {}, async () => {
await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
})
} else if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) {
// TODO a temporary solution while we are keeping Markup in Mongo
const content = await ctx.with('transform', {}, () => {
return this.transformer.fromYdoc(document, objectAttr)
})
await ctx.with('update', {}, async () => {
if ((current as any)[objectAttr] !== content) {
await client.update(current, { [objectAttr]: content })
}
await client.diffUpdate(current, { [objectAttr]: content })
})
}
})

15
server/collaborator/src/types.ts
View File

@ -13,10 +13,17 @@
// limitations under the License.
//
/** @public */
export interface DocumentId {
workspaceUrl: string
documentId: string
versionId: string
}
/** @public */
export type Action = DocumentCopyAction | DocumentFieldCopyAction | DocumentContentAction
export type StorageType = 'minio' | 'platform'
/** @public */
export interface DocumentContentAction {
action: 'document.content'
params: {
@ -25,6 +32,7 @@ export interface DocumentContentAction {
}
}
/** @public */
export interface DocumentCopyAction {
action: 'document.copy'
params: {
@ -33,6 +41,7 @@ export interface DocumentCopyAction {
}
}
/** @public */
export interface DocumentFieldCopyAction {
action: 'document.field.copy'
params: {
@ -42,8 +51,10 @@ export interface DocumentFieldCopyAction {
}
}
/** @public */
export type ActionStatus = 'completed' | 'failed'
/** @public */
export interface ActionStatusResponse {
action: Action
status: ActionStatus

Some files were not shown because too many files have changed in this diff Show More