hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 19:11:33 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

UBERF-5837 Document branching utilities (#5034)

Browse Source 
Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2024-04-09 16:47:07 +07:00 committed by GitHub
parent 10f59e1028
commit 15eab69f91
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 949 additions and 821 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
packages/collaborator-client/src/__tests__/utils.test.ts Normal file
View File

@ -0,0 +1,42 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { CollaborativeDoc } from '@hcengineering/core'
import { DocumentId } from '../types'
import { formatDocumentId, parseDocumentId } from '../utils'
describe('utils', () => {
it('formatDocumentId', () => {
expect(formatDocumentId('minio', 'ws1', 'doc1:HEAD:v1' as CollaborativeDoc)).toEqual(
'minio://ws1/doc1:HEAD' as DocumentId
)
expect(formatDocumentId('minio', 'ws1', 'doc1:HEAD:v1#doc2:v2:v2' as CollaborativeDoc)).toEqual(
'minio://ws1/doc1:HEAD/doc2:v2' as DocumentId
)
})
describe('parseDocumentId', () => {
expect(parseDocumentId('minio://ws1/doc1:HEAD' as DocumentId)).toEqual({
storage: 'minio',
workspaceUrl: 'ws1',
collaborativeDoc: 'doc1:HEAD:HEAD' as CollaborativeDoc
})
expect(parseDocumentId('minio://ws1/doc1:HEAD/doc2:v2' as DocumentId)).toEqual({
storage: 'minio',
workspaceUrl: 'ws1',
collaborativeDoc: 'doc1:HEAD:HEAD#doc2:v2:v2' as CollaborativeDoc
})
})
})

65
packages/collaborator-client/src/client.ts
View File

@ -15,22 +15,21 @@
import {
Account,
Class,
CollaborativeDoc,
Doc,
Hierarchy,
Markup,
Ref,
Timestamp,
WorkspaceId,
concatLink,
toCollaborativeDocVersion
collaborativeDocWithVersion,
concatLink
} from '@hcengineering/core'
import { DocumentURI, collaborativeDocumentUri, mongodbDocumentUri } from './uri'
import { DocumentId } from './types'
import { formatMinioDocumentId } from './utils'
/** @public */
export interface GetContentRequest {
documentId: DocumentURI
documentId: DocumentId
field: string
}
@ -41,7 +40,7 @@ export interface GetContentResponse {
/** @public */
export interface UpdateContentRequest {
documentId: DocumentURI
documentId: DocumentId
field: string
html: string
}
@ -52,7 +51,7 @@ export interface UpdateContentResponse {}
/** @public */
export interface CopyContentRequest {
documentId: DocumentURI
documentId: DocumentId
sourceField: string
targetField: string
}
@ -63,8 +62,8 @@ export interface CopyContentResponse {}
/** @public */
export interface BranchDocumentRequest {
sourceDocumentId: DocumentURI
targetDocumentId: DocumentURI
sourceDocumentId: DocumentId
targetDocumentId: DocumentId
}
/** @public */
@ -73,8 +72,7 @@ export interface BranchDocumentResponse {}
/** @public */
export interface RemoveDocumentRequest {
documentId: DocumentURI
collaborativeDoc: CollaborativeDoc
documentId: DocumentId
}
/** @public */
@ -83,8 +81,7 @@ export interface RemoveDocumentResponse {}
/** @public */
export interface TakeSnapshotRequest {
documentId: DocumentURI
collaborativeDoc: CollaborativeDoc
documentId: DocumentId
createdBy: Ref<Account>
snapshotName: string
}
@ -135,11 +132,6 @@ class CollaboratorClientImpl implements CollaboratorClient {
private readonly collaboratorUrl: string
) {}
initialContentId (workspace: string, classId: Ref<Class<Doc>>, docId: Ref<Doc>, attribute: string): DocumentURI {
const domain = this.hierarchy.getDomain(classId)
return mongodbDocumentUri(workspace, domain, docId, attribute)
}
private async rpc (method: string, payload: any): Promise<any> {
const url = concatLink(this.collaboratorUrl, '/rpc')
@ -161,59 +153,58 @@ class CollaboratorClientImpl implements CollaboratorClient {
return result
}
async getContent (collaborativeDoc: CollaborativeDoc, field: string): Promise<Markup> {
async getContent (document: CollaborativeDoc, field: string): Promise<Markup> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const documentId = formatMinioDocumentId(workspace, document)
const payload: GetContentRequest = { documentId, field }
const res = (await this.rpc('getContent', payload)) as GetContentResponse
return res.html ?? ''
}
async updateContent (collaborativeDoc: CollaborativeDoc, field: string, value: Markup): Promise<void> {
async updateContent (document: CollaborativeDoc, field: string, value: Markup): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const documentId = formatMinioDocumentId(workspace, document)
const payload: UpdateContentRequest = { documentId, field, html: value }
await this.rpc('updateContent', payload)
}
async copyContent (collaborativeDoc: CollaborativeDoc, sourceField: string, targetField: string): Promise<void> {
async copyContent (document: CollaborativeDoc, sourceField: string, targetField: string): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const documentId = formatMinioDocumentId(workspace, document)
const payload: CopyContentRequest = { documentId, sourceField, targetField }
await this.rpc('copyContent', payload)
}
async branch (source: CollaborativeDoc, target: CollaborativeDoc): Promise<void> {
const workspace = this.workspace.name
const sourceDocumentId = collaborativeDocumentUri(workspace, source)
const targetDocumentId = collaborativeDocumentUri(workspace, target)
const sourceDocumentId = formatMinioDocumentId(workspace, source)
const targetDocumentId = formatMinioDocumentId(workspace, target)
const payload: BranchDocumentRequest = { sourceDocumentId, targetDocumentId }
await this.rpc('branchDocument', payload)
}
async remove (collaborativeDoc: CollaborativeDoc): Promise<void> {
async remove (document: CollaborativeDoc): Promise<void> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: RemoveDocumentRequest = { documentId, collaborativeDoc }
const documentId = formatMinioDocumentId(workspace, document)
const payload: RemoveDocumentRequest = { documentId }
await this.rpc('removeDocument', payload)
}
async snapshot (
collaborativeDoc: CollaborativeDoc,
params: CollaborativeDocSnapshotParams
): Promise<CollaborativeDoc> {
async snapshot (document: CollaborativeDoc, params: CollaborativeDocSnapshotParams): Promise<CollaborativeDoc> {
const workspace = this.workspace.name
const documentId = collaborativeDocumentUri(workspace, collaborativeDoc)
const payload: TakeSnapshotRequest = { documentId, collaborativeDoc, ...params }
const documentId = formatMinioDocumentId(workspace, document)
const payload: TakeSnapshotRequest = { documentId, ...params }
const res = (await this.rpc('takeSnapshot', payload)) as TakeSnapshotResponse
return toCollaborativeDocVersion(collaborativeDoc, res.versionId)
return collaborativeDocWithVersion(document, res.versionId)
}
}

2
packages/collaborator-client/src/index.ts
View File

@ -14,5 +14,5 @@
//
export * from './client'
export * from './types'
export * from './utils'
export * from './uri'

20
packages/collaborator-client/src/types.ts Normal file
View File

@ -0,0 +1,20 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
/** @public */
export type DocumentId = string & { __documentId: true }
/** @public */
export type PlatformDocumentId = string & { __platformDocId: true }

41
packages/collaborator-client/src/uri.ts
View File

@ -1,41 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, CollaborativeDoc, Doc, Domain, Ref, parseCollaborativeDoc } from '@hcengineering/core'
export type DocumentURI = string & { __documentUri: true }
export function collaborativeDocumentUri (workspaceUrl: string, docId: CollaborativeDoc): DocumentURI {
const { documentId, versionId } = parseCollaborativeDoc(docId)
return `minio://${workspaceUrl}/${documentId}/${versionId}` as DocumentURI
}
export function platformDocumentUri (
workspaceUrl: string,
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `platform://${workspaceUrl}/${objectClass}/${objectId}/${objectAttr}` as DocumentURI
}
export function mongodbDocumentUri (
workspaceUrl: string,
domain: Domain,
docId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `mongodb://${workspaceUrl}/${domain}/${docId}/${objectAttr}` as DocumentURI
}

91
packages/collaborator-client/src/utils.ts
View File

@ -13,12 +13,93 @@
// limitations under the License.
//
import { Doc, Domain, Ref } from '@hcengineering/core'
import {
Class,
CollaborativeDoc,
Doc,
Domain,
Ref,
collaborativeDocChain,
collaborativeDocFormat,
collaborativeDocParse,
collaborativeDocUnchain
} from '@hcengineering/core'
import { DocumentId, PlatformDocumentId } from './types'
export function minioDocumentId (workspace: string, docId: Ref<Doc>, attribute?: string): string {
return attribute !== undefined ? `minio://${workspace}/${docId}%${attribute}` : `minio://${workspace}/${docId}`
/** @public */
export function formatMinioDocumentId (workspaceUrl: string, collaborativeDoc: CollaborativeDoc): DocumentId {
return formatDocumentId('minio', workspaceUrl, collaborativeDoc)
}
export function mongodbDocumentId (workspace: string, domain: Domain, docId: Ref<Doc>, attribute: string): string {
return `mongodb://${workspace}/${domain}/${docId}/${attribute}`
/**
* Formats collaborative document as Hocuspocus document name.
*
* The document name is used for document identification on the server so should remain the same even
* when document is updated. Hence, we remove lastVersionId component from CollaborativeDoc.
*
* Example:
* minio://workspace1/doc1:HEAD/doc2:v1
*
* @public
*/
export function formatDocumentId (
storage: string,
workspaceUrl: string,
collaborativeDoc: CollaborativeDoc
): DocumentId {
const path = collaborativeDocUnchain(collaborativeDoc)
.map((p) => {
const { documentId, versionId } = collaborativeDocParse(p)
return `${documentId}:${versionId}`
})
.join('/')
return `${storage}://${workspaceUrl}/${path}` as DocumentId
}
/** @public */
export function parseDocumentId (documentId: DocumentId): {
storage: string
workspaceUrl: string
collaborativeDoc: CollaborativeDoc
} {
const [storage, path] = documentId.split('://')
const [workspaceUrl, ...rest] = path.split('/')
const collaborativeDocs = rest.map((p) => {
const [documentId, versionId] = p.split(':')
return collaborativeDocFormat({ documentId, versionId, lastVersionId: versionId })
})
return {
storage,
workspaceUrl,
collaborativeDoc: collaborativeDocChain(...collaborativeDocs)
}
}
/** @public */
export function formatPlatformDocumentId (
objectDomain: Domain,
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): PlatformDocumentId {
return `${objectDomain}/${objectClass}/${objectId}/${objectAttr}` as PlatformDocumentId
}
/** @public */
export function parsePlatformDocumentId (platformDocumentId: PlatformDocumentId): {
objectDomain: Domain
objectClass: Ref<Class<Doc>>
objectId: Ref<Doc>
objectAttr: string
} {
const [objectDomain, objectClass, objectId, objectAttr] = platformDocumentId.split('/')
return {
objectDomain: objectDomain as Domain,
objectClass: objectClass as Ref<Class<Doc>>,
objectId: objectId as Ref<Doc>,
objectAttr
}
}

182
packages/core/src/__tests__/collaboration.test.ts
View File

@ -15,58 +15,192 @@
import {
CollaborativeDoc,
formatCollaborativeDoc,
formatCollaborativeDocVersion,
parseCollaborativeDoc,
updateCollaborativeDoc
collaborativeDocChain,
collaborativeDocUnchain,
collaborativeDocFormat,
collaborativeDocParse,
collaborativeDocFromCollaborativeDoc,
collaborativeDocFromLastVersion,
collaborativeDocWithVersion,
collaborativeDocWithLastVersion,
collaborativeDocWithSource
} from '../collaboration'
describe('collaborative-doc', () => {
describe('parseCollaborativeDoc', () => {
describe('collaborativeDocChain', () => {
it('chains one collaborative doc', async () => {
expect(collaborativeDocChain('doc1:v1:v1' as CollaborativeDoc)).toEqual('doc1:v1:v1' as CollaborativeDoc)
})
it('chains multiple collaborative docs', async () => {
expect(collaborativeDocChain('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2' as CollaborativeDoc)).toEqual(
'doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc
)
})
it('chains multiple chained collaborative docs', async () => {
expect(
collaborativeDocChain('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc, 'doc3:v3:v3' as CollaborativeDoc)
).toEqual('doc1:v1:v1#doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc)
})
})
describe('collaborativeDocUnchain', () => {
it('unchains one collaborative doc', async () => {
expect(collaborativeDocUnchain('doc1:v1:v1' as CollaborativeDoc)).toEqual(['doc1:v1:v1'] as CollaborativeDoc[])
})
it('unchains multiple collaborative docs', async () => {
expect(collaborativeDocUnchain('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc)).toEqual([
'doc1:v1:v1',
'doc2:v2:v2'
] as CollaborativeDoc[])
})
})
describe('collaborativeDocParse', () => {
it('parses collaborative doc id', async () => {
expect(parseCollaborativeDoc('minioDocumentId:HEAD:0' as CollaborativeDoc)).toEqual({
expect(collaborativeDocParse('minioDocumentId' as CollaborativeDoc)).toEqual({
documentId: 'minioDocumentId',
versionId: 'HEAD',
lastVersionId: '0'
lastVersionId: 'HEAD',
source: []
})
})
it('parses collaborative doc version id', async () => {
expect(parseCollaborativeDoc('minioDocumentId:main' as CollaborativeDoc)).toEqual({
it('parses collaborative doc id with versionId', async () => {
expect(collaborativeDocParse('minioDocumentId:main' as CollaborativeDoc)).toEqual({
documentId: 'minioDocumentId',
versionId: 'main',
lastVersionId: 'main'
lastVersionId: 'main',
source: []
})
})
it('parses collaborative doc id with versionId and lastVersionId', async () => {
expect(collaborativeDocParse('minioDocumentId:HEAD:0' as CollaborativeDoc)).toEqual({
documentId: 'minioDocumentId',
versionId: 'HEAD',
lastVersionId: '0',
source: []
})
})
it('parses collaborative doc id with versionId, lastVersionId, and source', async () => {
expect(
collaborativeDocParse('minioDocumentId:HEAD:0#minioDocumentId1:main#minioDocumentId2:HEAD' as CollaborativeDoc)
).toEqual({
documentId: 'minioDocumentId',
versionId: 'HEAD',
lastVersionId: '0',
source: ['minioDocumentId1:main' as CollaborativeDoc, 'minioDocumentId2:HEAD' as CollaborativeDoc]
})
})
})
describe('formatCollaborativeDoc', () => {
it('returns valid collaborative doc id', async () => {
describe('collaborativeDocFormat', () => {
it('formats collaborative doc id', async () => {
expect(
formatCollaborativeDoc({
collaborativeDocFormat({
documentId: 'minioDocumentId',
versionId: 'HEAD',
lastVersionId: '0'
})
).toEqual('minioDocumentId:HEAD:0')
})
})
describe('formatCollaborativeDocVersion', () => {
it('returns valid collaborative doc id', async () => {
it('formats collaborative doc id with sources', async () => {
expect(
formatCollaborativeDocVersion({
collaborativeDocFormat({
documentId: 'minioDocumentId',
versionId: 'versionId'
versionId: 'HEAD',
lastVersionId: '0',
source: ['minioDocumentId1:main' as CollaborativeDoc, 'minioDocumentId2:HEAD' as CollaborativeDoc]
})
).toEqual('minioDocumentId:versionId')
).toEqual('minioDocumentId:HEAD:0#minioDocumentId1:main#minioDocumentId2:HEAD')
})
it('formats collaborative doc id with invalid characters', async () => {
expect(
collaborativeDocFormat({
documentId: 'doc:id',
versionId: 'version#id',
lastVersionId: 'last:version#id'
})
).toEqual('doc%id:version%id:last%version%id')
})
})
describe('updateCollaborativeDoc', () => {
it('returns valid collaborative doc id', async () => {
expect(updateCollaborativeDoc('minioDocumentId:HEAD:0' as CollaborativeDoc, '1')).toEqual(
'minioDocumentId:HEAD:1'
describe('collaborativeDocWithVersion', () => {
it('updates collaborative doc version id', async () => {
expect(collaborativeDocWithVersion('doc1:HEAD:HEAD' as CollaborativeDoc, 'v1')).toEqual('doc1:v1:v1')
expect(collaborativeDocWithVersion('doc1:HEAD:v1' as CollaborativeDoc, 'v2')).toEqual('doc1:v2:v2')
expect(collaborativeDocWithVersion('doc1:HEAD:v1#doc2:v1:v1' as CollaborativeDoc, 'v2')).toEqual(
'doc1:v2:v2#doc2:v1:v1'
)
})
})
describe('collaborativeDocWithLastVersion', () => {
it('updates collaborative doc version id', async () => {
expect(collaborativeDocWithLastVersion('doc1:HEAD:HEAD' as CollaborativeDoc, 'v1')).toEqual('doc1:HEAD:v1')
expect(collaborativeDocWithLastVersion('doc1:HEAD:v1' as CollaborativeDoc, 'v2')).toEqual('doc1:HEAD:v2')
expect(collaborativeDocWithLastVersion('doc1:HEAD:v1#doc2:v1:v1' as CollaborativeDoc, 'v2')).toEqual(
'doc1:HEAD:v2#doc2:v1:v1'
)
expect(collaborativeDocWithLastVersion('doc1:v1:v1' as CollaborativeDoc, 'v2')).toEqual(
// cannot update last version for non HEAD
'doc1:v1:v1'
)
})
})
describe('collaborativeDocWithSource', () => {
it('updates collaborative doc version id', async () => {
expect(
collaborativeDocWithSource('doc1:HEAD:HEAD' as CollaborativeDoc, 'doc2:v1:v1' as CollaborativeDoc)
).toEqual('doc1:HEAD:HEAD#doc2:v1:v1' as CollaborativeDoc)
expect(collaborativeDocWithSource('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2' as CollaborativeDoc)).toEqual(
'doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc
)
expect(
collaborativeDocWithSource('doc1:v1:v1' as CollaborativeDoc, 'doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc)
).toEqual('doc1:v1:v1#doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc)
expect(
collaborativeDocWithSource('doc1:v1:v1#doc2:v2:v2' as CollaborativeDoc, 'doc3:v3:v3' as CollaborativeDoc)
).toEqual('doc1:v1:v1#doc3:v3:v3' as CollaborativeDoc)
})
})
describe('collaborativeDocFromLastVersion', () => {
it('returns valid collaborative doc id', async () => {
expect(collaborativeDocFromLastVersion('doc1:HEAD:HEAD#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual(
'doc1:HEAD:HEAD#doc2:main:v2#doc3:main:v3'
)
expect(collaborativeDocFromLastVersion('doc1:HEAD:v1#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual(
'doc1:v1:v1#doc2:main:v2#doc3:main:v3'
)
expect(collaborativeDocFromLastVersion('doc1:v1:v1#doc2:main:v2#doc3:main:v3' as CollaborativeDoc)).toEqual(
'doc1:v1:v1#doc2:main:v2#doc3:main:v3'
)
expect(collaborativeDocFromLastVersion('doc1:HEAD:v1' as CollaborativeDoc)).toEqual('doc1:v1:v1')
})
})
describe('collaborativeDocFromCollaborativeDoc', () => {
it('returns valid collaborative doc id', async () => {
expect(
collaborativeDocFromCollaborativeDoc(
'doc1:HEAD:HEAD' as CollaborativeDoc,
'doc2:HEAD:v2#doc3:v3:v3' as CollaborativeDoc
)
).toEqual('doc1:HEAD:HEAD#doc2:v2:v2#doc3:v3:v3')
expect(
collaborativeDocFromCollaborativeDoc(
'doc1:HEAD:HEAD' as CollaborativeDoc,
'doc2:v2:v2#doc3:v3:v3' as CollaborativeDoc
)
).toEqual('doc1:HEAD:HEAD#doc2:v2:v2#doc3:v3:v3')
expect(
collaborativeDocFromCollaborativeDoc(
'doc1:HEAD:HEAD' as CollaborativeDoc,
'doc2:HEAD:HEAD#doc3:v3:v3' as CollaborativeDoc
)
).toEqual('doc1:HEAD:HEAD#doc2:HEAD:HEAD#doc3:v3:v3')
})
})
})

146
packages/core/src/collaboration.ts
View File

@ -19,12 +19,23 @@ import { Doc, Ref } from './classes'
* Identifier of the collaborative document holding collaborative content.
*
* Format:
* {minioDocumentId}:{versionId}:{revisionId}
* {minioDocumentId}:{versionId}:{lastVersionId}
* {minioDocumentId}:{versionId}
*
* Where:
* - minioDocumentId is an identifier of the document in Minio
* - versionId is an identifier of the document version, HEAD for latest editable version
* - lastVersionId is an identifier of the latest available version
*
* The collaborative document may contain one or more such sections chained with # (hash):
* collaborativeDocId#collaborativeDocId#collaborativeDocId#...
*
* When collaborative document does not exist, it will be initialized from the first existing
* document in the list.
*
* @public
* */
export type CollaborativeDoc = string & { __collaborativeDocId: true }
export type CollaborativeDoc = string & { __collaborativeDoc: true }
/** @public */
export type CollaborativeDocVersion = string | typeof CollaborativeDocVersionHead
@ -39,51 +50,138 @@ export function getCollaborativeDocId (objectId: Ref<Doc>, objectAttr?: string |
/** @public */
export function getCollaborativeDoc (documentId: string): CollaborativeDoc {
return formatCollaborativeDoc({
return collaborativeDocFormat({
documentId,
versionId: CollaborativeDocVersionHead,
lastVersionId: '0'
lastVersionId: CollaborativeDocVersionHead
})
}
/** @public */
export interface CollaborativeDocData {
// Id of the document in object storage
documentId: string
// Id of the document version
// HEAD version represents the editable last document version
// Otherwise, it is a readonly version
versionId: CollaborativeDocVersion
// For HEAD versionId it is the latest available document version
// Otherwise, it is the same value as versionId
lastVersionId: string
source?: CollaborativeDoc[]
}
/**
* Merge several collaborative docs into single collaborative doc train.
*
* @public
*/
export function collaborativeDocChain (...docs: CollaborativeDoc[]): CollaborativeDoc {
return docs.join('#') as CollaborativeDoc
}
/**
* Split collaborative doc train into separate collaborative docs.
*
* @public
*/
export function collaborativeDocUnchain (doc: CollaborativeDoc): CollaborativeDoc[] {
return doc.split('#') as CollaborativeDoc[]
}
/** @public */
export function parseCollaborativeDoc (id: CollaborativeDoc): CollaborativeDocData {
const [documentId, versionId, lastVersionId] = id.split(':')
return { documentId, versionId, lastVersionId: lastVersionId ?? versionId }
export function collaborativeDocParse (doc: CollaborativeDoc): CollaborativeDocData {
const [first, ...other] = collaborativeDocUnchain(doc)
const [documentId, versionId, lastVersionId] = first.split(':')
return {
documentId,
versionId: versionId ?? CollaborativeDocVersionHead,
lastVersionId: lastVersionId ?? versionId ?? CollaborativeDocVersionHead,
source: other
}
}
const sanitize = (value: string): string => value.replace(/[:#]/g, '%')
/** @public */
export function formatCollaborativeDoc ({
export function collaborativeDocFormat ({
documentId,
versionId,
lastVersionId
lastVersionId,
source
}: CollaborativeDocData): CollaborativeDoc {
return `${documentId}:${versionId}:${lastVersionId}` as CollaborativeDoc
const parts = [sanitize(documentId), sanitize(versionId), sanitize(lastVersionId)]
const collaborativeDoc = parts.join(':') as CollaborativeDoc
return collaborativeDocChain(collaborativeDoc, ...(source ?? []))
}
/** @public */
export function updateCollaborativeDoc (collaborativeDoc: CollaborativeDoc, lastVersionId: string): CollaborativeDoc {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
return formatCollaborativeDoc({ documentId, versionId, lastVersionId })
/**
* Updates versionId component in the collaborative document.
* Both versionId and lastVersionId will refer to the same collaborative document version.
*
* When versionId is not HEAD, the document will represent a readonly document version (snapshot).
*
* @public
*/
export function collaborativeDocWithVersion (collaborativeDoc: CollaborativeDoc, versionId: string): CollaborativeDoc {
const { documentId, source } = collaborativeDocParse(collaborativeDoc)
return collaborativeDocFormat({ documentId, versionId, lastVersionId: versionId, source })
}
/** @public */
export function formatCollaborativeDocVersion ({
documentId,
versionId
}: Omit<CollaborativeDocData, 'lastVersionId'>): CollaborativeDoc {
return `${documentId}:${versionId}` as CollaborativeDoc
/**
* Updates lastVersionId component in the collaborative document.
*
* When document versionId is HEAD, the function is no-op.
*
* @public
*/
export function collaborativeDocWithLastVersion (
collaborativeDoc: CollaborativeDoc,
lastVersionId: string
): CollaborativeDoc {
const { documentId, versionId, source } = collaborativeDocParse(collaborativeDoc)
return versionId === CollaborativeDocVersionHead
? collaborativeDocFormat({ documentId, versionId, lastVersionId, source })
: collaborativeDoc
}
/** @public */
export function toCollaborativeDocVersion (collaborativeDoc: CollaborativeDoc, versionId: string): CollaborativeDoc {
const { documentId } = parseCollaborativeDoc(collaborativeDoc)
return formatCollaborativeDocVersion({ documentId, versionId })
/**
* Replaces source component in the collaborative document.
*
* @public
*/
export function collaborativeDocWithSource (
collaborativeDoc: CollaborativeDoc,
source: CollaborativeDoc
): CollaborativeDoc {
const { documentId, versionId, lastVersionId } = collaborativeDocParse(collaborativeDoc)
return collaborativeDocFormat({ documentId, versionId, lastVersionId, source: [source] })
}
/**
* Creates collaborative document that refers to the last version from the source collaborative document.
*
* @public
*/
export function collaborativeDocFromLastVersion (collaborativeDoc: CollaborativeDoc): CollaborativeDoc {
const { documentId, lastVersionId, source } = collaborativeDocParse(collaborativeDoc)
return collaborativeDocFormat({
documentId,
versionId: lastVersionId,
lastVersionId,
source
})
}
/**
* Creates collaborative document that refers to the last version from the source collaborative document.
*
* @public
*/
export function collaborativeDocFromCollaborativeDoc (
collaborativeDoc: CollaborativeDoc,
sourceCollaborativeDoc: CollaborativeDoc
): CollaborativeDoc {
return collaborativeDocWithSource(collaborativeDoc, collaborativeDocFromLastVersion(sourceCollaborativeDoc))
}

33
packages/text-editor/src/components/Collaboration.svelte
View File

@ -15,22 +15,39 @@
//
-->
<script lang="ts">
import { onDestroy, setContext } from 'svelte'
import { type Class, type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core'
import { type DocumentId, type PlatformDocumentId } from '@hcengineering/collaborator-client'
import { getMetadata } from '@hcengineering/platform'
import presentation from '@hcengineering/presentation'
import { onDestroy, setContext } from 'svelte'
import textEditorPlugin from '../plugin'
import { DocumentId, TiptapCollabProvider, createTiptapCollaborationData } from '../provider/tiptap'
import { TiptapCollabProvider, createTiptapCollaborationData } from '../provider/tiptap'
import { formatCollaborativeDocumentId, formatPlatformDocumentId } from '../provider/utils'
import { CollaborationIds } from '../types'
export let documentId: DocumentId
export let initialContentId: DocumentId | undefined = undefined
export let targetContentId: DocumentId | undefined = undefined
export let collaborativeDoc: CollaborativeDoc
export let initialCollaborativeDoc: CollaborativeDoc | undefined = undefined
export let objectClass: Ref<Class<Doc>> | undefined = undefined
export let objectId: Ref<Doc> | undefined = undefined
export let objectAttr: string | undefined = undefined
const token = getMetadata(presentation.metadata.Token) ?? ''
const collaboratorURL = getMetadata(textEditorPlugin.metadata.CollaboratorUrl) ?? ''
let _documentId = ''
let initialContentId: DocumentId | undefined
let platformDocumentId: PlatformDocumentId | undefined
$: documentId = formatCollaborativeDocumentId(collaborativeDoc)
$: if (initialCollaborativeDoc !== undefined) {
initialContentId = formatCollaborativeDocumentId(initialCollaborativeDoc)
}
$: if (objectClass !== undefined && objectId !== undefined && objectAttr !== undefined) {
platformDocumentId = formatPlatformDocumentId(objectClass, objectId, objectAttr)
}
let _documentId: DocumentId | undefined
let provider: TiptapCollabProvider | undefined
@ -40,10 +57,10 @@
provider.disconnect()
}
const data = createTiptapCollaborationData({
collaboratorURL,
documentId,
initialContentId,
targetContentId,
platformDocumentId,
collaboratorURL,
token
})
provider = data.provider

29
packages/text-editor/src/components/CollaborativeAttributeBox.svelte
View File

@ -21,8 +21,6 @@
import { FocusExtension } from './extension/focus'
import { type FileAttachFunction } from './extension/types'
import textEditorPlugin from '../plugin'
import { DocumentId } from '../provider/tiptap'
import { collaborativeDocumentId, mongodbDocumentId, platformDocumentId } from '../provider/utils'
import { RefAction, TextNodeAction } from '../types'
export let object: Doc
@ -47,28 +45,18 @@
let editor: CollaborativeTextEditor
$: documentId = getDocumentId(object, key)
$: initialContentId = getInitialContentId(object, key)
$: targetContentId = platformDocumentId(object._class, object._id, key.key)
$: collaborativeDoc = getCollaborativeDocFromAttribute(object, key)
function getDocumentId (object: Doc, key: KeyedAttribute): DocumentId {
function getCollaborativeDocFromAttribute (object: Doc, key: KeyedAttribute): CollaborativeDoc {
const value = getAttribute(getClient(), object, key)
if (key.attr.type._class === core.class.TypeCollaborativeDoc) {
return collaborativeDocumentId(value as CollaborativeDoc)
return value as CollaborativeDoc
} else if (key.attr.type._class === core.class.TypeCollaborativeDocVersion) {
return collaborativeDocumentId(value as CollaborativeDoc)
return value as CollaborativeDoc
} else {
// TODO Remove this when we migrate to minio
const collaborativeDocId = getCollaborativeDocId(object._id, key.key)
const collaborativeDoc = getCollaborativeDoc(collaborativeDocId)
return collaborativeDocumentId(collaborativeDoc)
}
}
function getInitialContentId (object: Doc, key: KeyedAttribute): DocumentId | undefined {
// TODO Remove this when we migrate all content to minio
if (key.attr.type._class === core.class.TypeCollaborativeMarkup) {
return mongodbDocumentId(object._id, key)
return getCollaborativeDoc(collaborativeDocId)
}
}
@ -113,9 +101,10 @@
<CollaborativeTextEditor
bind:this={editor}
{documentId}
{initialContentId}
{targetContentId}
{collaborativeDoc}
objectClass={object._class}
objectId={object._id}
objectAttr={key.key}
{textNodeActions}
{refActions}
{extensions}

28
packages/text-editor/src/components/CollaborativeTextEditor.svelte
View File

@ -15,6 +15,8 @@
//
-->
<script lang="ts">
import { type Class, type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core'
import { type DocumentId, type PlatformDocumentId } from '@hcengineering/collaborator-client'
import { IntlString, getMetadata, translate } from '@hcengineering/platform'
import presentation from '@hcengineering/presentation'
import { Button, IconSize, Loading, themeStore } from '@hcengineering/ui'
@ -31,7 +33,8 @@
import { EditorKit } from '../kits/editor-kit'
import textEditorPlugin from '../plugin'
import { MinioProvider } from '../provider/minio'
import { DocumentId, TiptapCollabProvider } from '../provider/tiptap'
import { TiptapCollabProvider } from '../provider/tiptap'
import { formatCollaborativeDocumentId, formatPlatformDocumentId } from '../provider/utils'
import {
CollaborationIds,
RefAction,
@ -54,10 +57,13 @@
import { InlineStyleToolbarExtension } from './extension/inlineStyleToolbar'
import { completionConfig } from './extensions'
export let documentId: DocumentId
export let collaborativeDoc: CollaborativeDoc
export let initialCollaborativeDoc: CollaborativeDoc | undefined = undefined
export let field: string | undefined = undefined
export let initialContentId: DocumentId | undefined = undefined
export let targetContentId: DocumentId | undefined = undefined
export let objectClass: Ref<Class<Doc>> | undefined
export let objectId: Ref<Doc> | undefined
export let objectAttr: string | undefined
export let readonly = false
@ -93,6 +99,18 @@
const token = getMetadata(presentation.metadata.Token) ?? ''
const collaboratorURL = getMetadata(textEditorPlugin.metadata.CollaboratorUrl) ?? ''
const documentId = formatCollaborativeDocumentId(collaborativeDoc)
let initialContentId: DocumentId | undefined
if (initialCollaborativeDoc !== undefined) {
initialContentId = formatCollaborativeDocumentId(collaborativeDoc)
}
let platformDocumentId: PlatformDocumentId | undefined
if (objectClass !== undefined && objectId !== undefined && objectAttr !== undefined) {
platformDocumentId = formatPlatformDocumentId(objectClass, objectId, objectAttr)
}
const ydoc = getContext<YDoc>(CollaborationIds.Doc) ?? new YDoc()
const contextProvider = getContext<TiptapCollabProvider>(CollaborationIds.Provider)
@ -107,7 +125,7 @@
token,
parameters: {
initialContentId,
targetContentId
platformDocumentId
}
})

19
packages/text-editor/src/components/CollaboratorEditor.svelte
View File

@ -15,23 +15,26 @@
//
-->
<script lang="ts">
import { type Class, type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core'
import { IntlString } from '@hcengineering/platform'
import { IconSize, registerFocus } from '@hcengineering/ui'
import { AnyExtension, Editor, FocusPosition, getMarkRange } from '@tiptap/core'
import { TextSelection } from '@tiptap/pm/state'
import textEditorPlugin from '../plugin'
import { DocumentId } from '../provider/tiptap'
import { TextEditorCommandHandler, TextFormatCategory, TextNodeAction } from '../types'
import CollaborativeTextEditor from './CollaborativeTextEditor.svelte'
import { FileAttachFunction } from './extension/types'
import { NodeUuidExtension, nodeElementQuerySelector } from './extension/nodeUuid'
export let documentId: DocumentId
export let collaborativeDoc: CollaborativeDoc
export let initialCollaborativeDoc: CollaborativeDoc | undefined = undefined
export let field: string | undefined = undefined
export let initialContentId: DocumentId | undefined = undefined
export let targetContentId: DocumentId | undefined = undefined
export let objectClass: Ref<Class<Doc>> | undefined = undefined
export let objectId: Ref<Doc> | undefined = undefined
export let objectAttr: string | undefined = undefined
export let readonly = false
@ -153,10 +156,12 @@
<div class="root">
<CollaborativeTextEditor
bind:this={collaborativeEditor}
{documentId}
{collaborativeDoc}
{initialCollaborativeDoc}
{field}
{initialContentId}
{targetContentId}
{objectClass}
{objectId}
{objectAttr}
{readonly}
{buttonSize}
{placeholder}

3
packages/text-editor/src/index.ts
View File

@ -66,12 +66,11 @@ export { TodoItemExtension, TodoListExtension } from './components/extension/tod
export * from './command/deleteAttachment'
export {
type DocumentId,
TiptapCollabProvider,
type TiptapCollabProviderConfiguration,
createTiptapCollaborationData
} from './provider/tiptap'
export { collaborativeDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils'
export { formatCollaborativeDocumentId, formatPlatformDocumentId } from './provider/utils'
export { CollaborationIds } from './types'
export { textEditorId }

26
packages/text-editor/src/provider/minio.ts
View File

@ -14,9 +14,10 @@
//
import { getMetadata } from '@hcengineering/platform'
import presentation from '@hcengineering/presentation'
import { concatLink } from '@hcengineering/core'
import { collaborativeDocParse, concatLink } from '@hcengineering/core'
import { ObservableV2 as Observable } from 'lib0/observable'
import { type Doc as YDoc, applyUpdate } from 'yjs'
import { type DocumentId, parseDocumentId } from '@hcengineering/collaborator-client'
interface EVENTS {
synced: (...args: any[]) => void
@ -48,23 +49,20 @@ async function fetchContent (doc: YDoc, name: string): Promise<void> {
export class MinioProvider extends Observable<EVENTS> {
loaded: Promise<void>
constructor (name: string, doc: YDoc) {
constructor (documentId: DocumentId, doc: YDoc) {
super()
if (name.startsWith('minio://')) {
name = name.split('://', 2)[1]
if (name.includes('/')) {
// drop workspace part
name = name.split('/', 2)[1]
}
}
void fetchContent(doc, name).then(() => {
this.emit('synced', [this])
})
this.loaded = new Promise((resolve) => {
this.on('synced', resolve)
})
const { collaborativeDoc } = parseDocumentId(documentId)
const { documentId: minioDocumentId, versionId } = collaborativeDocParse(collaborativeDoc)
if (versionId === 'HEAD' && minioDocumentId !== undefined) {
void fetchContent(doc, minioDocumentId).then(() => {
this.emit('synced', [this])
})
}
}
}

20
packages/text-editor/src/provider/tiptap.ts
View File

@ -12,12 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type DocumentURI } from '@hcengineering/collaborator-client'
import { type DocumentId, type PlatformDocumentId } from '@hcengineering/collaborator-client'
import { HocuspocusProvider, type HocuspocusProviderConfiguration } from '@hocuspocus/provider'
import { Doc as Ydoc } from 'yjs'
export type DocumentId = DocumentURI
export type TiptapCollabProviderConfiguration = HocuspocusProviderConfiguration &
Required<Pick<HocuspocusProviderConfiguration, 'token'>> &
Omit<HocuspocusProviderConfiguration, 'parameters'> & {
@ -26,7 +24,7 @@ Omit<HocuspocusProviderConfiguration, 'parameters'> & {
export interface TiptapCollabProviderURLParameters {
initialContentId?: DocumentId
targetContentId?: DocumentId
platformDocumentId?: PlatformDocumentId
}
export class TiptapCollabProvider extends HocuspocusProvider {
@ -40,9 +38,9 @@ export class TiptapCollabProvider extends HocuspocusProvider {
parameters.initialContentId = initialContentId
}
const targetContentId = configuration.parameters?.targetContentId
if (targetContentId !== undefined && targetContentId !== '') {
parameters.targetContentId = targetContentId
const platformDocumentId = configuration.parameters?.platformDocumentId
if (platformDocumentId !== undefined && platformDocumentId !== '') {
parameters.platformDocumentId = platformDocumentId
}
const hocuspocusConfig: HocuspocusProviderConfiguration = {
@ -63,10 +61,10 @@ export class TiptapCollabProvider extends HocuspocusProvider {
}
export const createTiptapCollaborationData = (params: {
documentId: string
initialContentId?: DocumentId
platformDocumentId?: PlatformDocumentId
collaboratorURL: string
documentId: DocumentId
initialContentId: DocumentId | undefined
targetContentId: DocumentId | undefined
token: string
}): { provider: TiptapCollabProvider, ydoc: Ydoc } => {
const ydoc: Ydoc = new Ydoc()
@ -79,7 +77,7 @@ export const createTiptapCollaborationData = (params: {
token: params.token,
parameters: {
initialContentId: params.initialContentId,
targetContentId: params.targetContentId
platformDocumentId: params.platformDocumentId
}
})
}

34
packages/text-editor/src/provider/utils.ts
View File

@ -1,5 +1,5 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
@ -13,32 +13,30 @@
// limitations under the License.
//
import { type Class, type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core'
import { type Ref, type CollaborativeDoc, type Doc, type Class } from '@hcengineering/core'
import {
type DocumentURI,
collaborativeDocumentUri,
mongodbDocumentUri,
platformDocumentUri
type DocumentId,
type PlatformDocumentId,
formatMinioDocumentId,
formatPlatformDocumentId as origFormatPlatformDocumentId
} from '@hcengineering/collaborator-client'
import { type KeyedAttribute, getClient } from '@hcengineering/presentation'
import { getCurrentLocation } from '@hcengineering/ui'
import { getClient } from '@hcengineering/presentation'
function getWorkspace (): string {
return getCurrentLocation().path[1] ?? ''
}
export function collaborativeDocumentId (docId: CollaborativeDoc): DocumentURI {
export function formatCollaborativeDocumentId (collaborativeDoc: CollaborativeDoc): DocumentId {
const workspace = getWorkspace()
return collaborativeDocumentUri(workspace, docId)
return formatMinioDocumentId(workspace, collaborativeDoc)
}
export function platformDocumentId (objectClass: Ref<Class<Doc>>, objectId: Ref<Doc>, objectAttr: string): DocumentURI {
const workspace = getWorkspace()
return platformDocumentUri(workspace, objectClass, objectId, objectAttr)
}
export function mongodbDocumentId (docId: Ref<Doc>, attr: KeyedAttribute): DocumentURI {
const workspace = getWorkspace()
const domain = getClient().getHierarchy().getDomain(attr.attr.attributeOf)
return mongodbDocumentUri(workspace, domain, docId, attr.key)
export function formatPlatformDocumentId (
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): PlatformDocumentId {
const objectDomain = getClient().getHierarchy().getDomain(objectClass)
return origFormatPlatformDocumentId(objectDomain, objectClass, objectId, objectAttr)
}

15
plugins/document-resources/src/components/DocumentEditor.svelte
View File

@ -16,16 +16,14 @@
-->
<script lang="ts">
import { Extensions, FocusPosition } from '@tiptap/core'
import document, { Document } from '@hcengineering/document'
import { Document } from '@hcengineering/document'
import {
CollaboratorEditor,
HeadingsExtension,
ImageOptions,
SvelteNodeViewRenderer,
TodoItemExtension,
TodoListExtension,
collaborativeDocumentId,
platformDocumentId
TodoListExtension
} from '@hcengineering/text-editor'
import { createEventDispatcher } from 'svelte'
@ -78,14 +76,13 @@
}
})
]
$: documentId = collaborativeDocumentId(object.content)
$: targetContentId = platformDocumentId(document.class.Document, object._id, 'content')
</script>
<CollaboratorEditor
{documentId}
{targetContentId}
collaborativeDoc={object.content}
objectClass={object._class}
objectId={object._id}
objectAttr="content"
{focusIndex}
{readonly}
{attachFile}

4
server-plugins/collaboration/src/fulltext.ts
View File

@ -24,7 +24,7 @@ import core, {
MeasureContext,
Ref,
WorkspaceId,
parseCollaborativeDoc
collaborativeDocParse
} from '@hcengineering/core'
import {
ContentTextAdapter,
@ -105,7 +105,7 @@ export class CollaborativeContentRetrievalStage implements FullTextPipelineStage
if (val.type._class === core.class.TypeCollaborativeDoc) {
const collaborativeDoc = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as CollaborativeDoc
if (collaborativeDoc !== undefined && collaborativeDoc !== '') {
const { documentId } = parseCollaborativeDoc(collaborativeDoc)
const { documentId } = collaborativeDocParse(collaborativeDoc)
const docInfo: Blob | undefined = await this.storageAdapter?.stat(this.metrics, this.workspace, documentId)

41
server/collaboration/src/uri.ts
View File

@ -1,41 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Class, CollaborativeDoc, Doc, Domain, Ref, parseCollaborativeDoc } from '@hcengineering/core'
export type DocumentURI = string & { __documentUri: true }
export function collaborativeDocumentUri (workspaceUrl: string, docId: CollaborativeDoc): DocumentURI {
const { documentId, versionId } = parseCollaborativeDoc(docId)
return `minio://${workspaceUrl}/${documentId}/${versionId}` as DocumentURI
}
export function platformDocumentUri (
workspaceUrl: string,
objectClass: Ref<Class<Doc>>,
objectId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `platform://${workspaceUrl}/${objectClass}/${objectId}/${objectAttr}` as DocumentURI
}
export function mongodbDocumentUri (
workspaceUrl: string,
domain: Domain,
docId: Ref<Doc>,
objectAttr: string
): DocumentURI {
return `mongodb://${workspaceUrl}/${domain}/${docId}/${objectAttr}` as DocumentURI
}

6
server/collaboration/src/utils/__tests__/collaborative-doc.test.ts
View File

@ -13,7 +13,7 @@
// limitations under the License.
//
import { formatCollaborativeDoc } from '@hcengineering/core'
import { collaborativeDocFormat } from '@hcengineering/core'
import { collaborativeHistoryDocId, isEditableDoc, isEditableDocVersion } from '../collaborative-doc'
describe('collaborative-doc', () => {
@ -29,7 +29,7 @@ describe('collaborative-doc', () => {
describe('isEditableDoc', () => {
it('returns true for HEAD version', async () => {
const doc = formatCollaborativeDoc({
const doc = collaborativeDocFormat({
documentId: 'example',
versionId: 'HEAD',
lastVersionId: '0'
@ -38,7 +38,7 @@ describe('collaborative-doc', () => {
})
it('returns false for other versions', async () => {
const doc = formatCollaborativeDoc({
const doc = collaborativeDocFormat({
documentId: 'example',
versionId: 'main',
lastVersionId: '0'

93
server/collaboration/src/utils/collaborative-doc.ts
View File

@ -19,7 +19,8 @@ import {
CollaborativeDocVersionHead,
MeasureContext,
WorkspaceId,
parseCollaborativeDoc
collaborativeDocParse,
collaborativeDocUnchain
} from '@hcengineering/core'
import { Doc as YDoc } from 'yjs'
@ -35,6 +36,41 @@ export function collaborativeHistoryDocId (id: string): string {
return id.endsWith(suffix) ? id : id + suffix
}
async function loadCollaborativeDocVersion (
ctx: MeasureContext,
storageAdapter: StorageAdapter,
workspace: WorkspaceId,
documentId: string,
versionId: string
): Promise<YDoc | undefined> {
const yContent = await ctx.with('yDocFromStorage', { type: 'content' }, async (ctx) => {
return await yDocFromStorage(ctx, storageAdapter, workspace, documentId, new YDoc({ gc: false }))
})
// the document does not exist
if (yContent === undefined) {
return undefined
}
if (versionId === 'HEAD') {
return yContent
}
const historyDocumentId = collaborativeHistoryDocId(documentId)
const yHistory = await ctx.with('yDocFromStorage', { type: 'history' }, async (ctx) => {
return await yDocFromStorage(ctx, storageAdapter, workspace, historyDocumentId, new YDoc())
})
// the history document does not exist
if (yHistory === undefined) {
return undefined
}
return await ctx.with('restoreYdocSnapshot', {}, () => {
return restoreYdocSnapshot(yContent, yHistory, versionId)
})
}
/** @public */
export async function loadCollaborativeDoc (
storageAdapter: StorageAdapter,
@ -42,35 +78,20 @@ export async function loadCollaborativeDoc (
collaborativeDoc: CollaborativeDoc,
ctx: MeasureContext
): Promise<YDoc | undefined> {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
const historyDocumentId = collaborativeHistoryDocId(documentId)
const sources = collaborativeDocUnchain(collaborativeDoc)
return await ctx.with('loadCollaborativeDoc', { type: 'content' }, async (ctx) => {
const yContent = await ctx.with('yDocFromMinio', { type: 'content' }, async () => {
return await yDocFromStorage(ctx, storageAdapter, workspace, documentId, new YDoc({ gc: false }))
})
for (const source of sources) {
const { documentId, versionId } = collaborativeDocParse(source)
// the document does not exist
if (yContent === undefined) {
return undefined
await ctx.info('loading collaborative document', { source })
const ydoc = await loadCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId)
if (ydoc !== undefined) {
return ydoc
}
}
if (versionId === 'HEAD') {
return yContent
}
const yHistory = await ctx.with('yDocFromMinio', { type: 'history' }, async () => {
return await yDocFromStorage(ctx, storageAdapter, workspace, historyDocumentId, new YDoc())
})
// the history document does not exist
if (yHistory === undefined) {
return undefined
}
return await ctx.with('restoreYdocSnapshot', {}, () => {
return restoreYdocSnapshot(yContent, yHistory, versionId)
})
return undefined
})
}
@ -82,7 +103,7 @@ export async function saveCollaborativeDoc (
ydoc: YDoc,
ctx: MeasureContext
): Promise<void> {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
const { documentId, versionId } = collaborativeDocParse(collaborativeDoc)
await saveCollaborativeDocVersion(storageAdapter, workspace, documentId, versionId, ydoc, ctx)
}
@ -97,7 +118,7 @@ export async function saveCollaborativeDocVersion (
): Promise<void> {
await ctx.with('saveCollaborativeDoc', {}, async (ctx) => {
if (versionId === 'HEAD') {
await ctx.with('yDocToMinio', {}, async () => {
await ctx.with('yDocToStorage', {}, async () => {
await yDocToStorage(ctx, storageAdapter, workspace, documentId, ydoc)
})
} else {
@ -116,7 +137,7 @@ export async function removeCollaborativeDoc (
await ctx.with('removeollaborativeDoc', {}, async (ctx) => {
const toRemove: string[] = []
for (const collaborativeDoc of collaborativeDocs) {
const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
const { documentId, versionId } = collaborativeDocParse(collaborativeDoc)
if (versionId === CollaborativeDocVersionHead) {
toRemove.push(documentId, collaborativeHistoryDocId(documentId))
} else {
@ -139,8 +160,8 @@ export async function copyCollaborativeDoc (
target: CollaborativeDoc,
ctx: MeasureContext
): Promise<YDoc | undefined> {
const { documentId: sourceDocumentId } = parseCollaborativeDoc(source)
const { documentId: targetDocumentId, versionId: targetVersionId } = parseCollaborativeDoc(target)
const { documentId: sourceDocumentId } = collaborativeDocParse(source)
const { documentId: targetDocumentId, versionId: targetVersionId } = collaborativeDocParse(target)
if (sourceDocumentId === targetDocumentId) {
// no need to copy into itself
@ -175,12 +196,12 @@ export async function takeCollaborativeDocSnapshot (
version: YDocVersion,
ctx: MeasureContext
): Promise<void> {
const { documentId } = parseCollaborativeDoc(collaborativeDoc)
const { documentId } = collaborativeDocParse(collaborativeDoc)
const historyDocumentId = collaborativeHistoryDocId(documentId)
await ctx.with('takeCollaborativeDocSnapshot', {}, async (ctx) => {
const yHistory =
(await ctx.with('yDocFromMinio', { type: 'history' }, async () => {
(await ctx.with('yDocFromStorage', { type: 'history' }, async (ctx) => {
return await yDocFromStorage(ctx, storageAdapter, workspace, historyDocumentId, new YDoc({ gc: false }))
})) ?? new YDoc()
@ -188,7 +209,7 @@ export async function takeCollaborativeDocSnapshot (
createYdocSnapshot(ydoc, yHistory, version)
})
await ctx.with('yDocToMinio', { type: 'history' }, async () => {
await ctx.with('yDocToStorage', { type: 'history' }, async (ctx) => {
await yDocToStorage(ctx, storageAdapter, workspace, historyDocumentId, yHistory)
})
})
@ -196,8 +217,8 @@ export async function takeCollaborativeDocSnapshot (
/** @public */
export function isEditableDoc (id: CollaborativeDoc): boolean {
const data = parseCollaborativeDoc(id)
return isEditableDocVersion(data.versionId)
const { versionId } = collaborativeDocParse(id)
return isEditableDocVersion(versionId)
}
/** @public */

15
server/collaborator/src/context.ts
View File

@ -13,6 +13,7 @@
// limitations under the License.
//
import { type DocumentId, type PlatformDocumentId } from '@hcengineering/collaborator-client'
import { WorkspaceId, generateId } from '@hcengineering/core'
import { decodeToken } from '@hcengineering/server-token'
import { onAuthenticatePayload } from '@hocuspocus/server'
@ -22,8 +23,9 @@ export interface Context {
connectionId: string
workspaceId: WorkspaceId
clientFactory: ClientFactory
initialContentId: string
targetContentId: string
initialContentId?: DocumentId
platformDocumentId?: PlatformDocumentId
}
interface WithContext {
@ -39,14 +41,15 @@ export function buildContext (data: onAuthenticatePayload, controller: Controlle
const connectionId = context.connectionId ?? generateId()
const decodedToken = decodeToken(data.token)
const initialContentId = data.requestParameters.get('initialContentId') as string
const targetContentId = data.requestParameters.get('targetContentId') as string
const initialContentId = (data.requestParameters.get('initialContentId') as DocumentId) ?? undefined
const platformDocumentId = (data.requestParameters.get('platformDocumentId') as PlatformDocumentId) ?? undefined
return {
connectionId,
workspaceId: decodedToken.workspace,
clientFactory: getClientFactory(decodedToken, controller),
initialContentId: initialContentId ?? '',
targetContentId: targetContentId ?? ''
initialContentId,
platformDocumentId
}
}

14
server/collaborator/src/extensions/authentication.ts
View File

@ -13,14 +13,14 @@
// limitations under the License.
//
import { isReadonlyDocVersion } from '@hcengineering/collaboration'
import { DocumentId, parseDocumentId } from '@hcengineering/collaborator-client'
import { isReadonlyDoc } from '@hcengineering/collaboration'
import { MeasureContext } from '@hcengineering/core'
import { Extension, onAuthenticatePayload } from '@hocuspocus/server'
import { getWorkspaceInfo } from '../account'
import { Context, buildContext } from '../context'
import { Controller } from '../platform'
import { parseDocumentId } from '../storage/minio'
export interface AuthenticationConfiguration {
ctx: MeasureContext
@ -37,21 +37,17 @@ export class AuthenticationExtension implements Extension {
async onAuthenticate (data: onAuthenticatePayload): Promise<Context> {
this.configuration.ctx.measure('authenticate', 1)
let documentName = data.documentName
if (documentName.includes('://')) {
documentName = documentName.split('://', 2)[1]
}
const { workspaceUrl, versionId } = parseDocumentId(documentName)
const { workspaceUrl, collaborativeDoc } = parseDocumentId(data.documentName as DocumentId)
// verify workspace can be accessed with the token
const workspaceInfo = await getWorkspaceInfo(data.token)
// verify workspace url in the document matches the token
if (workspaceInfo.workspace !== workspaceUrl) {
throw new Error('documentName must include workspace')
}
data.connection.readOnly = isReadonlyDocVersion(versionId)
data.connection.readOnly = isReadonlyDoc(collaborativeDoc)
return buildContext(data, this.configuration.controller)
}

101
server/collaborator/src/extensions/storage.ts
View File

@ -13,7 +13,7 @@
// limitations under the License.
//
import { YDocVersion } from '@hcengineering/collaboration'
import { DocumentId } from '@hcengineering/collaborator-client'
import { MeasureContext } from '@hcengineering/core'
import {
Document,
@ -27,11 +27,11 @@ import {
} from '@hocuspocus/server'
import { Doc as YDoc } from 'yjs'
import { Context, withContext } from '../context'
import { StorageAdapter } from '../storage/adapter'
import { CollabStorageAdapter } from '../storage/adapter'
export interface StorageConfiguration {
ctx: MeasureContext
adapter: StorageAdapter
adapter: CollabStorageAdapter
}
export class StorageExtension implements Extension {
@ -49,32 +49,32 @@ export class StorageExtension implements Extension {
}
async onLoadDocument ({ context, documentName }: withContext<onLoadDocumentPayload>): Promise<any> {
await this.configuration.ctx.info('load document', { documentId: documentName })
await this.configuration.ctx.info('load document', { documentName })
return await this.configuration.ctx.with('load-document', {}, async () => {
return await this.loadDocument(documentName, context)
return await this.loadDocument(documentName as DocumentId, context)
})
}
async onStoreDocument ({ context, documentName, document }: withContext<onStoreDocumentPayload>): Promise<void> {
const { ctx } = this.configuration
await ctx.info('store document', { documentId: documentName })
await ctx.info('store document', { documentName })
const collaborators = this.collaborators.get(documentName)
if (collaborators === undefined || collaborators.size === 0) {
await ctx.info('no changes for document', { documentId: documentName })
await ctx.info('no changes for document', { documentName })
return
}
this.collaborators.delete(documentName)
await ctx.with('store-document', {}, async () => {
await this.storeDocument(documentName, document, context)
await this.storeDocument(documentName as DocumentId, document, context)
})
}
async onConnect ({ context, documentName, instance }: withContext<onConnectPayload>): Promise<any> {
const connections = instance.documents.get(documentName)?.getConnectionsCount() ?? 0
const params = { documentId: documentName, connectionId: context.connectionId, connections }
const params = { documentName, connectionId: context.connectionId, connections }
await this.configuration.ctx.info('connect to document', params)
}
@ -82,98 +82,49 @@ export class StorageExtension implements Extension {
const { ctx } = this.configuration
const { connectionId } = context
const params = { documentId: documentName, connectionId, connections: document.getConnectionsCount() }
const params = { documentName, connectionId, connections: document.getConnectionsCount() }
await ctx.info('disconnect from document', params)
const collaborators = this.collaborators.get(documentName)
if (collaborators === undefined || !collaborators.has(connectionId)) {
await ctx.info('no changes for document', { documentId: documentName })
await ctx.info('no changes for document', { documentName })
return
}
this.collaborators.delete(documentName)
await ctx.with('store-document', {}, async () => {
await this.storeDocument(documentName, document, context)
await this.storeDocument(documentName as DocumentId, document, context)
})
}
async afterUnloadDocument ({ documentName }: afterUnloadDocumentPayload): Promise<any> {
await this.configuration.ctx.info('unload document', { documentId: documentName })
await this.configuration.ctx.info('unload document', { documentName })
this.collaborators.delete(documentName)
}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { adapter, ctx } = this.configuration
async loadDocument (documentId: DocumentId, context: Context): Promise<YDoc | undefined> {
const { ctx, adapter } = this.configuration
try {
await ctx.info('load document content', { documentId })
const ydoc = await adapter.loadDocument(documentId, context)
if (ydoc !== undefined) {
return ydoc
}
return await ctx.with('load-document', {}, async (ctx) => {
return await adapter.loadDocument(ctx, documentId, context)
})
} catch (err) {
await ctx.error('failed to load document', { documentId, error: err })
}
const { initialContentId } = context
if (initialContentId !== undefined && initialContentId.length > 0) {
await ctx.info('load document initial content', { documentId, initialContentId })
try {
const ydoc = await adapter.loadDocument(initialContentId, context)
// if document was loaded from the initial content we need to save
// it to ensure the next time we load ydoc document
if (ydoc !== undefined) {
await adapter.saveDocument(documentId, ydoc, undefined, context)
}
return ydoc
} catch (err) {
await ctx.error('failed to load document initial content', {
documentId,
initialContentId,
error: err
})
}
await ctx.error('failed to load document content', { documentId, error: err })
return undefined
}
}
async storeDocument (documentId: string, document: Document, context: Context): Promise<void> {
const { adapter, ctx } = this.configuration
let snapshot: YDocVersion | undefined
try {
await ctx.info('take document snapshot', { documentId })
snapshot = await ctx.with('take-snapshot', {}, async () => {
return await adapter.takeSnapshot(documentId, document, context)
})
} catch (err) {
await ctx.error('failed to take document snapshot', { documentId, error: err })
}
async storeDocument (documentId: DocumentId, document: Document, context: Context): Promise<void> {
const { ctx, adapter } = this.configuration
try {
await ctx.info('save document content', { documentId })
await ctx.with('save-document', {}, async () => {
await adapter.saveDocument(documentId, document, snapshot, context)
await ctx.with('save-document', {}, async (ctx) => {
await adapter.saveDocument(ctx, documentId, document, context)
})
} catch (err) {
await ctx.error('failed to save document', { documentId, error: err })
}
const { targetContentId } = context
if (targetContentId !== undefined && targetContentId.length > 0) {
await ctx.info('store document target content', { documentId, targetContentId })
try {
await ctx.with('save-target-document', {}, async () => {
await adapter.saveDocument(targetContentId, document, snapshot, context)
})
} catch (err) {
await ctx.error('failed to save document target content', {
documentId,
targetContentId,
error: err
})
}
await ctx.error('failed to save document content', { documentId, error: err })
return undefined
}
}
}

13
server/collaborator/src/rpc/methods/removeDocument.ts
View File

@ -14,8 +14,12 @@
//
import { collaborativeHistoryDocId } from '@hcengineering/collaboration'
import { type RemoveDocumentRequest, type RemoveDocumentResponse } from '@hcengineering/collaborator-client'
import { MeasureContext, parseCollaborativeDoc } from '@hcengineering/core'
import {
parseDocumentId,
type RemoveDocumentRequest,
type RemoveDocumentResponse
} from '@hcengineering/collaborator-client'
import { MeasureContext, collaborativeDocParse } from '@hcengineering/core'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
@ -25,7 +29,7 @@ export async function removeDocument (
payload: RemoveDocumentRequest,
params: RpcMethodParams
): Promise<RemoveDocumentResponse> {
const { documentId, collaborativeDoc } = payload
const { documentId } = payload
const { hocuspocus, minio } = params
const { workspaceId } = context
@ -35,7 +39,8 @@ export async function removeDocument (
hocuspocus.unloadDocument(document)
}
const { documentId: minioDocumentId } = parseCollaborativeDoc(collaborativeDoc)
const { collaborativeDoc } = parseDocumentId(documentId)
const { documentId: minioDocumentId } = collaborativeDocParse(collaborativeDoc)
const historyDocumentId = collaborativeHistoryDocId(minioDocumentId)
try {

13
server/collaborator/src/rpc/methods/takeSnapshot.ts
View File

@ -20,8 +20,12 @@ import {
yDocFromStorage,
yDocToStorage
} from '@hcengineering/collaboration'
import { type TakeSnapshotRequest, type TakeSnapshotResponse } from '@hcengineering/collaborator-client'
import { CollaborativeDocVersionHead, MeasureContext, generateId, parseCollaborativeDoc } from '@hcengineering/core'
import {
parseDocumentId,
type TakeSnapshotRequest,
type TakeSnapshotResponse
} from '@hcengineering/collaborator-client'
import { CollaborativeDocVersionHead, MeasureContext, collaborativeDocParse, generateId } from '@hcengineering/core'
import { Doc as YDoc } from 'yjs'
import { Context } from '../../context'
import { RpcMethodParams } from '../rpc'
@ -32,7 +36,7 @@ export async function takeSnapshot (
payload: TakeSnapshotRequest,
params: RpcMethodParams
): Promise<TakeSnapshotResponse> {
const { collaborativeDoc, documentId, snapshotName, createdBy } = payload
const { documentId, snapshotName, createdBy } = payload
const { hocuspocus, minio } = params
const { workspaceId } = context
@ -43,7 +47,8 @@ export async function takeSnapshot (
createdOn: Date.now()
}
const { documentId: minioDocumentId, versionId } = parseCollaborativeDoc(collaborativeDoc)
const { collaborativeDoc } = parseDocumentId(documentId)
const { documentId: minioDocumentId, versionId } = collaborativeDocParse(collaborativeDoc)
if (versionId !== CollaborativeDocVersionHead) {
throw new Error('invalid document version')
}

19
server/collaborator/src/server.ts
View File

@ -32,10 +32,7 @@ import { AuthenticationExtension } from './extensions/authentication'
import { StorageExtension } from './extensions/storage'
import { Controller, getClientFactory } from './platform'
import { RpcErrorResponse, RpcRequest, RpcResponse, methods } from './rpc'
import { MinioStorageAdapter } from './storage/minio'
import { MongodbStorageAdapter } from './storage/mongodb'
import { PlatformStorageAdapter } from './storage/platform'
import { RouterStorageAdapter } from './storage/router'
import { HtmlTransformer } from './transformers/html'
/**
@ -83,7 +80,6 @@ export async function start (
]
const extensionsCtx = ctx.newChild('extensions', {})
const storageCtx = ctx.newChild('storage', {})
const controller = new Controller()
@ -131,14 +127,7 @@ export async function start (
}),
new StorageExtension({
ctx: extensionsCtx.newChild('storage', {}),
adapter: new RouterStorageAdapter(
{
minio: new MinioStorageAdapter(storageCtx.newChild('minio', {}), minio),
mongodb: new MongodbStorageAdapter(storageCtx.newChild('mongodb', {}), mongo, transformer),
platform: new PlatformStorageAdapter(storageCtx.newChild('platform', {}), transformer)
},
'minio'
)
adapter: new PlatformStorageAdapter({ minio }, mongo, transformer)
})
],
@ -149,13 +138,11 @@ export async function start (
const rpcCtx = ctx.newChild('rpc', {})
const getContext = (token: Token, initialContentId?: string): Context => {
const getContext = (token: Token): Context => {
return {
connectionId: generateId(),
workspaceId: token.workspace,
clientFactory: getClientFactory(token, controller),
initialContentId: initialContentId ?? '',
targetContentId: ''
clientFactory: getClientFactory(token, controller)
}
}

17
server/collaborator/src/storage/adapter.ts
View File

@ -13,19 +13,12 @@
// limitations under the License.
//
import { YDocVersion } from '@hcengineering/collaboration'
import { DocumentId } from '@hcengineering/collaborator-client'
import { MeasureContext } from '@hcengineering/core'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
export interface StorageAdapter {
loadDocument: (documentId: string, context: Context) => Promise<YDoc | undefined>
saveDocument: (
documentId: string,
document: YDoc,
snapshot: YDocVersion | undefined,
context: Context
) => Promise<void>
takeSnapshot: (documentId: string, document: YDoc, context: Context) => Promise<YDocVersion | undefined>
export interface CollabStorageAdapter {
loadDocument: (ctx: MeasureContext, documentId: DocumentId, context: Context) => Promise<YDoc | undefined>
saveDocument: (ctx: MeasureContext, documentId: DocumentId, document: YDoc, context: Context) => Promise<void>
}
export type StorageAdapters = Record<string, StorageAdapter>

122
server/collaborator/src/storage/minio.ts
View File

@ -1,122 +0,0 @@
//
// Copyright © 2023, 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import {
YDocVersion,
loadCollaborativeDoc,
saveCollaborativeDocVersion,
takeCollaborativeDocSnapshot
} from '@hcengineering/collaboration'
import {
CollaborativeDocVersion,
CollaborativeDocVersionHead,
MeasureContext,
formatCollaborativeDocVersion
} from '@hcengineering/core'
import { MinioService } from '@hcengineering/minio'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { StorageAdapter } from './adapter'
export interface MinioDocumentId {
workspaceUrl: string
minioDocumentId: string
versionId: CollaborativeDocVersion
}
export function parseDocumentId (documentId: string): MinioDocumentId {
const [workspaceUrl, minioDocumentId, versionId] = documentId.split('/')
return {
workspaceUrl: workspaceUrl ?? '',
minioDocumentId: minioDocumentId ?? '',
versionId: versionId ?? CollaborativeDocVersionHead
}
}
function isValidDocumentId (documentId: Omit<MinioDocumentId, 'workspaceUrl'>): boolean {
return documentId.minioDocumentId !== '' && documentId.versionId !== ''
}
export class MinioStorageAdapter implements StorageAdapter {
constructor (
private readonly ctx: MeasureContext,
private readonly minio: MinioService
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { workspaceId } = context
const { minioDocumentId, versionId } = parseDocumentId(documentId)
if (!isValidDocumentId({ minioDocumentId, versionId })) {
await this.ctx.error('malformed document id', { documentId })
return undefined
}
return await this.ctx.with('load-document', {}, async (ctx) => {
try {
const collaborativeDoc = formatCollaborativeDocVersion({ documentId: minioDocumentId, versionId })
return await loadCollaborativeDoc(this.minio, workspaceId, collaborativeDoc, ctx)
} catch {
return undefined
}
})
}
async saveDocument (
documentId: string,
document: YDoc,
snapshot: YDocVersion | undefined,
context: Context
): Promise<void> {
const { workspaceId } = context
const { minioDocumentId, versionId } = parseDocumentId(documentId)
if (!isValidDocumentId({ minioDocumentId, versionId })) {
await this.ctx.error('malformed document id', { documentId })
return undefined
}
await this.ctx.with('save-document', {}, async (ctx) => {
await saveCollaborativeDocVersion(this.minio, workspaceId, minioDocumentId, versionId, document, ctx)
})
}
async takeSnapshot (documentId: string, document: YDoc, context: Context): Promise<YDocVersion | undefined> {
const { clientFactory, workspaceId } = context
const client = await clientFactory({ derived: false })
const timestamp = Date.now()
const yDocVersion: YDocVersion = {
versionId: `${timestamp}`,
name: 'Automatic snapshot',
createdBy: client.user,
createdOn: timestamp
}
const { minioDocumentId, versionId } = parseDocumentId(documentId)
const collaborativeDoc = formatCollaborativeDocVersion({ documentId: minioDocumentId, versionId })
await this.ctx.with('take-snapshot', {}, async (ctx) => {
await takeCollaborativeDocSnapshot(this.minio, workspaceId, collaborativeDoc, document, yDocVersion, ctx)
})
return yDocVersion
}
}

91
server/collaborator/src/storage/mongodb.ts
View File

@ -1,91 +0,0 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { YDocVersion } from '@hcengineering/collaboration'
import { Doc, MeasureContext, Ref, toWorkspaceString } from '@hcengineering/core'
import { Transformer } from '@hocuspocus/transformer'
import { MongoClient } from 'mongodb'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { StorageAdapter } from './adapter'
interface MongodbDocumentId {
workspaceUrl: string
objectDomain: string
objectId: string
objectAttr: string
}
function parseDocumentId (documentId: string): MongodbDocumentId {
const [workspace, objectDomain, objectId, objectAttr] = documentId.split('/')
return {
workspaceUrl: workspace ?? '',
objectId: objectId ?? '',
objectDomain: objectDomain ?? '',
objectAttr: objectAttr ?? ''
}
}
function isValidDocumentId (documentId: Omit<MongodbDocumentId, 'workspaceUrl'>, context: Context): boolean {
return documentId.objectDomain !== '' && documentId.objectId !== '' && documentId.objectAttr !== ''
}
export class MongodbStorageAdapter implements StorageAdapter {
constructor (
private readonly ctx: MeasureContext,
private readonly mongodb: MongoClient,
private readonly transformer: Transformer
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { objectId, objectDomain, objectAttr } = parseDocumentId(documentId)
if (!isValidDocumentId({ objectId, objectDomain, objectAttr }, context)) {
await this.ctx.error('malformed document id', { documentId })
return undefined
}
return await this.ctx.with('load-document', {}, async (ctx) => {
const doc = await ctx.with('query', {}, async () => {
const db = this.mongodb.db(toWorkspaceString(context.workspaceId))
return await db
.collection<Doc>(objectDomain)
.findOne({ _id: objectId as Ref<Doc> }, { projection: { [objectAttr]: 1 } })
})
const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : ''
return await ctx.with('transform', {}, () => {
return this.transformer.toYdoc(content, objectAttr)
})
})
}
async saveDocument (
documentId: string,
_document: YDoc,
snapshot: YDocVersion | undefined,
_context: Context
): Promise<void> {
await this.ctx.error('saving documents into mongodb not supported', { documentId })
}
async takeSnapshot (documentId: string, document: YDoc, context: Context): Promise<YDocVersion | undefined> {
await this.ctx.error('taking snapshotsin mongodb not supported', { documentId })
return undefined
}
}

317
server/collaborator/src/storage/platform.ts
View File

@ -1,5 +1,5 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
// Copyright © 2023, 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
@ -13,103 +13,272 @@
// limitations under the License.
//
import { YDocVersion } from '@hcengineering/collaboration'
import core, { Class, CollaborativeDoc, Doc, MeasureContext, Ref, updateCollaborativeDoc } from '@hcengineering/core'
import {
YDocVersion,
loadCollaborativeDoc,
saveCollaborativeDoc,
takeCollaborativeDocSnapshot
} from '@hcengineering/collaboration'
import {
DocumentId,
PlatformDocumentId,
parseDocumentId,
parsePlatformDocumentId
} from '@hcengineering/collaborator-client'
import core, {
CollaborativeDoc,
Doc,
MeasureContext,
collaborativeDocWithLastVersion,
toWorkspaceString
} from '@hcengineering/core'
import { StorageAdapter } from '@hcengineering/server-core'
import { Transformer } from '@hocuspocus/transformer'
import { MongoClient } from 'mongodb'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { StorageAdapter } from './adapter'
import { CollabStorageAdapter } from './adapter'
interface PlatformDocumentId {
workspaceUrl: string
objectClass: Ref<Class<Doc>>
objectId: Ref<Doc>
objectAttr: string
}
export type StorageAdapters = Record<string, StorageAdapter>
function parseDocumentId (documentId: string): PlatformDocumentId {
const [workspaceUrl, objectClass, objectId, objectAttr] = documentId.split('/')
return {
workspaceUrl: workspaceUrl ?? '',
objectClass: (objectClass ?? '') as Ref<Class<Doc>>,
objectId: (objectId ?? '') as Ref<Doc>,
objectAttr: objectAttr ?? ''
}
}
function isValidDocumentId (documentId: Omit<PlatformDocumentId, 'workspaceUrl'>, context: Context): boolean {
return documentId.objectClass !== '' && documentId.objectId !== '' && documentId.objectAttr !== ''
}
export class PlatformStorageAdapter implements StorageAdapter {
export class PlatformStorageAdapter implements CollabStorageAdapter {
constructor (
private readonly ctx: MeasureContext,
private readonly adapters: StorageAdapters,
private readonly mongodb: MongoClient,
private readonly transformer: Transformer
) {}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
await this.ctx.error('loading documents from the platform not supported', { documentId })
return undefined
async loadDocument (ctx: MeasureContext, documentId: DocumentId, context: Context): Promise<YDoc | undefined> {
try {
// try to load document content
try {
await ctx.info('load document content', { documentId })
const ydoc = await this.loadDocumentFromStorage(ctx, documentId, context)
if (ydoc !== undefined) {
return ydoc
}
} catch (err) {
await ctx.error('failed to load document content', { documentId, error: err })
}
// then try to load from inital content
const { initialContentId } = context
if (initialContentId !== undefined && initialContentId.length > 0) {
try {
await ctx.info('load document initial content', { documentId, initialContentId })
const ydoc = await this.loadDocumentFromStorage(ctx, initialContentId, context)
// if document was loaded from the initial content or storage we need to save
// it to ensure the next time we load it from the ydoc document
if (ydoc !== undefined) {
await ctx.info('save document content', { documentId, initialContentId })
await this.saveDocumentToStorage(ctx, documentId, ydoc, context)
return ydoc
}
} catch (err) {
await ctx.error('failed to load initial document content', { documentId, initialContentId, error: err })
}
}
// finally try to load from the platform
const { platformDocumentId } = context
if (platformDocumentId !== undefined) {
await ctx.info('load document platform content', { documentId, platformDocumentId })
const ydoc = await ctx.with('load-document', { storage: 'platform' }, async (ctx) => {
try {
return await this.loadDocumentFromPlatform(ctx, platformDocumentId, context)
} catch (err) {
await ctx.error('failed to load platform document', { documentId, platformDocumentId, error: err })
}
})
// if document was loaded from the initial content or storage we need to save
// it to ensure the next time we load it from the ydoc document
if (ydoc !== undefined) {
await ctx.info('save document content', { documentId, platformDocumentId })
await this.saveDocumentToStorage(ctx, documentId, ydoc, context)
return ydoc
}
}
// nothing found
return undefined
} catch (err) {
await ctx.error('failed to load document', { documentId, error: err })
}
}
async saveDocument (
documentId: string,
document: YDoc,
snapshot: YDocVersion | undefined,
context: Context
): Promise<void> {
const { clientFactory } = context
const { objectId, objectClass, objectAttr } = parseDocumentId(documentId)
if (!isValidDocumentId({ objectId, objectClass, objectAttr }, context)) {
await this.ctx.error('malformed document id', { documentId })
return undefined
async saveDocument (ctx: MeasureContext, documentId: DocumentId, document: YDoc, context: Context): Promise<void> {
let snapshot: YDocVersion | undefined
try {
await ctx.info('take document snapshot', { documentId })
snapshot = await this.takeSnapshot(ctx, documentId, document, context)
} catch (err) {
await ctx.error('failed to take document snapshot', { documentId, error: err })
}
await this.ctx.with('save-document', {}, async (ctx) => {
const client = await ctx.with('connect', {}, async () => {
return await clientFactory({ derived: false })
try {
await ctx.info('save document content', { documentId })
await this.saveDocumentToStorage(ctx, documentId, document, context)
} catch (err) {
await ctx.error('failed to save document', { documentId, error: err })
}
const { platformDocumentId } = context
if (platformDocumentId !== undefined) {
await ctx.info('save document content to platform', { documentId, platformDocumentId })
await ctx.with('save-document', { storage: 'platform' }, async (ctx) => {
await this.saveDocumentToPlatform(ctx, documentId, platformDocumentId, document, snapshot, context)
})
}
}
const attribute = client.getHierarchy().findAttribute(objectClass, objectAttr)
if (attribute === undefined) {
await this.ctx.info('attribute not found', { documentId, objectClass, objectAttr })
return
}
getStorageAdapter (storage: string): StorageAdapter {
const adapter = this.adapters[storage]
const current = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId })
})
if (adapter === undefined) {
throw new Error(`unknown storage adapter ${storage}`)
}
if (current === undefined) {
return
}
return adapter
}
const hierarchy = client.getHierarchy()
if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc
const newCollaborativeDoc =
snapshot !== undefined ? updateCollaborativeDoc(collaborativeDoc, snapshot.versionId) : collaborativeDoc
async loadDocumentFromStorage (
ctx: MeasureContext,
documentId: DocumentId,
context: Context
): Promise<YDoc | undefined> {
const { storage, collaborativeDoc } = parseDocumentId(documentId)
const adapter = this.getStorageAdapter(storage)
await ctx.with('update', {}, async () => {
await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
})
} else if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) {
// TODO a temporary solution while we are keeping Markup in Mongo
const content = await ctx.with('transform', {}, () => {
return this.transformer.fromYdoc(document, objectAttr)
})
await ctx.with('update', {}, async () => {
await client.diffUpdate(current, { [objectAttr]: content })
})
return await ctx.with('load-document', { storage }, async (ctx) => {
try {
return await loadCollaborativeDoc(adapter, context.workspaceId, collaborativeDoc, ctx)
} catch (err) {
await ctx.error('failed to load storage document', { documentId, collaborativeDoc, error: err })
return undefined
}
})
}
async takeSnapshot (documentId: string, document: YDoc, context: Context): Promise<YDocVersion | undefined> {
await this.ctx.error('taking snapshotsin mongodb not supported', { documentId })
async saveDocumentToStorage (
ctx: MeasureContext,
documentId: DocumentId,
document: YDoc,
context: Context
): Promise<void> {
const { storage, collaborativeDoc } = parseDocumentId(documentId)
const adapter = this.getStorageAdapter(storage)
await ctx.with('save-document', {}, async (ctx) => {
await saveCollaborativeDoc(adapter, context.workspaceId, collaborativeDoc, document, ctx)
})
}
async takeSnapshot (
ctx: MeasureContext,
documentId: DocumentId,
document: YDoc,
context: Context
): Promise<YDocVersion | undefined> {
const { storage, collaborativeDoc } = parseDocumentId(documentId)
const adapter = this.getStorageAdapter(storage)
const { clientFactory, workspaceId } = context
const client = await clientFactory({ derived: false })
const timestamp = Date.now()
const yDocVersion: YDocVersion = {
versionId: `${timestamp}`,
name: 'Automatic snapshot',
createdBy: client.user,
createdOn: timestamp
}
await ctx.with('take-snapshot', {}, async (ctx) => {
await takeCollaborativeDocSnapshot(adapter, workspaceId, collaborativeDoc, document, yDocVersion, ctx)
})
return yDocVersion
}
async loadDocumentFromPlatform (
ctx: MeasureContext,
platformDocumentId: PlatformDocumentId,
context: Context
): Promise<YDoc | undefined> {
const { mongodb, transformer } = this
const { workspaceId } = context
const { objectDomain, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId)
const doc = await ctx.with('query', {}, async () => {
const db = mongodb.db(toWorkspaceString(workspaceId))
return await db.collection<Doc>(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } })
})
const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : ''
if (content.startsWith('<') && content.endsWith('>')) {
return await ctx.with('transform', {}, () => {
return transformer.toYdoc(content, objectAttr)
})
}
// the content does not seem to be an HTML document
return undefined
}
async saveDocumentToPlatform (
ctx: MeasureContext,
documentName: string,
platformDocumentId: PlatformDocumentId,
document: YDoc,
snapshot: YDocVersion | undefined,
context: Context
): Promise<void> {
const { objectClass, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId)
const { clientFactory } = context
const client = await ctx.with('connect', {}, async () => {
return await clientFactory({ derived: false })
})
const attribute = client.getHierarchy().findAttribute(objectClass, objectAttr)
if (attribute === undefined) {
await ctx.info('attribute not found', { documentName, objectClass, objectAttr })
return
}
const current = await ctx.with('query', {}, async () => {
return await client.findOne(objectClass, { _id: objectId })
})
if (current === undefined) {
return
}
const hierarchy = client.getHierarchy()
if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc
const newCollaborativeDoc =
snapshot !== undefined
? collaborativeDocWithLastVersion(collaborativeDoc, snapshot.versionId)
: collaborativeDoc
await ctx.with('update', {}, async () => {
await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
})
} else if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) {
// TODO a temporary solution while we are keeping Markup in Mongo
const content = await ctx.with('transform', {}, () => {
return this.transformer.fromYdoc(document, objectAttr)
})
await ctx.with('update', {}, async () => {
await client.diffUpdate(current, { [objectAttr]: content })
})
}
}
}

62
server/collaborator/src/storage/router.ts
View File

@ -1,62 +0,0 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { YDocVersion } from '@hcengineering/collaboration'
import { Doc as YDoc } from 'yjs'
import { Context } from '../context'
import { StorageAdapter, StorageAdapters } from './adapter'
function parseDocumentName (documentId: string): { schema: string, documentName: string } {
const [schema, documentName] = documentId.split('://', 2)
return documentName !== undefined ? { documentName, schema } : { documentName: documentId, schema: '' }
}
export class RouterStorageAdapter implements StorageAdapter {
constructor (
private readonly adapters: StorageAdapters,
private readonly defaultAdapter: string
) {}
getStorageAdapter (schema: string): StorageAdapter | undefined {
return schema in this.adapters
? this.adapters[schema]
: this.defaultAdapter !== undefined
? this.adapters[this.defaultAdapter]
: undefined
}
async loadDocument (documentId: string, context: Context): Promise<YDoc | undefined> {
const { schema, documentName } = parseDocumentName(documentId)
const adapter = this.getStorageAdapter(schema)
return await adapter?.loadDocument?.(documentName, context)
}
async saveDocument (
documentId: string,
document: YDoc,
snapshot: YDocVersion | undefined,
context: Context
): Promise<void> {
const { schema, documentName } = parseDocumentName(documentId)
const adapter = this.getStorageAdapter(schema)
await adapter?.saveDocument?.(documentName, document, snapshot, context)
}
async takeSnapshot (documentId: string, document: YDoc, context: Context): Promise<YDocVersion | undefined> {
const { schema, documentName } = parseDocumentName(documentId)
const adapter = this.getStorageAdapter(schema)
return await adapter?.takeSnapshot?.(documentName, document, context)
}
}

46
server/collaborator/src/types.ts
View File

@ -1,5 +1,5 @@
//
// Copyright © 2023 Hardcore Engineering Inc.
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
@ -13,6 +13,8 @@
// limitations under the License.
//
import type { Class, Doc, Domain, Ref } from '@hcengineering/core'
/** @public */
export interface DocumentId {
workspaceUrl: string
@ -21,41 +23,9 @@ export interface DocumentId {
}
/** @public */
export type Action = DocumentCopyAction | DocumentFieldCopyAction | DocumentContentAction
/** @public */
export interface DocumentContentAction {
action: 'document.content'
params: {
field: string
content: string
}
}
/** @public */
export interface DocumentCopyAction {
action: 'document.copy'
params: {
sourceId: string
targetId: string
}
}
/** @public */
export interface DocumentFieldCopyAction {
action: 'document.field.copy'
params: {
documentId: string
srcFieldId: string
dstFieldId: string
}
}
/** @public */
export type ActionStatus = 'completed' | 'failed'
/** @public */
export interface ActionStatusResponse {
action: Action
status: ActionStatus
export interface PlatformDocumentId {
objectDomain: Domain
objectClass: Ref<Class<Doc>>
objectId: Ref<Doc>
objectAttr: string
}