From 138ab2fc79b67df2366a23ab33fdb29d7811c02d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 27 Feb 2024 11:56:02 +0700 Subject: [PATCH] EZQMS-606 Migrate documents to new model (#4745) Signed-off-by: Alexander Onnikov --- .../components/CollaborativeTextEditor.svelte | 10 +- .../src/components/CollaboratorEditor.svelte | 8 - packages/text-editor/src/index.ts | 2 +- packages/text-editor/src/provider/tiptap.ts | 24 --- packages/text-editor/src/provider/utils.ts | 15 +- packages/text-editor/src/utils.ts | 68 ------- packages/text/src/node.ts | 17 +- .../src/utils/collaborative-doc.ts | 15 +- server/collaborator/src/extensions/action.ts | 169 ------------------ server/collaborator/src/server.ts | 5 - server/collaborator/src/storage/minio.ts | 12 +- 11 files changed, 27 insertions(+), 318 deletions(-) delete mode 100644 server/collaborator/src/extensions/action.ts diff --git a/packages/text-editor/src/components/CollaborativeTextEditor.svelte b/packages/text-editor/src/components/CollaborativeTextEditor.svelte index cf5f356e1b..000d88cb3a 100644 --- a/packages/text-editor/src/components/CollaborativeTextEditor.svelte +++ b/packages/text-editor/src/components/CollaborativeTextEditor.svelte @@ -39,7 +39,7 @@ TextFormatCategory, TextNodeAction } from '../types' - import { copyDocumentContent, copyDocumentField, getCollaborationUser } from '../utils' + import { getCollaborationUser } from '../utils' import ImageStyleToolbar from './ImageStyleToolbar.svelte' import TextEditorStyleToolbar from './TextEditorStyleToolbar.svelte' @@ -150,14 +150,6 @@ return commandHandler } - export function takeSnapshot (snapshotId: DocumentId): void { - copyDocumentContent(documentId, snapshotId, { provider: remoteProvider }, initialContentId) - } - - export function copyField (srcFieldId: string, dstFieldId: string): void { - copyDocumentField(documentId, srcFieldId, dstFieldId, { provider: remoteProvider }, initialContentId) - } - export function isEditable (): boolean { return editor?.isEditable ?? false } diff --git a/packages/text-editor/src/components/CollaboratorEditor.svelte b/packages/text-editor/src/components/CollaboratorEditor.svelte index 476cabdc10..f43c441070 100644 --- a/packages/text-editor/src/components/CollaboratorEditor.svelte +++ b/packages/text-editor/src/components/CollaboratorEditor.svelte @@ -56,14 +56,6 @@ return collaborativeEditor?.commands() } - export function takeSnapshot (snapshotId: DocumentId): void { - collaborativeEditor?.takeSnapshot(snapshotId) - } - - export function copyField (srcFieldId: string, dstFieldId: string): void { - collaborativeEditor?.copyField(srcFieldId, dstFieldId) - } - // TODO Not collaborative export function getNodeElement (uuid: string): Element | null { if (editor === undefined || uuid === '') { diff --git a/packages/text-editor/src/index.ts b/packages/text-editor/src/index.ts index 1fd8093698..443f9de125 100644 --- a/packages/text-editor/src/index.ts +++ b/packages/text-editor/src/index.ts @@ -74,7 +74,7 @@ export { type TiptapCollabProviderConfiguration, createTiptapCollaborationData } from './provider/tiptap' -export { collaborativeDocumentId, minioDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils' +export { collaborativeDocumentId, mongodbDocumentId, platformDocumentId } from './provider/utils' export { CollaborationIds } from './types' export { textEditorId } diff --git a/packages/text-editor/src/provider/tiptap.ts b/packages/text-editor/src/provider/tiptap.ts index 7f23099f89..bdcf2979d0 100644 --- a/packages/text-editor/src/provider/tiptap.ts +++ b/packages/text-editor/src/provider/tiptap.ts @@ -56,30 +56,6 @@ export class TiptapCollabProvider extends HocuspocusProvider { }) } - setContent (field: string, content: string): void { - const payload = { - action: 'document.content', - params: { field, content } - } - this.sendStateless(JSON.stringify(payload)) - } - - copyContent (sourceId: DocumentId, targetId: DocumentId): void { - const payload = { - action: 'document.copy', - params: { sourceId, targetId } - } - this.sendStateless(JSON.stringify(payload)) - } - - copyField (documentId: DocumentId, srcFieldId: string, dstFieldId: string): void { - const payload = { - action: 'document.field.copy', - params: { documentId, srcFieldId, dstFieldId } - } - this.sendStateless(JSON.stringify(payload)) - } - destroy (): void { this.configuration.websocketProvider.disconnect() super.destroy() diff --git a/packages/text-editor/src/provider/utils.ts b/packages/text-editor/src/provider/utils.ts index a20d0ac321..8b4957f8a0 100644 --- a/packages/text-editor/src/provider/utils.ts +++ b/packages/text-editor/src/provider/utils.ts @@ -13,14 +13,7 @@ // limitations under the License. // -import { - type Class, - type CollaborativeDoc, - type Doc, - type Ref, - getCollaborativeDoc, - getCollaborativeDocId -} from '@hcengineering/core' +import { type Class, type CollaborativeDoc, type Doc, type Ref } from '@hcengineering/core' import { type DocumentURI, collaborativeDocumentUri, @@ -39,12 +32,6 @@ export function collaborativeDocumentId (docId: CollaborativeDoc): DocumentURI { return collaborativeDocumentUri(workspace, docId) } -// TODO remove this when migrated QMS documents to new model -export function minioDocumentId (docId: Ref, attr?: KeyedAttribute): DocumentURI { - const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(docId, attr?.key)) - return collaborativeDocumentId(collaborativeDoc) -} - export function platformDocumentId (objectClass: Ref>, objectId: Ref, objectAttr: string): DocumentURI { const workspace = getWorkspace() return platformDocumentUri(workspace, objectClass, objectId, objectAttr) diff --git a/packages/text-editor/src/utils.ts b/packages/text-editor/src/utils.ts index 78a2c58b13..9dc00c0a0d 100644 --- a/packages/text-editor/src/utils.ts +++ b/packages/text-editor/src/utils.ts @@ -13,10 +13,8 @@ // limitations under the License. // -import { type onStatelessParameters } from '@hocuspocus/provider' import { type Attribute } from '@tiptap/core' import { get } from 'svelte/store' -import * as Y from 'yjs' import contact, { type PersonAccount, formatName, AvatarType } from '@hcengineering/contact' import { getCurrentAccount } from '@hcengineering/core' @@ -28,74 +26,8 @@ import { themeStore } from '@hcengineering/ui' -import { type DocumentId, TiptapCollabProvider } from './provider/tiptap' import { type CollaborationUser } from './types' -type ProviderData = ( - | { - provider: TiptapCollabProvider - } - | { - collaboratorURL: string - token: string - } -) & { ydoc?: Y.Doc } - -function getProvider ( - documentId: DocumentId, - providerData: ProviderData, - initialContentId?: DocumentId, - targetContentId?: DocumentId -): TiptapCollabProvider { - if (!('provider' in providerData)) { - const provider = new TiptapCollabProvider({ - url: providerData.collaboratorURL, - name: documentId, - document: providerData.ydoc ?? new Y.Doc(), - token: providerData.token, - parameters: { - initialContentId, - targetContentId - }, - onStateless (data: onStatelessParameters) { - try { - const payload = JSON.parse(data.payload) - if ('status' in payload && payload.status === 'completed') { - provider.destroy() - } - } catch (e) { - console.error('Failed to check provider operation status', e) - } - } - }) - - return provider - } else { - return providerData.provider - } -} - -export function copyDocumentField ( - documentId: DocumentId, - srcFieldId: string, - dstFieldId: string, - providerData: ProviderData, - initialContentId?: DocumentId -): void { - const provider = getProvider(documentId, providerData, initialContentId) - provider.copyField(documentId, srcFieldId, dstFieldId) -} - -export function copyDocumentContent ( - documentId: DocumentId, - snapshotId: DocumentId, - providerData: ProviderData, - initialContentId?: DocumentId -): void { - const provider = getProvider(documentId, providerData, initialContentId) - provider.copyContent(documentId, snapshotId) -} - export function getDataAttribute ( name: string, options?: Omit diff --git a/packages/text/src/node.ts b/packages/text/src/node.ts index 75fa1c47d5..f33e563df9 100644 --- a/packages/text/src/node.ts +++ b/packages/text/src/node.ts @@ -24,13 +24,22 @@ import { Doc, applyUpdate } from 'yjs' * @public */ export function yDocContentToNode (extensions: Extensions, content: ArrayBuffer, field?: string): Node { + const ydoc = new Doc() + const uint8arr = new Uint8Array(content) + applyUpdate(ydoc, uint8arr) + + return yDocToNode(extensions, ydoc, field) +} + +/** + * Get ProseMirror node from Y.Doc + * + * @public + */ +export function yDocToNode (extensions: Extensions, ydoc: Doc, field?: string): Node { const schema = getSchema(extensions) try { - const ydoc = new Doc() - const uint8arr = new Uint8Array(content) - applyUpdate(ydoc, uint8arr) - const body = yDocToProsemirrorJSON(ydoc, field) return schema.nodeFromJSON(body) } catch (err: any) { diff --git a/server/collaboration/src/utils/collaborative-doc.ts b/server/collaboration/src/utils/collaborative-doc.ts index a928ed8b0c..0cecd08649 100644 --- a/server/collaboration/src/utils/collaborative-doc.ts +++ b/server/collaboration/src/utils/collaborative-doc.ts @@ -44,17 +44,6 @@ export async function loadCollaborativeDoc ( ctx: MeasureContext ): Promise { const { documentId, versionId } = parseCollaborativeDoc(collaborativeDoc) - return await loadCollaborativeDocVersion(minio, workspace, documentId, versionId, ctx) -} - -/** @public */ -export async function loadCollaborativeDocVersion ( - minio: MinioService, - workspace: WorkspaceId, - documentId: string, - versionId: CollaborativeDocVersion, - ctx: MeasureContext -): Promise { const historyDocumentId = collaborativeHistoryDocId(documentId) return await ctx.with('loadCollaborativeDoc', { type: 'content' }, async (ctx) => { @@ -141,7 +130,7 @@ export async function copyCollaborativeDoc ( target: CollaborativeDoc, ctx: MeasureContext ): Promise { - const { documentId: sourceDocumentId, versionId: sourceVersionId } = parseCollaborativeDoc(source) + const { documentId: sourceDocumentId } = parseCollaborativeDoc(source) const { documentId: targetDocumentId, versionId: targetVersionId } = parseCollaborativeDoc(target) if (sourceDocumentId === targetDocumentId) { @@ -151,7 +140,7 @@ export async function copyCollaborativeDoc ( await ctx.with('copyCollaborativeDoc', {}, async (ctx) => { const ySource = await ctx.with('loadCollaborativeDocVersion', {}, async (ctx) => { - return await loadCollaborativeDocVersion(minio, workspace, sourceDocumentId, sourceVersionId, ctx) + return await loadCollaborativeDoc(minio, workspace, source, ctx) }) if (ySource === undefined) { diff --git a/server/collaborator/src/extensions/action.ts b/server/collaborator/src/extensions/action.ts deleted file mode 100644 index 4a70164b52..0000000000 --- a/server/collaborator/src/extensions/action.ts +++ /dev/null @@ -1,169 +0,0 @@ -// -// Copyright © 2023 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { MeasureContext } from '@hcengineering/core' -import { Connection, Extension, Hocuspocus, onConfigurePayload, onStatelessPayload } from '@hocuspocus/server' -import { Transformer } from '@hocuspocus/transformer' -import * as Y from 'yjs' - -import { Context } from '../context' -import { - Action, - ActionStatus, - ActionStatusResponse, - DocumentContentAction, - DocumentCopyAction, - DocumentFieldCopyAction -} from '../types' - -export interface ActionsConfiguration { - ctx: MeasureContext - transformer: Transformer -} - -export class ActionsExtension implements Extension { - private readonly configuration: ActionsConfiguration - instance!: Hocuspocus - - constructor (configuration: ActionsConfiguration) { - this.configuration = configuration - } - - async onConfigure ({ instance }: onConfigurePayload): Promise { - this.instance = instance - } - - async onStateless (data: onStatelessPayload): Promise { - try { - const action = JSON.parse(data.payload) as Action - const context = data.connection.context - const { connection, documentName } = data - - console.log('process stateless message', action.action, documentName) - - await this.configuration.ctx.with(action.action, {}, async () => { - switch (action.action) { - case 'document.content': - await this.onDocumentContent(context, documentName, action) - this.sendActionStatus(connection, action, 'completed') - return - case 'document.copy': - await this.onCopyDocument(context, action) - this.sendActionStatus(connection, action, 'completed') - return - case 'document.field.copy': - await this.onCopyDocumentField(context, action) - this.sendActionStatus(connection, action, 'completed') - return - default: - console.error('unsupported action type', action) - } - }) - } catch (err: any) { - console.error('failed to process stateless message', err) - } - } - - sendActionStatus (connection: Connection, action: Action, status: ActionStatus): void { - const payload: ActionStatusResponse = { action, status } - connection.sendStateless(JSON.stringify(payload)) - } - - async onDocumentContent (context: Context, documentName: string, action: DocumentContentAction): Promise { - const instance = this.instance - const { content, field } = action.params - - const connection = await instance.openDirectConnection(documentName, context) - - try { - const document = connection.document - - if (document != null) { - if (!document.share.has(field)) { - const ydoc = this.configuration.transformer.toYdoc(content, field) - await connection.transact((target) => { - Y.applyUpdate(target, Y.encodeStateAsUpdate(ydoc), connection) - }) - } else { - console.warn(`document field '${field}' has already been initialized`) - } - } else { - console.warn('document is empty') - } - } finally { - await connection.disconnect() - } - } - - async onCopyDocument (context: Context, action: DocumentCopyAction): Promise { - const instance = this.instance - - const { sourceId, targetId } = action.params - console.info(`copy document content ${sourceId} -> ${targetId}`) - - const _context: Context = { ...context, initialContentId: '', targetContentId: '' } - - const sourceConnection = await instance.openDirectConnection(sourceId, _context) - const targetConnection = await instance.openDirectConnection(targetId, _context) - - try { - let updates = new Uint8Array() - - await sourceConnection.transact((source) => { - updates = Y.encodeStateAsUpdate(source) - }) - - await targetConnection.transact((target) => { - // TODO this does not work properly for existing documents - // we need to replace content, not only apply updates - Y.applyUpdate(target, updates) - }) - } finally { - await targetConnection.disconnect() - await sourceConnection.disconnect() - } - } - - async onCopyDocumentField (context: Context, action: DocumentFieldCopyAction): Promise { - const instance = this.instance - - const { documentId, srcFieldId, dstFieldId } = action.params - console.info(`copy document ${documentId} field content ${srcFieldId} -> ${dstFieldId}`) - - if (srcFieldId == null || srcFieldId === '' || dstFieldId == null || dstFieldId === '') { - console.error('empty srcFieldId or dstFieldId', srcFieldId, dstFieldId) - return - } - - const _context: Context = { ...context, initialContentId: '', targetContentId: '' } - - const docConnection = await instance.openDirectConnection(documentId, _context) - - try { - await docConnection.transact((doc) => { - const srcField = doc.getXmlFragment(srcFieldId) - const dstField = doc.getXmlFragment(dstFieldId) - - // similar to XmlFragment's clone method - dstField.insert( - 0, - srcField.toArray().map((item) => (item instanceof Y.AbstractType ? item.clone() : item)) as any - ) - }) - } finally { - await docConnection.disconnect() - } - } -} diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index 3585a9a941..40fd316e19 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -30,7 +30,6 @@ import { WebSocket, WebSocketServer } from 'ws' import { getWorkspaceInfo } from './account' import { Config } from './config' import { Context, buildContext } from './context' -import { ActionsExtension } from './extensions/action' import { HtmlTransformer } from './transformers/html' import { StorageExtension } from './extensions/storage' import { Controller, getClientFactory } from './platform' @@ -126,10 +125,6 @@ export async function start ( unloadImmediately: false, extensions: [ - new ActionsExtension({ - ctx: extensionsCtx.newChild('actions', {}), - transformer - }), new StorageExtension({ ctx: extensionsCtx.newChild('storage', {}), adapter: new RouterStorageAdapter( diff --git a/server/collaborator/src/storage/minio.ts b/server/collaborator/src/storage/minio.ts index c764216496..035ebb3ba1 100644 --- a/server/collaborator/src/storage/minio.ts +++ b/server/collaborator/src/storage/minio.ts @@ -13,8 +13,13 @@ // limitations under the License. // -import { loadCollaborativeDocVersion, saveCollaborativeDocVersion } from '@hcengineering/collaboration' -import { CollaborativeDocVersion, CollaborativeDocVersionHead, MeasureContext } from '@hcengineering/core' +import { loadCollaborativeDoc, saveCollaborativeDocVersion } from '@hcengineering/collaboration' +import { + CollaborativeDocVersion, + CollaborativeDocVersionHead, + MeasureContext, + formatCollaborativeDocVersion +} from '@hcengineering/core' import { MinioService } from '@hcengineering/minio' import { Doc as YDoc } from 'yjs' @@ -59,7 +64,8 @@ export class MinioStorageAdapter implements StorageAdapter { return await this.ctx.with('load-document', {}, async (ctx) => { try { - return await loadCollaborativeDocVersion(this.minio, workspaceId, minioDocumentId, versionId, ctx) + const collaborativeDoc = formatCollaborativeDocVersion({ documentId: minioDocumentId, versionId }) + return await loadCollaborativeDoc(this.minio, workspaceId, collaborativeDoc, ctx) } catch { return undefined }