UBERF-4725 Migrate collaborative content (#5717)

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>

.github/workflows/main.yml

@@ -243,6 +243,7 @@ jobs:
           docker logs $(docker ps | grep transactor | cut -f 1 -d ' ') > logs/transactor.log
           docker logs $(docker ps | grep account | cut -f 1 -d ' ') > logs/account.log
           docker logs $(docker ps | grep front | cut -f 1 -d ' ') > logs/front.log
+          docker logs $(docker ps | grep collaborator | cut -f 1 -d ' ') > logs/collaborator.log
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -438,6 +439,7 @@ jobs:
           docker logs $(docker ps | grep transactor | cut -f 1 -d ' ') > logs/uweb-transactor.log
           docker logs $(docker ps | grep account | cut -f 1 -d ' ') > logs/uweb-account.log
           docker logs $(docker ps | grep front | cut -f 1 -d ' ') > logs/uweb-front.log
+          docker logs $(docker ps | grep collaborator | cut -f 1 -d ' ') > logs/uweb-collaborator.log
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4

.vscode/launch.json

@@ -43,7 +43,6 @@
         "SERVER_SECRET": "secret",
         "ENABLE_CONSOLE": "true",
         "COLLABORATOR_URL": "ws://localhost:3078",
-        "COLLABORATOR_API_URL": "http://localhost:3078",
         "REKONI_URL": "http://localhost:4004",
         "FRONT_URL": "http://localhost:8080",
         "ACCOUNTS_URL": "http://localhost:3000",
@@ -104,7 +103,6 @@
         "UPLOAD_URL": "/files",
         "SERVER_PORT": "8087",
         "COLLABORATOR_URL": "ws://localhost:3078",
-        "COLLABORATOR_API_URL": "http://localhost:3078",
         "CALENDAR_URL": "http://localhost:8095",
         "GMAIL_URL": "http://localhost:8088",
         "TELEGRAM_URL": "http://localhost:8086",
@@ -239,7 +237,7 @@
         "CLIENT_ID": "${env:POD_GITHUB_CLIENTID}",
         "CLIENT_SECRET": "${env:POD_GITHUB_CLIENT_SECRET}",
         "PRIVATE_KEY": "${env:POD_GITHUB_PRIVATE_KEY}",
-        "COLLABORATOR_API_URL": "http://localhost:3078",
+        "COLLABORATOR_URL": "ws://localhost:3078",
         "SYSTEM_EMAIL": "anticrm@hc.engineering",
         "MINIO_ENDPOINT": "localhost",
         "MINIO_ACCESS_KEY": "minioadmin",
@@ -280,7 +278,7 @@
       "env": {
         "SERVER_SECRET": "secret",
         "ACCOUNTS_URL": "http://localhost:3000",
-        "COLLABORATOR_API_URL": "http://localhost:3078",
+        "COLLABORATOR_URL": "ws://localhost:3078",
         "STORAGE_CONFIG": "minio|localhost?accessKey=minioadmin&secretKey=minioadmin",
         "MONGO_URL": "mongodb://localhost:27017"
       },

desktop/src/ui/platform.ts

@@ -203,7 +203,6 @@ export async function configurePlatform (): Promise<void> {
   setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
   setMetadata(presentation.metadata.FilesURL, config.FILES_URL)
   setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
-  setMetadata(presentation.metadata.CollaboratorApiUrl, config.COLLABORATOR_API_URL)
   setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG))
   setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL)
 

desktop/src/ui/types.ts

@@ -6,7 +6,6 @@ import { ScreenSource } from '@hcengineering/love'
 export interface Config {
   ACCOUNTS_URL: string
   COLLABORATOR_URL: string
-  COLLABORATOR_API_URL: string
   FRONT_URL: string
   FILES_URL: string
   UPLOAD_URL: string

dev/doc-import-tool/package.json

@@ -15,7 +15,7 @@
     "build:watch": "compile",
     "_phase:bundle": "rushx bundle",
     "bundle": "mkdir -p bundle && node esbuild.js",
-    "run-local": "cross-env SERVER_SECRET=secret MONGO_URL=mongodb://localhost:27017 COLLABORATOR_URL=ws://localhost:3078 COLLABORATOR_API_URL=http://localhost:3078 STORAGE_CONFIG=minio|minio?accessKey=minioadmin&secretKey=minioadmin PRODUCT_ID=ezqms node --nolazy -r ts-node/register ./src/__start.ts",
+    "run-local": "cross-env SERVER_SECRET=secret MONGO_URL=mongodb://localhost:27017 COLLABORATOR_URL=ws://localhost:3078 STORAGE_CONFIG=minio|minio?accessKey=minioadmin&secretKey=minioadmin PRODUCT_ID=ezqms node --nolazy -r ts-node/register ./src/__start.ts",
     "run": "cross-env node -r ts-node/register --max-old-space-size=8000 ./src/__start.ts",
     "format": "format src",
     "test": "jest --passWithNoTests --silent",

dev/doc-import-tool/src/config.ts

@@ -8,7 +8,7 @@ import { HtmlConversionBackend } from './convert/convert'
 export interface Config {
   doc: string
   token: string
-  collaboratorApiURL: string
+  collaboratorURL: string
   uploadURL: string
   workspaceId: WorkspaceId
   owner: Ref<Employee>

dev/doc-import-tool/src/import.ts

@@ -21,7 +21,7 @@ import core, {
   Ref,
   TxOperations,
   generateId,
-  getCollaborativeDoc,
+  makeCollaborativeDoc,
   systemAccountEmail,
   type Blob
 } from '@hcengineering/core'
@@ -100,7 +100,7 @@ async function createDocument (
     abstract: '',
     effectiveDate: 0,
     reviewInterval: DEFAULT_PERIODIC_REVIEW_INTERVAL,
-    content: getCollaborativeDoc(generateId()),
+    content: makeCollaborativeDoc(generateId()),
     snapshots: 0,
     plannedEffectiveDate: 0
   }
@@ -168,7 +168,7 @@ async function createTemplateIfNotExist (
     approvers: [],
     coAuthors: [],
     changeControl: ccRecordId,
-    content: getCollaborativeDoc(generateId()),
+    content: makeCollaborativeDoc(generateId()),
     snapshots: 0,
     plannedEffectiveDate: 0
   }
@@ -208,8 +208,8 @@ async function createSections (
     throw new Error(`Invalid document: ${JSON.stringify(doc)}`)
   }
 
-  const { collaboratorApiURL, token, workspaceId } = config
+  const { collaboratorURL, token, workspaceId } = config
-  const collaborator = getCollaboratorClient(txops.getHierarchy(), workspaceId, token, collaboratorApiURL)
+  const collaborator = getCollaboratorClient(workspaceId, token, collaboratorURL)
 
   console.log('Creating document content')
 
@@ -229,7 +229,7 @@ async function createSections (
       content += `<h1>${section.title}</h1>${section.content}`
     }
 
-    await collaborator.updateContent(collabId, 'content', content)
+    await collaborator.updateContent(collabId, { content })
   } finally {
     // do nothing
   }

dev/doc-import-tool/src/index.ts

@@ -44,8 +44,8 @@ export function docImportTool (): void {
     process.exit(1)
   }
 
-  const collaboratorApiUrl = process.env.COLLABORATOR_API_URL
+  const collaboratorUrl = process.env.COLLABORATOR_URL
-  if (collaboratorApiUrl === undefined) {
+  if (collaboratorUrl === undefined) {
     console.error('please provide collaborator url')
     process.exit(1)
   }
@@ -104,7 +104,7 @@ export function docImportTool (): void {
             space: cmd.space,
             uploadURL: uploadUrl,
             storageAdapter,
-            collaboratorApiURL: collaboratorApiUrl,
+            collaboratorURL: collaboratorUrl,
             token: generateToken(systemAccountEmail, workspaceId)
           }
 

dev/docker-compose.yaml

@@ -106,7 +106,6 @@ services:
       - TELEGRAM_URL=http://localhost:8086
       - REKONI_URL=http://localhost:4004
       - COLLABORATOR_URL=ws://localhost:3078
-      - COLLABORATOR_API_URL=http://localhost:3078
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - GITHUB_URL=http://localhost:3500
       - PRINT_URL=http://localhost:4005

dev/local-mongo/docker-compose.yaml

@@ -93,7 +93,6 @@ services:
       - TELEGRAM_URL=http://localhost:8086
       - REKONI_URL=http://localhost:4004
       - COLLABORATOR_URL=ws://localhost:3078
-      - COLLABORATOR_API_URL=http://localhost:3078
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - GITHUB_URL=http://localhost:3500
       - PRINT_URL=http://localhost:4005

dev/prod/.env

@@ -5,7 +5,6 @@ FRONT_URL=http://localhost:8080
 REKONI_URL=http://localhost:4004
 
 COLLABORATOR_URL=ws://locahost:3078
-COLLABORATOR_API_URL=http://locahost:3078
 
 PRINT_URL=http://localhost:4005
 SIGN_URL=http://localhost:4006

dev/prod/config.json

@@ -1,7 +1,6 @@
 {
   "ACCOUNTS_URL":"http://localhost:3000",
   "COLLABORATOR_URL": "ws://localhost:3078",
-  "COLLABORATOR_API_URL": "http://localhost:3078",
   "UPLOAD_URL":"/files",
   "REKONI_URL": "http://localhost:4004",
   "PRINT_URL": "http://localhost:4005",

dev/prod/public/config-dev.json

@@ -6,6 +6,5 @@
   "GMAIL_URL": "https://gmail.hc.engineering",
   "CALENDAR_URL": "https://calendar.hc.engineering",
   "REKONI_URL": "https://rekoni.hc.engineering",
-  "COLLABORATOR_URL": "wss://collaborator.hc.engineering",
+  "COLLABORATOR_URL": "wss://collaborator.hc.engineering"
-  "COLLABORATOR_API_URL": "https://collaborator.hc.engineering"
 }

dev/prod/public/config.json

@@ -1,7 +1,6 @@
 {
   "ACCOUNTS_URL":"/account",
   "COLLABORATOR_URL": "ws://localhost:3078",
-  "COLLABORATOR_API_URL": "http://localhost:3078",
   "UPLOAD_URL":"/files",
   "TELEGRAM_URL": "http://localhost:8086",
   "GMAIL_URL": "http://localhost:8088",

dev/prod/src/platform.ts

@@ -124,7 +124,6 @@ export interface Config {
   MODEL_VERSION: string
   VERSION: string
   COLLABORATOR_URL: string
-  COLLABORATOR_API_URL: string
   REKONI_URL: string
   TELEGRAM_URL: string
   GMAIL_URL: string
@@ -288,7 +287,6 @@ export async function configurePlatform() {
   setMetadata(presentation.metadata.FilesURL, config.FILES_URL)
   setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
   setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
-  setMetadata(presentation.metadata.CollaboratorApiUrl, config.COLLABORATOR_API_URL)
 
   setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL)
   setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG))

dev/tool/src/clean.ts

@@ -1214,10 +1214,6 @@ async function updateId (
             const markup = (contentDoc as any)[attrName] as Markup
             const newMarkup = markup.replaceAll(doc._id, newId)
             await update(h, db, contentDoc, { [attrName]: newMarkup })
-          } else if (attr.type._class === core.class.TypeCollaborativeMarkup) {
-            const markup = (contentDoc as any)[attrName]
-            const newMarkup = markup.replaceAll(doc._id, newId)
-            await update(h, db, contentDoc, { [attrName]: newMarkup })
           } else if (attr.type._class === core.class.TypeCollaborativeDoc) {
             const collaborativeDoc = (contentDoc as any)[attr.name] as CollaborativeDoc
             await updateYDoc(ctx, collaborativeDoc, storage, workspaceId, contentDoc, newId, doc)

dev/tool/src/markup.ts

@@ -7,7 +7,7 @@ import core, {
   type MeasureContext,
   type Ref,
   type WorkspaceId,
-  getCollaborativeDocId
+  makeCollaborativeDoc
 } from '@hcengineering/core'
 import { getMongoClient, getWorkspaceDB } from '@hcengineering/mongo'
 import { type StorageAdapter } from '@hcengineering/server-core'
@@ -39,10 +39,7 @@ export async function fixJsonMarkup (
 
       const attributes = hierarchy.getAllAttributes(_class)
       const filtered = Array.from(attributes.values()).filter((attribute) => {
-        return (
+        return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup)
-          hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) ||
-          hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)
-        )
       })
       if (filtered.length === 0) continue
 
@@ -88,7 +85,7 @@ async function processFixJsonMarkupFor (
           }
           if (res !== value) {
             update[attribute.name] = res
-            remove.push(getCollaborativeDocId(doc._id, attribute.name))
+            remove.push(makeCollaborativeDoc(doc._id, attribute.name))
           }
         }
       } catch {}

models/activity/src/migration.ts

@@ -46,9 +46,7 @@ async function migrateMarkup (client: MigrationClient): Promise<void> {
     DOMAIN_ACTIVITY,
     {
       _class: activity.class.DocUpdateMessage,
-      'attributeUpdates.attrClass': {
+      'attributeUpdates.attrClass': core.class.TypeMarkup
-        $in: [core.class.TypeMarkup, core.class.TypeCollaborativeMarkup]
-      }
     },
     {
       projection: {

models/contact/src/index.ts

@@ -39,8 +39,8 @@ import {
   IndexKind,
   type Blob,
   type Class,
+  type CollaborativeDoc,
   type Domain,
-  type Markup,
   type Ref,
   type Timestamp
 } from '@hcengineering/core'
@@ -54,7 +54,7 @@ import {
   ReadOnly,
   TypeBlob,
   TypeBoolean,
-  TypeCollaborativeMarkup,
+  TypeCollaborativeDoc,
   TypeDate,
   TypeRecord,
   TypeRef,
@@ -173,9 +173,9 @@ export class TMember extends TAttachedDoc implements Member {
 @Model(contact.class.Organization, contact.class.Contact)
 @UX(contact.string.Organization, contact.icon.Company, 'ORG', 'name', undefined, contact.string.Organizations)
 export class TOrganization extends TContact implements Organization {
-  @Prop(TypeCollaborativeMarkup(), core.string.Description)
+  @Prop(TypeCollaborativeDoc(), core.string.Description)
   @Index(IndexKind.FullText)
-    description?: Markup
+    description!: CollaborativeDoc
 
   @Prop(Collection(contact.class.Member), contact.string.Members)
     members!: number

models/controlled-documents/src/migration.ts

@@ -8,7 +8,7 @@ import {
   TxOperations,
   generateId,
   DOMAIN_TX,
-  getCollaborativeDoc,
+  makeCollaborativeDoc,
   MeasureMetricsContext,
   type Class,
   type Doc,
@@ -143,7 +143,7 @@ async function createProductChangeControlTemplate (tx: TxOperations): Promise<vo
         minor: 1,
         state: DocumentState.Effective,
         commentSequence: 0,
-        content: getCollaborativeDoc(generateId())
+        content: makeCollaborativeDoc(generateId())
       },
       ccCategory
     )

models/core/package.json

@@ -31,6 +31,8 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/model": "^0.6.11",
     "@hcengineering/platform": "^0.6.11",
-    "@hcengineering/storage": "^0.6.0"
+    "@hcengineering/storage": "^0.6.0",
+    "@hcengineering/collaboration": "^0.6.0",
+    "@hcengineering/text": "^0.6.5"
   }
 }

models/core/src/core.ts

@@ -233,10 +233,6 @@ export class TTypeFileSize extends TType {}
 @Model(core.class.TypeMarkup, core.class.Type)
 export class TTypeMarkup extends TType {}
 
-@UX(core.string.Collaborative)
-@Model(core.class.TypeCollaborativeMarkup, core.class.Type)
-export class TTypeCollaborativeMarkup extends TType {}
-
 @UX(core.string.Ref)
 @Model(core.class.RefTo, core.class.Type)
 export class TRefTo extends TType implements RefTo<Doc> {

models/core/src/index.ts

@@ -61,7 +61,6 @@ import {
   TTypeBoolean,
   TTypeCollaborativeDoc,
   TTypeCollaborativeDocVersion,
-  TTypeCollaborativeMarkup,
   TTypeDate,
   TTypeFileSize,
   TTypeHyperlink,
@@ -141,7 +140,6 @@ export function createModel (builder: Builder): void {
     TTypeMarkup,
     TTypeCollaborativeDoc,
     TTypeCollaborativeDocVersion,
-    TTypeCollaborativeMarkup,
     TArrOf,
     TRefTo,
     TTypeDate,

models/core/src/migration.ts

@@ -13,16 +13,23 @@
 // limitations under the License.
 //
 
+import { saveCollaborativeDoc, takeCollaborativeDocSnapshot } from '@hcengineering/collaboration'
 import core, {
   DOMAIN_BLOB,
   DOMAIN_DOC_INDEX_STATE,
   DOMAIN_STATUS,
   DOMAIN_TX,
   MeasureMetricsContext,
+  collaborativeDocParse,
   coreId,
   generateId,
   isClassIndexable,
+  makeCollaborativeDoc,
+  type AnyAttribute,
   type Blob,
+  type Doc,
+  type Domain,
+  type MeasureContext,
   type Ref,
   type Space,
   type Status,
@@ -33,10 +40,14 @@ import {
   tryMigrate,
   tryUpgrade,
   type MigrateOperation,
+  type MigrateUpdate,
   type MigrationClient,
+  type MigrationDocumentQuery,
+  type MigrationIterator,
   type MigrationUpgradeClient
 } from '@hcengineering/model'
-import { type StorageAdapterEx } from '@hcengineering/storage'
+import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage'
+import { markupToYDoc } from '@hcengineering/text'
 import { DOMAIN_SPACE } from './security'
 
 async function migrateStatusesToModel (client: MigrationClient): Promise<void> {
@@ -143,6 +154,101 @@ async function migrateStatusTransactions (client: MigrationClient): Promise<void
   )
 }
 
+async function migrateCollaborativeContentToStorage (client: MigrationClient): Promise<void> {
+  const ctx = new MeasureMetricsContext('migrate_content', {})
+  const storageAdapter = client.storageAdapter
+
+  const hierarchy = client.hierarchy
+  const classes = hierarchy.getDescendants(core.class.Doc)
+  for (const _class of classes) {
+    const domain = hierarchy.findDomain(_class)
+    if (domain === undefined) continue
+
+    const attributes = hierarchy.getAllAttributes(_class)
+    const filtered = Array.from(attributes.values()).filter((attribute) => {
+      return hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)
+    })
+    if (filtered.length === 0) continue
+
+    const iterator = await client.traverse(domain, { _class })
+    try {
+      console.log('processing', _class)
+      await processMigrateContentFor(ctx, domain, filtered, client, storageAdapter, iterator)
+    } finally {
+      await iterator.close()
+    }
+  }
+}
+
+async function processMigrateContentFor (
+  ctx: MeasureContext,
+  domain: Domain,
+  attributes: AnyAttribute[],
+  client: MigrationClient,
+  storageAdapter: StorageAdapter,
+  iterator: MigrationIterator<Doc>
+): Promise<void> {
+  let processed = 0
+  while (true) {
+    const docs = await iterator.next(1000)
+    if (docs === null || docs.length === 0) {
+      break
+    }
+
+    const timestamp = Date.now()
+    const revisionId = `${timestamp}`
+
+    const operations: { filter: MigrationDocumentQuery<Doc>, update: MigrateUpdate<Doc> }[] = []
+
+    for (const doc of docs) {
+      const update: MigrateUpdate<Doc> = {}
+
+      for (const attribute of attributes) {
+        const collaborativeDoc = makeCollaborativeDoc(doc._id, attribute.name, revisionId)
+
+        const value = (doc as any)[attribute.name] as string
+        if (value != null && value.startsWith('{')) {
+          const { documentId } = collaborativeDocParse(collaborativeDoc)
+          const blob = await storageAdapter.stat(ctx, client.workspaceId, documentId)
+          // only for documents not in storage
+          if (blob === undefined) {
+            const ydoc = markupToYDoc(value, attribute.name)
+            await saveCollaborativeDoc(storageAdapter, client.workspaceId, collaborativeDoc, ydoc, ctx)
+            await takeCollaborativeDocSnapshot(
+              storageAdapter,
+              client.workspaceId,
+              collaborativeDoc,
+              ydoc,
+              {
+                versionId: revisionId,
+                name: 'Migration to storage',
+                createdBy: core.account.System,
+                createdOn: Date.now()
+              },
+              ctx
+            )
+          }
+
+          update[attribute.name] = collaborativeDoc
+        } else if (value == null) {
+          update[attribute.name] = makeCollaborativeDoc(doc._id, attribute.name, revisionId)
+        }
+      }
+
+      if (Object.keys(update).length > 0) {
+        operations.push({ filter: { _id: doc._id }, update })
+      }
+    }
+
+    if (operations.length > 0) {
+      await client.bulk(domain, operations)
+    }
+
+    processed += docs.length
+    console.log('...processed', processed)
+  }
+}
+
 export const coreOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     // We need to delete all documents in doc index state for missing classes
@@ -189,6 +295,10 @@ export const coreOperation: MigrateOperation = {
         func: async (client: MigrationClient) => {
           await client.update(DOMAIN_DOC_INDEX_STATE, {}, { $set: { needIndex: true } })
         }
+      },
+      {
+        state: 'collaborative-content-to-storage',
+        func: migrateCollaborativeContentToStorage
       }
     ])
   },

models/document/src/migration.ts

@@ -13,15 +13,7 @@
 // limitations under the License.
 //
 
-import { type Attachment } from '@hcengineering/attachment'
+import { DOMAIN_TX, MeasureMetricsContext } from '@hcengineering/core'
-import {
-  DOMAIN_TX,
-  getCollaborativeDoc,
-  MeasureMetricsContext,
-  type Class,
-  type Doc,
-  type Ref
-} from '@hcengineering/core'
 import { type Document, type Teamspace } from '@hcengineering/document'
 import {
   tryMigrate,
@@ -29,126 +21,12 @@ import {
   type MigrationClient,
   type MigrationUpgradeClient
 } from '@hcengineering/model'
-import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { type Asset } from '@hcengineering/platform'
 
 import document, { documentId, DOMAIN_DOCUMENT } from './index'
 import { loadCollaborativeDoc, saveCollaborativeDoc, yDocCopyXmlField } from '@hcengineering/collaboration'
 
-async function migrateCollaborativeContent (client: MigrationClient): Promise<void> {
-  const attachedFiles = await client.find<Attachment>(DOMAIN_ATTACHMENT, {
-    _class: 'document:class:CollaboratorDocument' as Ref<Class<Doc>>,
-    attachedToClass: document.class.Document
-  })
-
-  for (const attachment of attachedFiles) {
-    const collaborativeDoc = getCollaborativeDoc(attachment._id)
-
-    await client.update(
-      DOMAIN_DOCUMENT,
-      {
-        _id: attachment.attachedTo,
-        _class: attachment.attachedToClass,
-        content: {
-          $exists: false
-        }
-      },
-      {
-        $set: {
-          content: collaborativeDoc
-        }
-      }
-    )
-  }
-
-  // delete snapshots in old format
-  await client.deleteMany(DOMAIN_DOCUMENT, {
-    _class: 'document:class:DocumentSnapshot' as Ref<Class<Doc>>,
-    contentId: { $exists: true }
-  })
-
-  await client.update(
-    DOMAIN_DOCUMENT,
-    {
-      _class: document.class.Document,
-      snapshots: { $gt: 0 }
-    },
-    {
-      $set: {
-        snapshots: 0
-      }
-    }
-  )
-
-  // delete old snapshot transactions
-  await client.deleteMany(DOMAIN_TX, {
-    _class: core.class.TxCollectionCUD,
-    objectClass: document.class.Document,
-    collection: 'snapshots'
-  })
-}
-
-async function fixCollaborativeContentId (client: MigrationClient): Promise<void> {
-  const documents = await client.find<Document>(DOMAIN_DOCUMENT, {
-    content: { $exists: true }
-  })
-
-  // there was a wrong migration that assigned incorrect collaborative doc id
-  for (const document of documents) {
-    if (!document.content.includes(':')) {
-      await client.update(DOMAIN_DOCUMENT, { _id: document._id }, { content: getCollaborativeDoc(document.content) })
-    }
-  }
-}
-
-async function migrateWrongDomainContent (client: MigrationClient): Promise<void> {
-  // migrate content saved into wrong domain
-  const attachedFiles = await client.find<Attachment>(DOMAIN_DOCUMENT, {
-    _class: 'document:class:CollaboratorDocument' as Ref<Class<Doc>>,
-    attachedToClass: document.class.Document
-  })
-
-  for (const attachment of attachedFiles) {
-    const collaborativeDoc = getCollaborativeDoc(attachment._id)
-
-    await client.update(
-      DOMAIN_DOCUMENT,
-      {
-        _id: attachment.attachedTo,
-        _class: attachment.attachedToClass,
-        content: {
-          $exists: false
-        }
-      },
-      {
-        $set: {
-          content: collaborativeDoc
-        }
-      }
-    )
-  }
-
-  await client.move(
-    DOMAIN_DOCUMENT,
-    {
-      _class: 'document:class:CollaboratorDocument' as Ref<Class<Doc>>,
-      attachedToClass: document.class.Document
-    },
-    DOMAIN_ATTACHMENT
-  )
-}
-
-async function migrateDeleteCollaboratorDocument (client: MigrationClient): Promise<void> {
-  await client.deleteMany(DOMAIN_ATTACHMENT, { _class: 'document:class:CollaboratorDocument' as Ref<Class<Doc>> })
-  await client.deleteMany(DOMAIN_DOCUMENT, { _class: 'document:class:CollaboratorDocument' as Ref<Class<Doc>> })
-  await client.deleteMany(DOMAIN_TX, {
-    _class: core.class.TxCollectionCUD,
-    collection: 'attachments',
-    'tx.objectClass': 'document:class:CollaboratorDocument' as Ref<Class<Doc>>
-  })
-}
-
 async function migrateDocumentIcons (client: MigrationClient): Promise<void> {
   await client.update<Teamspace>(
     DOMAIN_SPACE,
@@ -173,60 +51,6 @@ async function migrateDocumentIcons (client: MigrationClient): Promise<void> {
   )
 }
 
-async function setNoParent (client: MigrationClient): Promise<void> {
-  await client.update(
-    DOMAIN_DOCUMENT,
-    {
-      _class: document.class.Document,
-      attachedTo: { $exists: false }
-    },
-    {
-      $set: {
-        attachedTo: document.ids.NoParent,
-        attachedToClass: document.class.Document
-      }
-    }
-  )
-  const docsWithParent = (await client.find(DOMAIN_DOCUMENT, {
-    _class: document.class.Document,
-    attachedTo: { $exists: true, $ne: document.ids.NoParent }
-  })) as Document[]
-  for (const doc of docsWithParent) {
-    const parent = await client.find(DOMAIN_DOCUMENT, {
-      _class: document.class.Document,
-      _id: doc.attachedTo
-    })
-    if (parent.length === 0) continue
-    if (parent[0].space !== doc.space) {
-      await client.update(
-        DOMAIN_DOCUMENT,
-        {
-          _class: document.class.Document,
-          _id: doc._id
-        },
-        {
-          $set: {
-            attachedTo: document.ids.NoParent
-          }
-        }
-      )
-    }
-  }
-
-  await client.update(
-    DOMAIN_DOCUMENT,
-    {
-      _class: document.class.Document,
-      attachedTo: ''
-    },
-    {
-      $set: {
-        attachedTo: document.ids.NoParent
-      }
-    }
-  )
-}
-
 async function migrateTeamspaces (client: MigrationClient): Promise<void> {
   await client.update(
     DOMAIN_SPACE,
@@ -306,28 +130,6 @@ async function migrateContentField (client: MigrationClient): Promise<void> {
 export const documentOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     await tryMigrate(client, documentId, [
-      {
-        state: 'migrate-no-parent',
-        func: async (client) => {
-          await setNoParent(client)
-        }
-      },
-      {
-        state: 'collaborativeContent',
-        func: migrateCollaborativeContent
-      },
-      {
-        state: 'fixCollaborativeContentId',
-        func: fixCollaborativeContentId
-      },
-      {
-        state: 'wrongDomainContent',
-        func: migrateWrongDomainContent
-      },
-      {
-        state: 'deleteCollaboratorDocument',
-        func: migrateDeleteCollaboratorDocument
-      },
       {
         state: 'updateDocumentIcons',
         func: migrateDocumentIcons

models/lead/src/types.ts

@@ -17,6 +17,7 @@ import type { Employee } from '@hcengineering/contact'
 import {
   Account,
   IndexKind,
+  type CollaborativeDoc,
   type Role,
   type RolesAssignment,
   type Ref,
@@ -31,7 +32,7 @@ import {
   Model,
   Prop,
   ReadOnly,
-  TypeCollaborativeMarkup,
+  TypeCollaborativeDoc,
   TypeDate,
   TypeMarkup,
   TypeRef,
@@ -94,9 +95,9 @@ export class TCustomer extends TContact implements Customer {
   @Prop(Collection(lead.class.Lead), lead.string.Leads)
     leads?: number
 
-  @Prop(TypeCollaborativeMarkup(), core.string.Description)
+  @Prop(TypeCollaborativeDoc(), core.string.Description)
   @Index(IndexKind.FullText)
-    description!: string
+    description!: CollaborativeDoc
 }
 
 @Mixin(lead.mixin.DefaultFunnelTypeData, lead.class.Funnel)

models/recruit/src/types.ts

@@ -17,6 +17,7 @@ import type { Employee, Organization } from '@hcengineering/contact'
 import {
   Account,
   IndexKind,
+  type CollaborativeDoc,
   type Domain,
   type Markup,
   type Ref,
@@ -34,7 +35,7 @@ import {
   Prop,
   ReadOnly,
   TypeBoolean,
-  TypeCollaborativeMarkup,
+  TypeCollaborativeDoc,
   TypeDate,
   TypeMarkup,
   TypeRef,
@@ -63,9 +64,9 @@ import recruit from './plugin'
 @Model(recruit.class.Vacancy, task.class.Project)
 @UX(recruit.string.Vacancy, recruit.icon.Vacancy, 'VCN', 'name', undefined, recruit.string.Vacancies)
 export class TVacancy extends TProject implements Vacancy {
-  @Prop(TypeCollaborativeMarkup(), recruit.string.FullDescription)
+  @Prop(TypeCollaborativeDoc(), recruit.string.FullDescription)
   @Index(IndexKind.FullText)
-    fullDescription?: string
+    fullDescription!: CollaborativeDoc
 
   @Prop(Collection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files })
     attachments?: number

models/text-editor/src/migration.ts

@@ -34,10 +34,7 @@ async function migrateMarkup (client: MigrationClient): Promise<void> {
 
     const attributes = hierarchy.getAllAttributes(_class)
     const filtered = Array.from(attributes.values()).filter((attribute) => {
-      return (
+      return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup)
-        hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) ||
-        hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)
-      )
     })
     if (filtered.length === 0) continue
 
@@ -106,10 +103,7 @@ async function fixMigrateMarkup (client: MigrationClient): Promise<void> {
 
     const attributes = hierarchy.getAllAttributes(_class)
     const filtered = Array.from(attributes.values()).filter((attribute) => {
-      return (
+      return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup)
-        hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) ||
-        hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)
-      )
     })
     if (filtered.length === 0) continue
 

models/tracker/src/types.ts

@@ -18,8 +18,9 @@ import contact, { type Employee, type Person } from '@hcengineering/contact'
 import {
   DOMAIN_MODEL,
   DateRangeMode,
-  type Domain,
   IndexKind,
+  type CollaborativeDoc,
+  type Domain,
   type Markup,
   type Ref,
   type RelatedDocument,
@@ -39,7 +40,7 @@ import {
   Model,
   Prop,
   ReadOnly,
-  TypeCollaborativeMarkup,
+  TypeCollaborativeDoc,
   TypeDate,
   TypeMarkup,
   TypeNumber,
@@ -182,9 +183,9 @@ export class TIssue extends TTask implements Issue {
   @Index(IndexKind.FullText)
     title!: string
 
-  @Prop(TypeCollaborativeMarkup(), tracker.string.Description)
+  @Prop(TypeCollaborativeDoc(), tracker.string.Description)
   @Index(IndexKind.FullText)
-    description!: Markup
+    description!: CollaborativeDoc
 
   @Prop(TypeRef(tracker.class.IssueStatus), tracker.string.Status, {
     _id: tracker.attribute.IssueStatus,
@@ -275,7 +276,7 @@ export class TIssueTemplate extends TDoc implements IssueTemplate {
   @Index(IndexKind.FullText)
     title!: string
 
-  @Prop(TypeCollaborativeMarkup(), tracker.string.Description)
+  @Prop(TypeMarkup(), tracker.string.Description)
   @Index(IndexKind.FullText)
     description!: Markup
 

models/view/src/index.ts

@@ -495,17 +495,8 @@ export function createModel (builder: Builder): void {
     editor: view.component.HTMLEditor
   })
 
-  classPresenter(
+  builder.mixin(core.class.TypeCollaborativeDoc, core.class.Class, view.mixin.ActivityAttributePresenter, {
-    builder,
+    presenter: view.component.MarkupDiffPresenter
-    core.class.TypeCollaborativeMarkup,
-    view.component.MarkupPresenter,
-    undefined,
-    undefined,
-    view.component.MarkupDiffPresenter
-  )
-
-  builder.mixin(core.class.TypeCollaborativeMarkup, core.class.Class, view.mixin.InlineAttributEditor, {
-    editor: view.component.CollaborativeHTMLEditor
   })
 
   builder.mixin(core.class.TypeCollaborativeDoc, core.class.Class, view.mixin.InlineAttributEditor, {

packages/collaborator-client/src/client.ts

@@ -16,33 +16,39 @@
 import {
   Account,
   CollaborativeDoc,
-  Hierarchy,
   Markup,
   Ref,
   Timestamp,
   WorkspaceId,
+  collaborativeDocWithLastVersion,
   collaborativeDocWithVersion,
   concatLink
 } from '@hcengineering/core'
 import { DocumentId } from './types'
 import { formatMinioDocumentId } from './utils'
 
+/** @public */
+export interface DocumentSnapshotParams {
+  createdBy: Ref<Account>
+  versionId: string
+  versionName?: string
+}
+
 /** @public */
 export interface GetContentRequest {
   documentId: DocumentId
-  field: string
 }
 
 /** @public */
 export interface GetContentResponse {
-  html: string
+  content: Record<string, Markup>
 }
 
 /** @public */
 export interface UpdateContentRequest {
   documentId: DocumentId
-  field: string
+  content: Record<string, Markup>
-  html: string
+  snapshot?: DocumentSnapshotParams
 }
 
 /** @public */
@@ -54,6 +60,7 @@ export interface CopyContentRequest {
   documentId: DocumentId
   sourceField: string
   targetField: string
+  snapshot?: DocumentSnapshotParams
 }
 
 /** @public */
@@ -82,8 +89,7 @@ export interface RemoveDocumentResponse {}
 /** @public */
 export interface TakeSnapshotRequest {
   documentId: DocumentId
-  createdBy: Ref<Account>
+  snapshot: DocumentSnapshotParams
-  snapshotName: string
 }
 
 /** @public */
@@ -95,38 +101,36 @@ export interface TakeSnapshotResponse {
   createdOn: Timestamp
 }
 
-/** @public */
-export interface CollaborativeDocSnapshotParams {
-  snapshotName: string
-  createdBy: Ref<Account>
-}
-
 /** @public */
 export interface CollaboratorClient {
   // field operations
-  getContent: (collaborativeDoc: CollaborativeDoc, field: string) => Promise<Markup>
+  getContent: (collaborativeDoc: CollaborativeDoc) => Promise<Record<string, Markup>>
-  updateContent: (collaborativeDoc: CollaborativeDoc, field: string, value: Markup) => Promise<void>
+  updateContent: (
-  copyContent: (collaborativeDoc: CollaborativeDoc, sourceField: string, targetField: string) => Promise<void>
+    document: CollaborativeDoc,
+    content: Record<string, Markup>,
+    snapshot?: DocumentSnapshotParams
+  ) => Promise<CollaborativeDoc>
+  copyContent: (
+    document: CollaborativeDoc,
+    sourceField: string,
+    targetField: string,
+    snapshot?: DocumentSnapshotParams
+  ) => Promise<CollaborativeDoc>
 
   // document operations
   branch: (source: CollaborativeDoc, target: CollaborativeDoc) => Promise<void>
   remove: (collaborativeDoc: CollaborativeDoc) => Promise<void>
-  snapshot: (collaborativeDoc: CollaborativeDoc, params: CollaborativeDocSnapshotParams) => Promise<CollaborativeDoc>
+  snapshot: (collaborativeDoc: CollaborativeDoc, params: DocumentSnapshotParams) => Promise<CollaborativeDoc>
 }
 
 /** @public */
-export function getClient (
+export function getClient (workspaceId: WorkspaceId, token: string, collaboratorUrl: string): CollaboratorClient {
-  hierarchy: Hierarchy,
+  const url = collaboratorUrl.replaceAll('wss://', 'https://').replace('ws://', 'http://')
-  workspaceId: WorkspaceId,
+  return new CollaboratorClientImpl(workspaceId, token, url)
-  token: string,
-  collaboratorUrl: string
-): CollaboratorClient {
-  return new CollaboratorClientImpl(hierarchy, workspaceId, token, collaboratorUrl)
 }
 
 class CollaboratorClientImpl implements CollaboratorClient {
   constructor (
-    private readonly hierarchy: Hierarchy,
     private readonly workspace: WorkspaceId,
     private readonly token: string,
     private readonly collaboratorUrl: string
@@ -153,30 +157,43 @@ class CollaboratorClientImpl implements CollaboratorClient {
     return result
   }
 
-  async getContent (document: CollaborativeDoc, field: string): Promise<Markup> {
+  async getContent (document: CollaborativeDoc): Promise<Record<string, Markup>> {
     const workspace = this.workspace.name
 
     const documentId = formatMinioDocumentId(workspace, document)
-    const payload: GetContentRequest = { documentId, field }
+    const payload: GetContentRequest = { documentId }
     const res = (await this.rpc('getContent', payload)) as GetContentResponse
 
-    return res.html ?? ''
+    return res.content ?? {}
   }
 
-  async updateContent (document: CollaborativeDoc, field: string, value: Markup): Promise<void> {
+  async updateContent (
+    document: CollaborativeDoc,
+    content: Record<string, Markup>,
+    snapshot?: DocumentSnapshotParams
+  ): Promise<CollaborativeDoc> {
     const workspace = this.workspace.name
 
     const documentId = formatMinioDocumentId(workspace, document)
-    const payload: UpdateContentRequest = { documentId, field, html: value }
+    const payload: UpdateContentRequest = { documentId, content, snapshot }
     await this.rpc('updateContent', payload)
+
+    return snapshot !== undefined ? collaborativeDocWithLastVersion(document, snapshot.versionId) : document
   }
 
-  async copyContent (document: CollaborativeDoc, sourceField: string, targetField: string): Promise<void> {
+  async copyContent (
+    document: CollaborativeDoc,
+    sourceField: string,
+    targetField: string,
+    snapshot?: DocumentSnapshotParams
+  ): Promise<CollaborativeDoc> {
     const workspace = this.workspace.name
 
     const documentId = formatMinioDocumentId(workspace, document)
-    const payload: CopyContentRequest = { documentId, sourceField, targetField }
+    const payload: CopyContentRequest = { documentId, sourceField, targetField, snapshot }
     await this.rpc('copyContent', payload)
+
+    return snapshot !== undefined ? collaborativeDocWithLastVersion(document, snapshot.versionId) : document
   }
 
   async branch (source: CollaborativeDoc, target: CollaborativeDoc): Promise<void> {
@@ -198,11 +215,11 @@ class CollaboratorClientImpl implements CollaboratorClient {
     await this.rpc('removeDocument', payload)
   }
 
-  async snapshot (document: CollaborativeDoc, params: CollaborativeDocSnapshotParams): Promise<CollaborativeDoc> {
+  async snapshot (document: CollaborativeDoc, snapshot: DocumentSnapshotParams): Promise<CollaborativeDoc> {
     const workspace = this.workspace.name
 
     const documentId = formatMinioDocumentId(workspace, document)
-    const payload: TakeSnapshotRequest = { documentId, ...params }
+    const payload: TakeSnapshotRequest = { documentId, snapshot }
     const res = (await this.rpc('takeSnapshot', payload)) as TakeSnapshotResponse
 
     return collaborativeDocWithVersion(document, res.versionId)

packages/collaborator-client/src/utils.ts

@@ -17,7 +17,6 @@ import {
   Class,
   CollaborativeDoc,
   Doc,
-  Domain,
   Ref,
   collaborativeDocChain,
   collaborativeDocFormat,
@@ -80,24 +79,21 @@ export function parseDocumentId (documentId: DocumentId): {
 
 /** @public */
 export function formatPlatformDocumentId (
-  objectDomain: Domain,
   objectClass: Ref<Class<Doc>>,
   objectId: Ref<Doc>,
   objectAttr: string
 ): PlatformDocumentId {
-  return `${objectDomain}/${objectClass}/${objectId}/${objectAttr}` as PlatformDocumentId
+  return `${objectClass}/${objectId}/${objectAttr}` as PlatformDocumentId
 }
 
 /** @public */
 export function parsePlatformDocumentId (platformDocumentId: PlatformDocumentId): {
-  objectDomain: Domain
   objectClass: Ref<Class<Doc>>
   objectId: Ref<Doc>
   objectAttr: string
 } {
-  const [objectDomain, objectClass, objectId, objectAttr] = platformDocumentId.split('/')
+  const [objectClass, objectId, objectAttr] = platformDocumentId.split('/')
   return {
-    objectDomain: objectDomain as Domain,
     objectClass: objectClass as Ref<Class<Doc>>,
     objectId: objectId as Ref<Doc>,
     objectAttr

packages/core/src/collaboration.ts

@@ -44,16 +44,16 @@ export type CollaborativeDocVersion = string | typeof CollaborativeDocVersionHea
 export const CollaborativeDocVersionHead = 'HEAD'
 
 /** @public */
-export function getCollaborativeDocId (objectId: Ref<Doc>, objectAttr?: string | undefined): string {
+export function makeCollaborativeDoc (
-  return objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}`
+  objectId: Ref<Doc>,
-}
+  objectAttr?: string | undefined,
-
+  versionId?: string | undefined
-/** @public */
+): CollaborativeDoc {
-export function getCollaborativeDoc (documentId: string): CollaborativeDoc {
+  const storageDocumentId = objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}`
   return collaborativeDocFormat({
-    documentId,
+    documentId: storageDocumentId,
     versionId: CollaborativeDocVersionHead,
-    lastVersionId: CollaborativeDocVersionHead
+    lastVersionId: versionId ?? '0'
   })
 }
 

packages/core/src/component.ts

@@ -34,7 +34,6 @@ import type {
   Hyperlink,
   IndexingConfiguration,
   Interface,
-  Markup,
   MigrationState,
   Obj,
   Permission,
@@ -122,7 +121,6 @@ export default plugin(coreId, {
     TypeDate: '' as Ref<Class<Type<Timestamp | Date>>>,
     TypeCollaborativeDoc: '' as Ref<Class<Type<CollaborativeDoc>>>,
     TypeCollaborativeDocVersion: '' as Ref<Class<Type<CollaborativeDoc>>>,
-    TypeCollaborativeMarkup: '' as Ref<Class<Type<Markup>>>,
     RefTo: '' as Ref<Class<RefTo<Doc>>>,
     ArrOf: '' as Ref<Class<ArrOf<Doc>>>,
     Enum: '' as Ref<Class<Enum>>,

packages/model/src/dsl.ts

@@ -265,21 +265,25 @@ function _generateTx (tx: ClassTxes): Tx[] {
     [ClassifierKind.INTERFACE]: core.class.Interface,
     [ClassifierKind.MIXIN]: core.class.Mixin
   }
-  const createTx = txFactory.createTxCreateDoc<Doc>(
+  const createTx = txFactory.createTxCreateDoc<Classifier>(
     _cl[tx.kind],
     core.space.Model,
     {
       ...(tx.domain !== undefined ? { domain: tx.domain } : {}),
       kind: tx.kind,
+      label: tx.label,
+      icon: tx.icon,
       ...(tx.kind === ClassifierKind.INTERFACE
         ? { extends: tx.implements }
         : { extends: tx.extends, implements: tx.implements }),
-      label: tx.label,
+      ...(tx.kind === ClassifierKind.INTERFACE
-      icon: tx.icon,
+        ? { extends: tx.implements }
-      shortLabel: tx.shortLabel,
+        : {
-      sortingKey: tx.sortingKey,
+            shortLabel: tx.shortLabel,
-      filteringKey: tx.filteringKey,
+            sortingKey: tx.sortingKey,
-      pluralLabel: tx.pluralLabel
+            filteringKey: tx.filteringKey,
+            pluralLabel: tx.pluralLabel
+          })
     },
     objectId
   )
@@ -412,13 +416,6 @@ export function TypeMarkup (): Type<Markup> {
   return { _class: core.class.TypeMarkup, label: core.string.Markup }
 }
 
-/**
- * @public
- */
-export function TypeCollaborativeMarkup (): Type<Markup> {
-  return { _class: core.class.TypeCollaborativeMarkup, label: core.string.Collaborative }
-}
-
 /**
  * @public
  */

packages/presentation/src/collaborator.ts

@@ -13,34 +13,36 @@
 // limitations under the License.
 //
 
-import { type CollaboratorClient, getClient as getCollaborator } from '@hcengineering/collaborator-client'
+import {
+  type CollaboratorClient,
+  getClient as getCollaborator,
+  type DocumentSnapshotParams
+} from '@hcengineering/collaborator-client'
 import { type CollaborativeDoc, type Markup, getCurrentAccount, getWorkspaceId } from '@hcengineering/core'
 import { getMetadata } from '@hcengineering/platform'
 import { getCurrentLocation } from '@hcengineering/ui'
 
-import { getClient } from '.'
 import presentation from './plugin'
 
 /** @public */
 export function getCollaboratorClient (): CollaboratorClient {
   const workspaceId = getWorkspaceId(getCurrentLocation().path[1] ?? '')
-  const hierarchy = getClient().getHierarchy()
   const token = getMetadata(presentation.metadata.Token) ?? ''
-  const collaboratorURL = getMetadata(presentation.metadata.CollaboratorApiUrl) ?? ''
+  const collaboratorURL = getMetadata(presentation.metadata.CollaboratorUrl) ?? ''
 
-  return getCollaborator(hierarchy, workspaceId, token, collaboratorURL)
+  return getCollaborator(workspaceId, token, collaboratorURL)
 }
 
 /** @public */
-export async function getMarkup (collaborativeDoc: CollaborativeDoc, field: string): Promise<Markup> {
+export async function getMarkup (collaborativeDoc: CollaborativeDoc): Promise<Record<string, Markup>> {
   const client = getCollaboratorClient()
-  return await client.getContent(collaborativeDoc, field)
+  return await client.getContent(collaborativeDoc)
 }
 
 /** @public */
-export async function updateMarkup (collaborativeDoc: CollaborativeDoc, field: string, value: Markup): Promise<void> {
+export async function updateMarkup (collaborativeDoc: CollaborativeDoc, content: Record<string, Markup>): Promise<void> {
   const client = getCollaboratorClient()
-  await client.updateContent(collaborativeDoc, field, value)
+  await client.updateContent(collaborativeDoc, content)
 }
 
 /** @public */
@@ -60,12 +62,10 @@ export async function copyDocument (source: CollaborativeDoc, target: Collaborat
 }
 
 /** @public */
-export async function takeSnapshot (
+export async function takeSnapshot (collaborativeDoc: CollaborativeDoc, versionName: string): Promise<CollaborativeDoc> {
-  collaborativeDoc: CollaborativeDoc,
-  snapshotName: string
-): Promise<CollaborativeDoc> {
   const client = getCollaboratorClient()
   const createdBy = getCurrentAccount()._id
 
-  return await client.snapshot(collaborativeDoc, { createdBy, snapshotName })
+  const snapshot: DocumentSnapshotParams = { createdBy, versionId: `${Date.now()}`, versionName }
+  return await client.snapshot(collaborativeDoc, snapshot)
 }

packages/presentation/src/plugin.ts

@@ -132,7 +132,6 @@ export default plugin(presentationId, {
     UploadURL: '' as Metadata<string>,
     FilesURL: '' as Metadata<string>,
     CollaboratorUrl: '' as Metadata<string>,
-    CollaboratorApiUrl: '' as Metadata<string>,
     Token: '' as Metadata<string>,
     Endpoint: '' as Metadata<string>,
     Workspace: '' as Metadata<string>,

packages/presentation/src/utils.ts

@@ -533,9 +533,6 @@ export function getAttributePresenterClass (
   if (hierarchy.isDerived(attrClass, core.class.TypeMarkup)) {
     category = 'inplace'
   }
-  if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeMarkup)) {
-    category = 'inplace'
-  }
   if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeDoc)) {
     category = 'inplace'
   }

packages/text/src/ydoc.ts

@@ -64,7 +64,7 @@ const defaultSchema = getSchema(defaultExtensions)
  * @public
  */
 export function yDocToNode (ydoc: YDoc, field?: string, schema?: Schema, extensions?: Extensions): Node {
-  schema ??= extensions === undefined ? defaultSchema : getSchema(extensions ?? defaultExtensions)
+  schema ??= getSchema(extensions ?? defaultExtensions)
 
   try {
     const body = yDocToProsemirrorJSON(ydoc, field)

plugins/activity-resources/src/activityMessagesUtils.ts

@@ -59,7 +59,6 @@ const valueTypes: ReadonlyArray<Ref<Class<Doc>>> = [
   core.class.TypeDate,
   core.class.TypeFileSize,
   core.class.TypeMarkup,
-  core.class.TypeCollaborativeMarkup,
   core.class.TypeHyperlink
 ]
 

plugins/activity-resources/src/utils.ts

@@ -144,7 +144,6 @@ export function getIsTextType (attributeModel?: AttributeModel): boolean {
 
   return (
     attributeModel.attribute?.type?._class === core.class.TypeMarkup ||
-    attributeModel.attribute?.type?._class === core.class.TypeCollaborativeMarkup ||
     attributeModel.attribute?.type?._class === core.class.TypeCollaborativeDoc
   )
 }

plugins/bitrix/src/hr.ts

@@ -1,5 +1,16 @@
 import { Organization } from '@hcengineering/contact'
-import core, { Account, Client, Data, Doc, Ref, SortingOrder, Status, TxOperations } from '@hcengineering/core'
+import core, {
+  Account,
+  Client,
+  Data,
+  Doc,
+  Ref,
+  SortingOrder,
+  Status,
+  TxOperations,
+  generateId,
+  makeCollaborativeDoc
+} from '@hcengineering/core'
 import recruit, { Applicant, Vacancy } from '@hcengineering/recruit'
 import task, { ProjectType, makeRank } from '@hcengineering/task'
 
@@ -23,17 +34,25 @@ export async function createVacancy (
 
   const incResult = await client.update(sequence, { $inc: { sequence: 1 } }, true)
 
-  const id = await client.createDoc(recruit.class.Vacancy, core.space.Space, {
+  const id: Ref<Vacancy> = generateId()
-    name,
+  await client.createDoc(
-    description: type.shortDescription ?? '',
+    recruit.class.Vacancy,
-    fullDescription: type.description,
+    core.space.Space,
-    private: false,
+    {
-    archived: false,
+      name,
-    company,
+      description: type.shortDescription ?? '',
-    number: (incResult as any).object.sequence,
+      fullDescription: makeCollaborativeDoc(id, 'fullDescription'),
-    members: [],
+      private: false,
-    type: typeId
+      archived: false,
-  })
+      company,
+      number: (incResult as any).object.sequence,
+      members: [],
+      type: typeId
+    },
+    id
+  )
+
+  // TODO type.description
 
   return id
 }

plugins/contact-resources/package.json

@@ -52,6 +52,7 @@
     "@hcengineering/presentation": "^0.6.3",
     "@hcengineering/setting": "^0.6.17",
     "@hcengineering/templates": "^0.6.11",
+    "@hcengineering/text": "^0.6.5",
     "@hcengineering/text-editor": "^0.6.0",
     "@hcengineering/text-editor-resources": "^0.6.0",
     "@hcengineering/theme": "^0.6.5",

plugins/contact-resources/src/components/CreateOrganization.svelte

@@ -13,13 +13,22 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import { Channel, ContactEvents, findContacts, Organization } from '@hcengineering/contact'
+  import { Attachment } from '@hcengineering/attachment'
-  import core, { AttachedData, fillDefaults, generateId, Ref, TxOperations, WithLookup } from '@hcengineering/core'
+  import { AttachmentPresenter, AttachmentStyledBox } from '@hcengineering/attachment-resources'
-  import { Card, getClient, InlineAttributeBar } from '@hcengineering/presentation'
+  import { Channel, ContactEvents, Organization, findContacts } from '@hcengineering/contact'
+  import core, {
+    AttachedData,
+    fillDefaults,
+    generateId,
+    makeCollaborativeDoc,
+    Ref,
+    TxOperations,
+    WithLookup
+  } from '@hcengineering/core'
+  import { Card, getClient, InlineAttributeBar, updateMarkup } from '@hcengineering/presentation'
+  import { EmptyMarkup } from '@hcengineering/text'
   import { Button, createFocusManager, EditBox, FocusHandler, IconAttachment, IconInfo, Label } from '@hcengineering/ui'
   import { createEventDispatcher } from 'svelte'
-  import { AttachmentPresenter, AttachmentStyledBox } from '@hcengineering/attachment-resources'
-  import { Attachment } from '@hcengineering/attachment'
 
   import contact from '../plugin'
   import ChannelsDropdown from './ChannelsDropdown.svelte'
@@ -37,7 +46,7 @@
 
   const object: Organization = {
     name: '',
-    description: '',
+    description: makeCollaborativeDoc(id, 'description'),
     attachments: 0
   } as unknown as Organization
 
@@ -45,9 +54,12 @@
   const client = getClient()
   const hierarchy = client.getHierarchy()
 
+  let description = EmptyMarkup
+
   fillDefaults(hierarchy, object, contact.class.Organization)
 
   async function createOrganization (): Promise<void> {
+    await updateMarkup(object.description, { description })
     await client.createDoc(contact.class.Organization, contact.space.Contacts, object, id)
     await descriptionBox.createAttachments(id)
 
@@ -119,7 +131,7 @@
     space={contact.space.Contacts}
     alwaysEdit
     showButtons={false}
-    bind:content={object.description}
+    bind:content={description}
     placeholder={core.string.Description}
     kind="indented"
     isScrollable={false}

plugins/contact/src/index.ts

@@ -18,6 +18,7 @@ import {
   Account,
   AttachedDoc,
   Class,
+  CollaborativeDoc,
   Doc,
   Ref,
   Space,
@@ -136,7 +137,7 @@ export interface Member extends AttachedDoc {
  */
 export interface Organization extends Contact {
   members: number
-  description?: string
+  description: CollaborativeDoc
 }
 
 /**

plugins/controlled-documents-resources/src/components/CreateDocument.svelte

@@ -21,7 +21,7 @@
     AttachedData,
     Class,
     generateId,
-    getCollaborativeDoc,
+    makeCollaborativeDoc,
     getCurrentAccount,
     Mixin,
     Ref,
@@ -78,7 +78,7 @@
     approvers: [],
     coAuthors: [],
     changeControl: '' as Ref<ChangeControl>,
-    content: getCollaborativeDoc(generateId())
+    content: makeCollaborativeDoc(generateId())
   }
 
   let templateId: Ref<DocumentTemplate> | undefined = initTemplateId

plugins/controlled-documents-resources/src/components/create-doc/QmsDocumentWizard.svelte

@@ -18,11 +18,11 @@
   import {
     generateId,
     getCurrentAccount,
+    makeCollaborativeDoc,
     type AttachedData,
     type Class,
     type Data,
-    type Ref,
+    type Ref
-    getCollaborativeDoc
   } from '@hcengineering/core'
   import { MessageBox, getClient } from '@hcengineering/presentation'
   import {
@@ -122,7 +122,7 @@
     state: DocumentState.Draft,
     snapshots: 0,
     changeControl: ccRecordId,
-    content: getCollaborativeDoc(generateId()),
+    content: makeCollaborativeDoc(generateId()),
 
     requests: 0,
     reviewers: [],

plugins/controlled-documents-resources/src/components/create-doc/QmsTemplateWizard.svelte

@@ -32,8 +32,8 @@
     type Ref,
     type Mixin,
     generateId,
-    getCollaborativeDoc,
+    getCurrentAccount,
-    getCurrentAccount
+    makeCollaborativeDoc
   } from '@hcengineering/core'
   import { MessageBox, getClient } from '@hcengineering/presentation'
   import {
@@ -118,7 +118,7 @@
     state: DocumentState.Draft,
     snapshots: 0,
     changeControl: ccRecordId,
-    content: getCollaborativeDoc(generateId()),
+    content: makeCollaborativeDoc(generateId()),
 
     requests: 0,
     reviewers: [],

plugins/controlled-documents/src/docutils.ts

@@ -18,11 +18,12 @@ import {
   type AttachedData,
   type Class,
   type CollaborativeDoc,
+  type Doc,
   type Ref,
   type TxOperations,
   Mixin,
   generateId,
-  getCollaborativeDoc
+  makeCollaborativeDoc
 } from '@hcengineering/core'
 import {
   type Document,
@@ -33,6 +34,7 @@ import {
   type DocumentMeta,
   type Project,
   DocumentState,
+  HierarchyDocument,
   ProjectDocument
 } from './types'
 
@@ -261,7 +263,7 @@ export async function createDocumentTemplate (
     }
   )
 
-  await ops.addCollection(
+  await ops.addCollection<DocumentMeta, HierarchyDocument>(
     _class,
     space,
     metaId,
@@ -301,5 +303,7 @@ export function getCollaborativeDocForDocument (
     prefix = prefix.substring(0, prefix.length - 1)
   }
 
-  return getCollaborativeDoc(`${prefix}-${seqNumber}-${major}.${minor}${next ? '.next' : ''}-` + generateId())
+  return makeCollaborativeDoc(
+    (`${prefix}-${seqNumber}-${major}.${minor}${next ? '.next' : ''}-` + generateId()) as Ref<Doc>
+  )
 }

plugins/document-resources/src/components/EditDoc.svelte

@@ -308,6 +308,7 @@
                   size: 'large',
                   fill: doc.color !== undefined ? getPlatformColorDef(doc.color, $themeStore.dark).icon : 'currentColor'
                 }}
+            disabled={readonly}
             on:click={chooseIcon}
           />
         </div>

plugins/document-resources/src/utils.ts

@@ -13,15 +13,8 @@
 // limitations under the License.
 //
 
-import {
+import { type AttachedData, type Client, type Ref, type TxOperations, makeCollaborativeDoc } from '@hcengineering/core'
-  getCollaborativeDoc,
+import { type Document, type Teamspace, documentId } from '@hcengineering/document'
-  getCollaborativeDocId,
-  type AttachedData,
-  type Client,
-  type Ref,
-  type TxOperations
-} from '@hcengineering/core'
-import { documentId, type Document, type Teamspace } from '@hcengineering/document'
 import { getMetadata, translate } from '@hcengineering/platform'
 import presentation, { getClient } from '@hcengineering/presentation'
 import { getCurrentResolvedLocation, getPanelURI, type Location, type ResolvedLocation } from '@hcengineering/ui'
@@ -39,11 +32,10 @@ export async function createEmptyDocument (
   data: Partial<Pick<AttachedData<Document>, 'name' | 'icon' | 'color'>> = {}
 ): Promise<void> {
   const name = await translate(document.string.Untitled, {})
-  const collaborativeDocId = getCollaborativeDocId(id, 'content')
 
   const object: AttachedData<Document> = {
     name,
-    content: getCollaborativeDoc(collaborativeDocId),
+    content: makeCollaborativeDoc(id, 'content'),
     attachments: 0,
     children: 0,
     embeddings: 0,

plugins/lead-resources/src/components/CreateCustomer.svelte

@@ -16,9 +16,20 @@
   import { AvatarType, Channel, combineName, Contact, findContacts } from '@hcengineering/contact'
   import { ChannelsDropdown, EditableAvatar, PersonPresenter } from '@hcengineering/contact-resources'
   import contact from '@hcengineering/contact-resources/src/plugin'
-  import { AttachedData, Class, Data, Doc, generateId, MixinData, Ref, WithLookup } from '@hcengineering/core'
+  import {
+    AttachedData,
+    Class,
+    Data,
+    Doc,
+    MixinData,
+    Ref,
+    WithLookup,
+    generateId,
+    makeCollaborativeDoc
+  } from '@hcengineering/core'
   import { Customer, LeadEvents } from '@hcengineering/lead'
-  import { Card, getClient, InlineAttributeBar } from '@hcengineering/presentation'
+  import { Card, getClient, InlineAttributeBar, updateMarkup } from '@hcengineering/presentation'
+  import { EmptyMarkup, StyledTextBox } from '@hcengineering/text-editor-resources'
   import {
     Button,
     createFocusManager,
@@ -36,6 +47,7 @@
 
   let firstName = ''
   let lastName = ''
+  let description = EmptyMarkup
 
   export function canClose (): boolean {
     return firstName === '' && lastName === ''
@@ -58,9 +70,9 @@
     return targetClass === contact.class.Person ? combineName(firstName.trim(), lastName.trim()) : objectName
   }
 
-  async function createCustomer () {
+  async function createCustomer (): Promise<void> {
     const candidate: Data<Contact> = {
-      name: formatName(targetClass._id, firstName, lastName, object.name),
+      name,
       city: object.city,
       avatarType: AvatarType.COLOR
     }
@@ -71,9 +83,11 @@
       candidate.avatarProps = info.avatarProps
     }
     const candidateData: MixinData<Contact, Customer> = {
-      description: object.description
+      description: makeCollaborativeDoc(customerId, 'description')
     }
 
+    await updateMarkup(candidateData.description, { description })
+
     const id = await client.createDoc(targetClass._id, contact.space.Contacts, { ...candidate, ...object }, customerId)
     await client.createMixin(
       id as Ref<Contact>,
@@ -131,19 +145,18 @@
       }
     )
   }
-  $: canSave = formatName(targetClass._id, firstName, lastName, object.name).length > 0
+  $: name = formatName(targetClass._id, firstName, lastName, object.name)
+  $: canSave = name.trim().length > 0
 
   const manager = createFocusManager()
 
   let matches: WithLookup<Contact>[] = []
   let matchedChannels: AttachedData<Channel>[] = []
   $: if (targetClass !== undefined) {
-    findContacts(client, targetClass._id, formatName(targetClass._id, firstName, lastName, object.name), channels).then(
+    void findContacts(client, targetClass._id, name, channels).then((p) => {
-      (p) => {
+      matches = p.contacts
-        matches = p.contacts
+      matchedChannels = p.channels
-        matchedChannels = p.channels
+    })
-      }
-    )
   }
 </script>
 
@@ -185,12 +198,16 @@
             focusIndex={3}
           />
         </div>
-        <EditBox
+        <div class="mt-1">
-          placeholder={lead.string.IssueDescriptionPlaceholder}
+          <StyledTextBox
-          bind:value={object.description}
+            bind:content={description}
-          kind={'small-style'}
+            placeholder={lead.string.IssueDescriptionPlaceholder}
-          focusIndex={4}
+            kind={'normal'}
-        />
+            alwaysEdit={true}
+            showButtons={false}
+            focusIndex={4}
+          />
+        </div>
       </div>
       <div class="ml-4 flex">
         <EditableAvatar

plugins/lead/src/index.ts

@@ -15,7 +15,7 @@
 //
 
 import type { Contact } from '@hcengineering/contact'
-import type { Attribute, Class, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core'
+import type { Attribute, Class, CollaborativeDoc, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core'
 import { Mixin } from '@hcengineering/core'
 import type { Asset, IntlString, Plugin } from '@hcengineering/platform'
 import { plugin } from '@hcengineering/platform'
@@ -36,7 +36,7 @@ export interface Funnel extends Project {
 export interface Customer extends Contact {
   leads?: number
 
-  description: string
+  description: CollaborativeDoc
 }
 
 /**

plugins/recruit-resources/src/components/CreateVacancy.svelte

@@ -18,6 +18,7 @@
   import { AccountArrayEditor, UserBox } from '@hcengineering/contact-resources'
   import core, {
     Account,
+    AttachedData,
     Data,
     Ref,
     Role,
@@ -25,10 +26,18 @@
     SortingOrder,
     fillDefaults,
     generateId,
-    getCurrentAccount
+    getCurrentAccount,
+    makeCollaborativeDoc
   } from '@hcengineering/core'
   import { getEmbeddedLabel } from '@hcengineering/platform'
-  import { Card, InlineAttributeBar, MessageBox, createQuery, getClient } from '@hcengineering/presentation'
+  import {
+    Card,
+    InlineAttributeBar,
+    MessageBox,
+    createQuery,
+    getClient,
+    updateMarkup
+  } from '@hcengineering/presentation'
   import { RecruitEvents, Vacancy, Vacancy as VacancyClass } from '@hcengineering/recruit'
   import tags from '@hcengineering/tags'
   import task, { ProjectType, makeRank } from '@hcengineering/task'
@@ -88,7 +97,7 @@
     attachments: 0,
     comments: 0,
     company: '' as Ref<Organization>,
-    fullDescription: '',
+    fullDescription: makeCollaborativeDoc(objectId, 'fullDescription'),
     location: '',
     type: typeId as Ref<ProjectType>
   }
@@ -103,7 +112,7 @@
   $: typeId &&
     templateQ.query(task.class.ProjectType, { _id: typeId }, (result) => {
       const { _class, _id, description, targetClass, ...templateData } = result[0]
-      vacancyData = { ...(templateData as unknown as Data<VacancyClass>), fullDescription: description }
+      vacancyData = { ...(templateData as unknown as Data<VacancyClass>) }
       if (appliedTemplateId !== typeId) {
         fullDescription = description ?? ''
         appliedTemplateId = typeId
@@ -172,10 +181,11 @@
     }
     const number = (incResult as any).object.sequence
 
+    const resId: Ref<Issue> = generateId()
     const identifier = `${project?.identifier}-${number}`
-    const resId = await client.addCollection(tracker.class.Issue, space, parent, tracker.class.Issue, 'subIssues', {
+    const data: AttachedData<Issue> = {
       title: template.title + ` (${name})`,
-      description: template.description,
+      description: makeCollaborativeDoc(resId, 'description'),
       assignee: template.assignee,
       component: template.component,
       milestone: template.milestone,
@@ -195,7 +205,10 @@
       childInfo: [],
       kind: taskType._id,
       identifier
-    })
+    }
+
+    await updateMarkup(data.description, { description: template.description })
+    await client.addCollection(tracker.class.Issue, space, parent, tracker.class.Issue, 'subIssues', data, resId)
     if ((template.labels?.length ?? 0) > 0) {
       const tagElements = await client.findAll(tags.class.TagElement, { _id: { $in: template.labels } })
       for (const label of tagElements) {
@@ -224,7 +237,7 @@
       ...vacancyData,
       name,
       description: template?.shortDescription ?? '',
-      fullDescription,
+      fullDescription: makeCollaborativeDoc(objectId, 'fullDescription'),
       private: false,
       archived: false,
       number: (incResult as any).object.sequence,
@@ -235,6 +248,8 @@
       type: typeId
     }
 
+    await updateMarkup(data.fullDescription, { fullDescription })
+
     const id = await client.createDoc(recruit.class.Vacancy, core.space.Space, data, objectId)
 
     Analytics.handleEvent(RecruitEvents.VacancyCreated, {

plugins/recruit/src/types.ts

@@ -15,13 +15,13 @@
 
 import { Event } from '@hcengineering/calendar'
 import type { Channel, Organization, Person } from '@hcengineering/contact'
-import type { AttachedData, AttachedDoc, Markup, Ref, Status, Timestamp } from '@hcengineering/core'
+import type { AttachedData, AttachedDoc, CollaborativeDoc, Markup, Ref, Status, Timestamp } from '@hcengineering/core'
 import { TagReference } from '@hcengineering/tags'
 import type { Project, Task } from '@hcengineering/task'
 
 /** @public */
 export interface Vacancy extends Project {
-  fullDescription?: string
+  fullDescription: CollaborativeDoc
   attachments?: number
   dueTo?: Timestamp
   location?: string

plugins/text-editor-resources/src/components/CollaborativeAttributeBox.svelte

@@ -13,7 +13,7 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import core, { CollaborativeDoc, Doc, getCollaborativeDoc, getCollaborativeDocId } from '@hcengineering/core'
+  import { Doc } from '@hcengineering/core'
   import { IntlString } from '@hcengineering/platform'
   import { KeyedAttribute, getAttribute, getClient } from '@hcengineering/presentation'
   import { AnySvelteComponent, registerFocus } from '@hcengineering/ui'
@@ -47,20 +47,7 @@
 
   let editor: CollaborativeTextEditor
 
-  $: collaborativeDoc = getCollaborativeDocFromAttribute(object, key)
+  $: collaborativeDoc = getAttribute(getClient(), object, key)
-
-  function getCollaborativeDocFromAttribute (object: Doc, key: KeyedAttribute): CollaborativeDoc {
-    const value = getAttribute(getClient(), object, key)
-    if (key.attr.type._class === core.class.TypeCollaborativeDoc) {
-      return value as CollaborativeDoc
-    } else if (key.attr.type._class === core.class.TypeCollaborativeDocVersion) {
-      return value as CollaborativeDoc
-    } else {
-      // TODO Remove this when we migrate to minio
-      const collaborativeDocId = getCollaborativeDocId(object._id, key.key)
-      return getCollaborativeDoc(collaborativeDocId)
-    }
-  }
 
   // Focusable control with index
   let canBlur = true
@@ -104,8 +91,8 @@
 <CollaborativeTextEditor
   bind:this={editor}
   {collaborativeDoc}
-  objectClass={object._class}
   objectId={object._id}
+  objectClass={key.attr.attributeOf}
   objectSpace={object.space}
   objectAttr={key.key}
   {user}

plugins/text-editor-resources/src/components/CollaborativeTextEditor.svelte

@@ -71,10 +71,10 @@
   export let initialCollaborativeDoc: CollaborativeDoc | undefined = undefined
   export let field: string
 
-  export let objectClass: Ref<Class<Doc>> | undefined
+  export let objectClass: Ref<Class<Doc>> | undefined = undefined
-  export let objectId: Ref<Doc> | undefined
+  export let objectId: Ref<Doc> | undefined = undefined
   export let objectSpace: Ref<Space> | undefined = undefined
-  export let objectAttr: string | undefined
+  export let objectAttr: string | undefined = undefined
 
   export let user: CollaborationUser
   export let userComponent: AnySvelteComponent | undefined = undefined

plugins/text-editor-resources/src/provider/utils.ts

@@ -21,7 +21,6 @@ import {
   formatPlatformDocumentId as origFormatPlatformDocumentId
 } from '@hcengineering/collaborator-client'
 import { getCurrentLocation } from '@hcengineering/ui'
-import { getClient } from '@hcengineering/presentation'
 
 function getWorkspace (): string {
   return getCurrentLocation().path[1] ?? ''
@@ -37,6 +36,5 @@ export function formatPlatformDocumentId (
   objectId: Ref<Doc>,
   objectAttr: string
 ): PlatformDocumentId {
-  const objectDomain = getClient().getHierarchy().getDomain(objectClass)
+  return origFormatPlatformDocumentId(objectClass, objectId, objectAttr)
-  return origFormatPlatformDocumentId(objectDomain, objectClass, objectId, objectAttr)
 }

plugins/tracker-resources/src/components/CreateIssue.svelte

@@ -27,6 +27,7 @@
     SortingOrder,
     fillDefaults,
     generateId,
+    makeCollaborativeDoc,
     toIdMap
   } from '@hcengineering/core'
   import { getResource, translate } from '@hcengineering/platform'
@@ -41,7 +42,9 @@
     MultipleDraftController,
     SpaceSelector,
     createQuery,
-    getClient
+    getClient,
+    getMarkup,
+    updateMarkup
   } from '@hcengineering/presentation'
   import tags, { TagElement, TagReference } from '@hcengineering/tags'
   import { TaskType, makeRank } from '@hcengineering/task'
@@ -190,7 +193,6 @@
     if (originalIssue !== undefined && !ignoreOriginal) {
       const res: IssueDraft = {
         ...base,
-        description: originalIssue.description,
         status: originalIssue.status,
         priority: originalIssue.priority,
         component: originalIssue.component,
@@ -200,6 +202,9 @@
         parentIssue: originalIssue.parents[0]?.parentId,
         title: `${originalIssue.title} (copy)`
       }
+      void getMarkup(originalIssue.description).then((res) => {
+        object.description = res.description
+      })
       void client.findAll(tags.class.TagReference, { attachedTo: originalIssue._id }).then((p) => {
         object.labels = p
       })
@@ -471,7 +476,7 @@
 
       const value: DocData<Issue> = {
         title: getTitle(object.title),
-        description: object.description,
+        description: makeCollaborativeDoc(_id, 'description'),
         assignee: object.assignee,
         component: object.component,
         milestone: object.milestone,
@@ -504,6 +509,8 @@
         identifier
       }
 
+      await updateMarkup(value.description, { description: object.description })
+
       await docCreateManager.commit(operations, _id, currentProject, value, 'pre')
 
       await operations.addCollection(

plugins/tracker-resources/src/components/SubIssues.svelte

@@ -14,8 +14,8 @@
  -->
 <script lang="ts">
   import attachment, { Attachment } from '@hcengineering/attachment'
-  import core, { AttachedData, Doc, Ref, SortingOrder } from '@hcengineering/core'
+  import core, { AttachedData, Doc, Ref, SortingOrder, makeCollaborativeDoc } from '@hcengineering/core'
-  import { DraftController, draftsStore, getClient, deleteFile } from '@hcengineering/presentation'
+  import { DraftController, draftsStore, getClient, deleteFile, updateMarkup } from '@hcengineering/presentation'
   import tags from '@hcengineering/tags'
   import { makeRank } from '@hcengineering/task'
   import { Component, Issue, IssueDraft, IssueParentInfo, Milestone, Project } from '@hcengineering/tracker'
@@ -71,7 +71,7 @@
       const childId = subIssue._id
       const cvalue: AttachedData<Issue> = {
         title: subIssue.title.trim(),
-        description: subIssue.description,
+        description: makeCollaborativeDoc(childId, 'description'),
         assignee: subIssue.assignee,
         component: subIssue.component,
         milestone: subIssue.milestone,
@@ -92,6 +92,7 @@
         kind: subIssue.kind,
         identifier: `${project.identifier}-${number}`
       }
+      await updateMarkup(cvalue.description, { description: subIssue.description })
       await client.addCollection(
         tracker.class.Issue,
         project._id,

plugins/tracker-resources/src/components/issues/IssuePreview.svelte

@@ -15,11 +15,13 @@
 <script lang="ts">
   import attachment from '@hcengineering/attachment'
   import { AttachmentDocList } from '@hcengineering/attachment-resources'
+  import { ChatMessagePopup } from '@hcengineering/chunter-resources'
   import { Ref } from '@hcengineering/core'
-  import { IconForward, MessageViewer, createQuery, getClient } from '@hcengineering/presentation'
+  import { IconForward, createQuery, getClient } from '@hcengineering/presentation'
+  import { CollaborativeTextEditor } from '@hcengineering/text-editor-resources'
   import { Issue } from '@hcengineering/tracker'
   import { Label, Scroller, resizeObserver } from '@hcengineering/ui'
-  import { ChatMessagePopup } from '@hcengineering/chunter-resources'
+  import { getCollaborationUser } from '@hcengineering/view-resources'
 
   import tracker from '../../plugin'
   import AssigneeEditor from './AssigneeEditor.svelte'
@@ -34,6 +36,8 @@
   const client = getClient()
   const issueQuery = createQuery()
 
+  const user = getCollaborationUser()
+
   $: issueQuery.query(
     object._class,
     { _id: object._id },
@@ -91,7 +95,9 @@
     {#if issue.description}
       <div class="description-container" class:masked={cHeight > limit} style:max-height={`${limit}px`}>
         <div class="description-content" use:resizeObserver={(element) => (cHeight = element.clientHeight)}>
-          <MessageViewer message={issue.description} />
+          {#key issue._id}
+            <CollaborativeTextEditor collaborativeDoc={issue.description} field="description" {user} readonly />
+          {/key}
         </div>
       </div>
     {:else}

plugins/tracker-resources/src/components/templates/EditIssueTemplate.svelte

@@ -13,7 +13,7 @@
 // limitations under the License.
 -->
 <script lang="ts">
-  import { AttachmentStyleBoxCollabEditor } from '@hcengineering/attachment-resources'
+  import { AttachmentStyleBoxEditor } from '@hcengineering/attachment-resources'
   import { Class, Doc, Ref, WithLookup } from '@hcengineering/core'
   import notification from '@hcengineering/notification'
   import { Panel } from '@hcengineering/panel'
@@ -45,7 +45,7 @@
   let title = ''
   let innerWidth: number
 
-  let descriptionBox: AttachmentStyleBoxCollabEditor
+  let descriptionBox: AttachmentStyleBoxEditor
 
   const inboxClient = getResource(notification.function.GetInboxNotificationsClient).then((res) => res())
 
@@ -147,7 +147,7 @@
   >
     <EditBox bind:value={title} placeholder={tracker.string.IssueTitlePlaceholder} kind="large-style" on:blur={save} />
     <div class="w-full mt-6">
-      <AttachmentStyleBoxCollabEditor
+      <AttachmentStyleBoxEditor
         focusIndex={30}
         object={template}
         key={{ key: 'description', attr: descriptionKey }}

plugins/tracker/src/index.ts

@@ -18,6 +18,7 @@ import {
   AttachedDoc,
   Attribute,
   Class,
+  CollaborativeDoc,
   CollectionSize,
   Data,
   Doc,
@@ -181,7 +182,7 @@ export interface Milestone extends Doc {
 export interface Issue extends Task {
   attachedTo: Ref<Issue>
   title: string
-  description: Markup
+  description: CollaborativeDoc
   status: Ref<IssueStatus>
   priority: IssuePriority
 

plugins/workbench-resources/src/components/ServerManagerCollaboratorStatistics.svelte

@@ -8,8 +8,9 @@
   const token: string = getMetadata(presentation.metadata.Token) ?? ''
 
   async function fetchCollabStats (tick: number): Promise<void> {
-    const collaborator = getMetadata(presentation.metadata.CollaboratorApiUrl)
+    const collaboratorUrl = getMetadata(presentation.metadata.CollaboratorUrl) ?? ''
-    await fetch(collaborator + `/api/v1/statistics?token=${token}`, {})
+    const collaboratorApiUrl = collaboratorUrl.replaceAll('wss://', 'https://').replace('ws://', 'http://')
+    await fetch(collaboratorApiUrl + `/api/v1/statistics?token=${token}`, {})
       .then(async (json) => {
         dataCollab = await json.json()
       })

pods/front/run.sh

@@ -4,7 +4,6 @@ export ACCOUNTS_URL=http://localhost:3333
 export UPLOAD_URL=http://localhost:3333/files
 export ELASTIC_URL=http://elastic:9200
 export COLLABORATOR_URL=ws://localhost:3078
-export COLLABORATOR_API_URL=http://localhost:3078
 export MINIO_ENDPOINT=minio
 export MINIO_ACCESS_KEY=minioadmin
 export MINIO_SECRET_KEY=minioadmin 

pods/server/src/__start.ts

@@ -54,6 +54,7 @@ setMetadata(notification.metadata.PushPublicKey, config.pushPublicKey)
 setMetadata(serverNotification.metadata.PushPrivateKey, config.pushPrivateKey)
 setMetadata(serverNotification.metadata.PushSubject, config.pushSubject)
 setMetadata(serverCore.metadata.ElasticIndexName, config.elasticIndexName)
+setMetadata(serverCore.metadata.ElasticIndexVersion, 'v1')
 setMetadata(serverTelegram.metadata.BotUrl, process.env.TELEGRAM_BOT_URL)
 setMetadata(serverAiBot.metadata.SupportWorkspaceId, process.env.SUPPORT_WORKSPACE)
 

qms-tests/docker-compose.yaml

@@ -80,7 +80,6 @@ services:
       - REKONI_URL=http://rekoni:4005
       - TELEGRAM_URL=http://localhost:8086
       - COLLABORATOR_URL=ws://localhost:3079
-      - COLLABORATOR_API_URL=http://localhost:3079
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - BRANDING_URL=http://localhost:8083/branding-test.json
   transactor:

server-plugins/activity-resources/src/references.ts

@@ -375,7 +375,7 @@ async function getMessageNotifyResult (
 }
 
 function isMarkupType (type: Ref<Class<Type<any>>>): boolean {
-  return type === core.class.TypeMarkup || type === core.class.TypeCollaborativeMarkup
+  return type === core.class.TypeMarkup
 }
 
 function isCollaborativeType (type: Ref<Class<Type<any>>>): boolean {

server-plugins/activity-resources/src/utils.ts

@@ -303,7 +303,6 @@ export async function getTxAttributesUpdates (
 
     if (
       hierarchy.isDerived(attrClass, core.class.TypeMarkup) ||
-      hierarchy.isDerived(attrClass, core.class.TypeCollaborativeMarkup) ||
       hierarchy.isDerived(attrClass, core.class.TypeCollaborativeDoc) ||
       mixin === notification.mixin.Collaborators
     ) {

server-plugins/collaboration-resources/src/index.ts

@@ -16,7 +16,7 @@
 import type { CollaborativeDoc, Doc, Tx, TxRemoveDoc } from '@hcengineering/core'
 import core, { TxProcessor } from '@hcengineering/core'
 import { removeCollaborativeDoc } from '@hcengineering/collaboration'
-import type { TriggerControl } from '@hcengineering/server-core'
+import { type TriggerControl } from '@hcengineering/server-core'
 
 /**
  * @public

server/collaborator/package.json

@@ -14,14 +14,6 @@
   "scripts": {
     "build": "compile",
     "build:watch": "compile",
-    "_phase:bundle": "rushx bundle",
-    "_phase:docker-build": "rushx docker:build",
-    "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle  --platform=node > bundle/bundle.js",
-    "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/collaborator",
-    "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging",
-    "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator",
-    "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts",
     "format": "format src",
     "test": "jest --passWithNoTests --silent",
     "_phase:build": "compile transpile src",

server/collaborator/src/account.ts

@@ -13,10 +13,10 @@
 // limitations under the License.
 //
 
-import { WorkspaceLoginInfo } from '@hcengineering/account'
+import { ClientWorkspaceInfo } from '@hcengineering/account'
 import config from './config'
 
-export async function getWorkspaceInfo (token: string): Promise<WorkspaceLoginInfo> {
+export async function getWorkspaceInfo (token: string): Promise<ClientWorkspaceInfo> {
   const accountsUrl = config.AccountsUrl
   const workspaceInfo = await (
     await fetch(accountsUrl, {
@@ -32,5 +32,5 @@ export async function getWorkspaceInfo (token: string): Promise<WorkspaceLoginIn
     })
   ).json()
 
-  return workspaceInfo.result as WorkspaceLoginInfo
+  return workspaceInfo.result as ClientWorkspaceInfo
 }

server/collaborator/src/rpc/methods/copyContent.ts

@@ -13,9 +13,10 @@
 // limitations under the License.
 //
 
-import { yDocCopyXmlField } from '@hcengineering/collaboration'
+import { YDocVersion, takeCollaborativeDocSnapshot, yDocCopyXmlField } from '@hcengineering/collaboration'
-import { type CopyContentRequest, type CopyContentResponse } from '@hcengineering/collaborator-client'
+import { parseDocumentId, type CopyContentRequest, type CopyContentResponse } from '@hcengineering/collaborator-client'
 import { MeasureContext } from '@hcengineering/core'
+import { Doc as YDoc } from 'yjs'
 import { Context } from '../../context'
 import { RpcMethodParams } from '../rpc'
 
@@ -25,17 +26,36 @@ export async function copyContent (
   payload: CopyContentRequest,
   params: RpcMethodParams
 ): Promise<CopyContentResponse> {
-  const { documentId, sourceField, targetField } = payload
+  const { documentId, sourceField, targetField, snapshot } = payload
-  const { hocuspocus } = params
+  const { hocuspocus, storageAdapter } = params
+  const { workspaceId } = context
 
   const connection = await ctx.with('connect', {}, async () => {
     return await hocuspocus.openDirectConnection(documentId, context)
   })
 
   try {
-    await connection.transact((document) => {
+    await ctx.with('copy', {}, async () => {
-      yDocCopyXmlField(document, sourceField, targetField)
+      await connection.transact((document) => {
+        yDocCopyXmlField(document, sourceField, targetField)
+      })
     })
+
+    if (snapshot !== undefined && snapshot.versionId !== 'HEAD') {
+      const ydoc = connection.document ?? new YDoc()
+      const { collaborativeDoc } = parseDocumentId(documentId)
+
+      const version: YDocVersion = {
+        versionId: snapshot.versionId,
+        name: snapshot.versionName ?? snapshot.versionId,
+        createdBy: snapshot.createdBy,
+        createdOn: Date.now()
+      }
+
+      await ctx.with('snapshot', {}, async () => {
+        await takeCollaborativeDocSnapshot(storageAdapter, workspaceId, collaborativeDoc, ydoc, version, ctx)
+      })
+    }
   } finally {
     await connection.disconnect()
   }

server/collaborator/src/rpc/methods/getContent.ts

@@ -24,7 +24,7 @@ export async function getContent (
   payload: GetContentRequest,
   params: RpcMethodParams
 ): Promise<GetContentResponse> {
-  const { documentId, field } = payload
+  const { documentId } = payload
   const { hocuspocus, transformer } = params
 
   const connection = await ctx.with('connect', {}, async () => {
@@ -32,15 +32,17 @@ export async function getContent (
   })
 
   try {
-    const html = await ctx.with('transform', {}, async () => {
+    const content = await ctx.with('transform', {}, async () => {
-      let content = ''
+      const object: Record<string, string> = {}
       await connection.transact((document) => {
-        content = transformer.fromYdoc(document, field)
+        document.share.forEach((_, field) => {
+          object[field] = transformer.fromYdoc(document, field)
+        })
       })
-      return content
+      return object
     })
 
-    return { html }
+    return { content }
   } finally {
     await connection.disconnect()
   }

server/collaborator/src/rpc/methods/takeSnapshot.ts

@@ -13,19 +13,13 @@
 // limitations under the License.
 //
 
-import {
+import { YDocVersion, takeCollaborativeDocSnapshot } from '@hcengineering/collaboration'
-  YDocVersion,
-  collaborativeHistoryDocId,
-  createYdocSnapshot,
-  yDocFromStorage,
-  yDocToStorage
-} from '@hcengineering/collaboration'
 import {
   parseDocumentId,
   type TakeSnapshotRequest,
   type TakeSnapshotResponse
 } from '@hcengineering/collaborator-client'
-import { CollaborativeDocVersionHead, MeasureContext, collaborativeDocParse, generateId } from '@hcengineering/core'
+import { CollaborativeDocVersionHead, MeasureContext, collaborativeDocParse } from '@hcengineering/core'
 import { Doc as YDoc } from 'yjs'
 import { Context } from '../../context'
 import { RpcMethodParams } from '../rpc'
@@ -36,19 +30,19 @@ export async function takeSnapshot (
   payload: TakeSnapshotRequest,
   params: RpcMethodParams
 ): Promise<TakeSnapshotResponse> {
-  const { documentId, snapshotName, createdBy } = payload
+  const { documentId, snapshot } = payload
   const { hocuspocus, storageAdapter } = params
   const { workspaceId } = context
 
   const version: YDocVersion = {
-    versionId: generateId(),
+    versionId: snapshot.versionId,
-    name: snapshotName,
+    name: snapshot.versionName ?? snapshot.versionId,
-    createdBy,
+    createdBy: snapshot.createdBy,
     createdOn: Date.now()
   }
 
   const { collaborativeDoc } = parseDocumentId(documentId)
-  const { documentId: contentDocumentId, versionId } = collaborativeDocParse(collaborativeDoc)
+  const { versionId } = collaborativeDocParse(collaborativeDoc)
   if (versionId !== CollaborativeDocVersionHead) {
     throw new Error('invalid document version')
   }
@@ -58,21 +52,10 @@ export async function takeSnapshot (
   })
 
   try {
-    // load history document directly from storage
+    const ydoc = connection.document ?? new YDoc()
-    const historyDocumentId = collaborativeHistoryDocId(contentDocumentId)
-    const yHistory =
-      (await ctx.with('yDocFromStorage', {}, async () => {
-        return await yDocFromStorage(ctx, storageAdapter, workspaceId, historyDocumentId)
-      })) ?? new YDoc()
 
-    await ctx.with('createYdocSnapshot', {}, async () => {
+    await ctx.with('snapshot', {}, async () => {
-      await connection.transact((yContent) => {
+      await takeCollaborativeDocSnapshot(storageAdapter, workspaceId, collaborativeDoc, ydoc, version, ctx)
-        createYdocSnapshot(yContent, yHistory, version)
-      })
-    })
-
-    await ctx.with('yDocToStorage', {}, async () => {
-      await yDocToStorage(ctx, storageAdapter, workspaceId, historyDocumentId, yHistory)
     })
 
     return { ...version }

server/collaborator/src/rpc/methods/updateContent.ts

@@ -14,8 +14,13 @@
 //
 
 import { MeasureContext } from '@hcengineering/core'
-import { type UpdateContentRequest, type UpdateContentResponse } from '@hcengineering/collaborator-client'
+import {
-import { applyUpdate, encodeStateAsUpdate } from 'yjs'
+  parseDocumentId,
+  type UpdateContentRequest,
+  type UpdateContentResponse
+} from '@hcengineering/collaborator-client'
+import { YDocVersion, takeCollaborativeDocSnapshot } from '@hcengineering/collaboration'
+import { Doc as YDoc, applyUpdate, encodeStateAsUpdate } from 'yjs'
 import { Context } from '../../context'
 import { RpcMethodParams } from '../rpc'
 
@@ -25,12 +30,18 @@ export async function updateContent (
   payload: UpdateContentRequest,
   params: RpcMethodParams
 ): Promise<UpdateContentResponse> {
-  const { documentId, field, html } = payload
+  const { documentId, content, snapshot } = payload
-  const { hocuspocus, transformer } = params
+  const { hocuspocus, transformer, storageAdapter } = params
+  const { workspaceId } = context
 
-  const update = await ctx.with('transform', {}, () => {
+  const updates = await ctx.with('transform', {}, () => {
-    const ydoc = transformer.toYdoc(html, field)
+    const updates: Record<string, Uint8Array> = {}
-    return encodeStateAsUpdate(ydoc)
+
+    Object.entries(content).forEach(([field, markup]) => {
+      const ydoc = transformer.toYdoc(markup, field)
+      updates[field] = encodeStateAsUpdate(ydoc)
+    })
+    return updates
   })
 
   const connection = await ctx.with('connect', {}, async () => {
@@ -40,13 +51,31 @@ export async function updateContent (
   try {
     await ctx.with('update', {}, async () => {
       await connection.transact((document) => {
-        const fragment = document.getXmlFragment(field)
         document.transact(() => {
-          fragment.delete(0, fragment.length)
+          Object.entries(updates).forEach(([field, update]) => {
-          applyUpdate(document, update)
+            const fragment = document.getXmlFragment(field)
+            fragment.delete(0, fragment.length)
+            applyUpdate(document, update)
+          })
         })
       })
     })
+
+    if (snapshot !== undefined && snapshot.versionId !== 'HEAD') {
+      const ydoc = connection.document ?? new YDoc()
+      const { collaborativeDoc } = parseDocumentId(documentId)
+
+      const version: YDocVersion = {
+        versionId: snapshot.versionId,
+        name: snapshot.versionName ?? snapshot.versionId,
+        createdBy: snapshot.createdBy,
+        createdOn: Date.now()
+      }
+
+      await ctx.with('snapshot', {}, async () => {
+        await takeCollaborativeDocSnapshot(storageAdapter, workspaceId, collaborativeDoc, ydoc, version, ctx)
+      })
+    }
   } finally {
     await connection.disconnect()
   }

server/collaborator/src/server.ts

@@ -15,7 +15,6 @@
 
 import { Analytics } from '@hcengineering/analytics'
 import { MeasureContext, generateId, metricsAggregate } from '@hcengineering/core'
-import type { MongoClientReference } from '@hcengineering/mongo'
 import type { StorageAdapter } from '@hcengineering/server-core'
 import { Token, decodeToken } from '@hcengineering/server-token'
 import { ServerKit } from '@hcengineering/text'
@@ -44,16 +43,10 @@ export type Shutdown = () => Promise<void>
 /**
  * @public
  */
-export async function start (
+export async function start (ctx: MeasureContext, config: Config, storageAdapter: StorageAdapter): Promise<Shutdown> {
-  ctx: MeasureContext,
-  config: Config,
-  storageAdapter: StorageAdapter,
-  mongoClient: MongoClientReference
-): Promise<Shutdown> {
   const port = config.Port
 
   ctx.info('Starting collaborator server', { port })
-  const mongo = await mongoClient.getClient()
 
   const app = express()
   app.use(cors())
@@ -116,7 +109,7 @@ export async function start (
       }),
       new StorageExtension({
         ctx: extensionsCtx.newChild('storage', {}),
-        adapter: new PlatformStorageAdapter(storageAdapter, mongo, transformerFactory)
+        adapter: new PlatformStorageAdapter(storageAdapter)
       })
     ]
   })

server/collaborator/src/starter.ts

@@ -18,7 +18,6 @@ import { setMetadata } from '@hcengineering/platform'
 import serverToken from '@hcengineering/server-token'
 
 import type { MeasureContext } from '@hcengineering/core'
-import { getMongoClient } from '@hcengineering/mongo'
 import type { StorageConfiguration } from '@hcengineering/server-core'
 import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
 import config from './config'
@@ -33,15 +32,11 @@ export async function startCollaborator (ctx: MeasureContext, onClose?: () => vo
   const storageConfig: StorageConfiguration = storageConfigFromEnv()
   const storageAdapter = buildStorageFromConfig(storageConfig, config.MongoUrl)
 
-  const mongoClient = getMongoClient(config.MongoUrl)
+  const shutdown = await start(ctx, config, storageAdapter)
-
-  const shutdown = await start(ctx, config, storageAdapter, mongoClient)
 
   const close = (): void => {
     void storageAdapter.close()
-    void shutdown().then(() => {
+    void shutdown()
-      mongoClient.close()
-    })
     onClose?.()
   }
 

server/collaborator/src/storage/platform.ts

@@ -27,27 +27,18 @@ import {
 } from '@hcengineering/collaborator-client'
 import core, {
   CollaborativeDoc,
-  Doc,
   MeasureContext,
   TxOperations,
-  collaborativeDocWithLastVersion,
+  collaborativeDocWithLastVersion
-  toWorkspaceString
 } from '@hcengineering/core'
 import { StorageAdapter } from '@hcengineering/server-core'
-import { areEqualMarkups } from '@hcengineering/text'
-import { MongoClient } from 'mongodb'
 import { Doc as YDoc } from 'yjs'
 import { Context } from '../context'
-import { TransformerFactory } from '../types'
 
 import { CollabStorageAdapter } from './adapter'
 
 export class PlatformStorageAdapter implements CollabStorageAdapter {
-  constructor (
+  constructor (private readonly storage: StorageAdapter) {}
-    private readonly storage: StorageAdapter,
-    private readonly mongodb: MongoClient,
-    private readonly transformerFactory: TransformerFactory
-  ) {}
 
   async loadDocument (ctx: MeasureContext, documentId: DocumentId, context: Context): Promise<YDoc | undefined> {
     // try to load document content
@@ -83,28 +74,6 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
       }
     }
 
-    // finally try to load from the platform
-    const { platformDocumentId } = context
-    if (platformDocumentId !== undefined) {
-      ctx.info('load document platform content', { documentId, platformDocumentId })
-      const ydoc = await ctx.with('load-from-platform', {}, async (ctx) => {
-        try {
-          return await this.loadDocumentFromPlatform(ctx, platformDocumentId, context)
-        } catch (err) {
-          ctx.error('failed to load platform document', { documentId, platformDocumentId, error: err })
-          throw err
-        }
-      })
-
-      // if document was loaded from the initial content or storage we need to save
-      // it to ensure the next time we load it from the ydoc document
-      if (ydoc !== undefined) {
-        ctx.info('save document content', { documentId, platformDocumentId })
-        await this.saveDocumentToStorage(ctx, documentId, ydoc, context)
-        return ydoc
-      }
-    }
-
     // nothing found
     return undefined
   }
@@ -139,7 +108,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
       if (platformDocumentId !== undefined) {
         ctx.info('save document content to platform', { documentId, platformDocumentId })
         await ctx.with('save-to-platform', {}, async (ctx) => {
-          await this.saveDocumentToPlatform(ctx, client, documentId, platformDocumentId, document, snapshot, context)
+          await this.saveDocumentToPlatform(ctx, client, documentId, platformDocumentId, snapshot)
         })
       }
     } finally {
@@ -203,46 +172,18 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
     return yDocVersion
   }
 
-  async loadDocumentFromPlatform (
-    ctx: MeasureContext,
-    platformDocumentId: PlatformDocumentId,
-    context: Context
-  ): Promise<YDoc | undefined> {
-    const { workspaceId } = context
-    const { objectDomain, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId)
-
-    const doc = await ctx.with('query', {}, async () => {
-      const db = this.mongodb.db(toWorkspaceString(workspaceId))
-      return await db.collection<Doc>(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } })
-    })
-
-    const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : ''
-    if (content.startsWith('{') && content.endsWith('}')) {
-      return await ctx.with('transform', {}, () => {
-        const transformer = this.transformerFactory(workspaceId)
-        return transformer.toYdoc(content, objectAttr)
-      })
-    }
-
-    // the content does not seem to be an HTML document
-    return undefined
-  }
-
   async saveDocumentToPlatform (
     ctx: MeasureContext,
     client: Omit<TxOperations, 'close'>,
     documentName: string,
     platformDocumentId: PlatformDocumentId,
-    document: YDoc,
+    snapshot: YDocVersion | undefined
-    snapshot: YDocVersion | undefined,
-    context: Context
   ): Promise<void> {
-    const { workspaceId } = context
     const { objectClass, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId)
 
     const attribute = client.getHierarchy().findAttribute(objectClass, objectAttr)
     if (attribute === undefined) {
-      ctx.info('attribute not found', { documentName, objectClass, objectAttr })
+      ctx.warn('attribute not found', { documentName, objectClass, objectAttr })
       return
     }
 
@@ -251,6 +192,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
     })
 
     if (current === undefined) {
+      ctx.warn('document not found', { documentName, objectClass, objectId })
       return
     }
 
@@ -265,17 +207,8 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
       await ctx.with('update', {}, async () => {
         await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
       })
-    } else if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) {
+    } else {
-      // TODO a temporary solution while we are keeping Markup in Mongo
+      ctx.error('unsupported attribute type', { documentName, objectClass, objectAttr })
-      const content = await ctx.with('transform', {}, () => {
-        const transformer = this.transformerFactory(workspaceId)
-        return transformer.fromYdoc(document, objectAttr)
-      })
-      if (!areEqualMarkups(content, (current as any)[objectAttr])) {
-        await ctx.with('update', {}, async () => {
-          await client.diffUpdate(current, { [objectAttr]: content })
-        })
-      }
     }
   }
 }

server/core/src/indexer/fulltextPush.ts

@@ -374,10 +374,7 @@ function updateDoc2Elastic (
                 ))
             ) {
               let vvv = vv
-              if (
+              if (attribute.type._class === core.class.TypeMarkup) {
-                attribute.type._class === core.class.TypeMarkup ||
-                attribute.type._class === core.class.TypeCollaborativeMarkup
-              ) {
                 ctx.withSync('markup-to-json-text', {}, () => {
                   vvv = jsonToText(markupToJSON(vv))
                 })

server/core/src/indexer/summary.ts

@@ -251,7 +251,7 @@ export async function extractIndexedValues (
         continue
       }
 
-      if (keyAttr.type._class === core.class.TypeMarkup || keyAttr.type._class === core.class.TypeCollaborativeMarkup) {
+      if (keyAttr.type._class === core.class.TypeMarkup) {
         sourceContent = jsonToText(markupToJSON(sourceContent))
       }
 

server/front/readme.md

@@ -14,7 +14,6 @@ Front service is suited to deliver application bundles and resource assets, it a
 * TELEGRAM_URL: Specifies the URL of the Telegram service.
 * REKONI_URL: Specifies the URL of the Rekoni service.
 * COLLABORATOR_URL: Specifies the URL of the collaborator service.
-* COLLABORATOR_API_URL: Specifies the URL of the collaborator API.
 * MODEL_VERSION: Specifies the required model version.
 * SERVER_SECRET: Specifies the server secret.
 * PREVIEW_CONFIG: Specifies the preview configuration.

server/front/src/index.ts

@@ -248,7 +248,6 @@ export function start (
     gmailUrl: string
     calendarUrl: string
     collaboratorUrl: string
-    collaboratorApiUrl: string
     brandingUrl?: string
     previewConfig: string
     pushPublicKey?: string
@@ -299,7 +298,6 @@ export function start (
       GMAIL_URL: config.gmailUrl,
       CALENDAR_URL: config.calendarUrl,
       COLLABORATOR_URL: config.collaboratorUrl,
-      COLLABORATOR_API_URL: config.collaboratorApiUrl,
       BRANDING_URL: config.brandingUrl,
       PREVIEW_CONFIG: config.previewConfig,
       PUSH_PUBLIC_KEY: config.pushPublicKey,

server/front/src/starter.ts

@@ -81,12 +81,6 @@ export function startFront (ctx: MeasureContext, extraConfig?: Record<string, st
     process.exit(1)
   }
 
-  const collaboratorApiUrl = process.env.COLLABORATOR_API_URL
-  if (collaboratorApiUrl === undefined) {
-    console.error('please provide collaborator api url')
-    process.exit(1)
-  }
-
   const modelVersion = process.env.MODEL_VERSION
   if (modelVersion === undefined) {
     console.error('please provide model version requirement')
@@ -135,7 +129,6 @@ export function startFront (ctx: MeasureContext, extraConfig?: Record<string, st
     rekoniUrl,
     calendarUrl,
     collaboratorUrl,
-    collaboratorApiUrl,
     brandingUrl,
     previewConfig,
     pushPublicKey

services/ai-bot/pod-ai-bot/package.json

@@ -22,7 +22,7 @@
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/ai-bot staging",
     "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/ai-bot",
     "docker:tbuild": "rush bundle --to @hcengineering/pod-ai-bot && docker build -t hardcoreeng/ai-bot . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/ai-bot",
-    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 COLLABORATOR_API_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
+    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
     "format": "format src",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent",

services/analytics-collector/pod-analytics-collector/package.json

@@ -22,7 +22,7 @@
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/analytics-collector staging",
     "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/analytics-collector",
     "docker:tbuild": "rush bundle --to @hcengineering/pod-analytics-collector && docker build -t hardcoreeng/analytics-collector . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/analytics-collector",
-    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 COLLABORATOR_API_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
+    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
     "format": "format src",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent",

services/github/model-github/src/migration.ts

@@ -248,10 +248,7 @@ async function migrateMarkup (client: MigrationClient): Promise<void> {
   for (const _class of classes) {
     const attributes = hierarchy.getAllAttributes(_class)
     const filtered = Array.from(attributes.values()).filter((attribute) => {
-      return (
+      return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup)
-        hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) ||
-        hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)
-      )
     })
     if (filtered.length === 0) continue
 

services/github/pod-github/package.json

@@ -22,7 +22,7 @@
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/github staging",
     "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/github",
     "docker:tbuild": "rush bundle --to @hcengineering/pod-github && docker build -t hardcoreeng/github . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/github",
-    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 COLLABORATOR_API_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
+    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
     "format": "format src",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent",

services/github/pod-github/src/collaborator.ts

@@ -4,14 +4,14 @@
 //
 
 import { CollaboratorClient, getClient as getCollaboratorClient } from '@hcengineering/collaborator-client'
-import { Hierarchy, WorkspaceId } from '@hcengineering/core'
+import { WorkspaceId } from '@hcengineering/core'
 import { generateToken } from '@hcengineering/server-token'
 import config from './config'
 
 /**
  * @public
  */
-export function createCollaboratorClient (hierarchy: Hierarchy, workspaceId: WorkspaceId): CollaboratorClient {
+export function createCollaboratorClient (workspaceId: WorkspaceId): CollaboratorClient {
   const token = generateToken(config.SystemEmail, workspaceId, { mode: 'github' })
-  return getCollaboratorClient(hierarchy, workspaceId, token, config.CollaboratorURL)
+  return getCollaboratorClient(workspaceId, token, config.CollaboratorURL)
 }

services/github/pod-github/src/config.ts

@@ -54,7 +54,7 @@ const envMap: { [key in keyof Config]: string } = {
   MongoURL: 'MONGO_URL',
   ConfigurationDB: 'MONGO_DB',
 
-  CollaboratorURL: 'COLLABORATOR_API_URL',
+  CollaboratorURL: 'COLLABORATOR_URL',
 
   ProductID: 'PRODUCT_ID',
 

services/github/pod-github/src/sync/guest.ts

@@ -5,7 +5,7 @@
 import { Branding, TxOperations, WorkspaceIdWithUrl } from '@hcengineering/core'
 import { MarkupMarkType, MarkupNode, MarkupNodeType, traverseMarkupNode } from '@hcengineering/text'
 import { getPublicLink } from '@hcengineering/server-guest-resources'
-import { Issue } from '@hcengineering/tracker'
+import { Task } from '@hcengineering/task'
 
 const githubLinkText = process.env.LINK_TEXT ?? 'Huly®:'
 
@@ -46,7 +46,7 @@ export async function stripGuestLink (markdown: MarkupNode): Promise<void> {
 }
 export async function appendGuestLink (
   client: TxOperations,
-  doc: Issue,
+  doc: Task,
   markdown: MarkupNode,
   workspace: WorkspaceIdWithUrl,
   branding: Branding | null

services/github/pod-github/src/sync/issueBase.ts

@@ -14,16 +14,16 @@ import core, {
   Account,
   AttachedDoc,
   Class,
+  CollaborativeDoc,
   Doc,
   DocumentUpdate,
+  Markup,
   MeasureContext,
   Ref,
   Space,
   Status,
   TxOperations,
-  generateId,
+  generateId
-  getCollaborativeDoc,
-  getCollaborativeDocId
 } from '@hcengineering/core'
 import { IntlString } from '@hcengineering/platform'
 import { LiveQuery } from '@hcengineering/query'
@@ -72,11 +72,18 @@ import {
   isGHWriteAllowed
 } from './utils'
 
+/**
+ * @public
+ */
+export type WithMarkup<T> = {
+  [P in keyof T]: T[P] extends CollaborativeDoc ? Markup : T[P]
+}
+
 /**
  * @public
  */
 export type GithubIssueData = Omit<
-Issue,
+WithMarkup<Issue>,
 | 'commits'
 | 'attachments'
 | 'commits'
@@ -103,6 +110,11 @@ Issue,
 > &
 Record<string, any>
 
+/**
+ * @public
+ */
+export type IssueUpdate = DocumentUpdate<WithMarkup<Issue>>
+
 /**
  * @public
  */
@@ -325,7 +337,7 @@ export abstract class IssueSyncManagerBase {
   async handleUpdate (
     external: IssueExternalData,
     derivedClient: TxOperations,
-    update: DocumentUpdate<Issue>,
+    update: IssueUpdate,
     account: Ref<Account>,
     prj: GithubProject,
     needSync: boolean,
@@ -334,7 +346,7 @@ export abstract class IssueSyncManagerBase {
       state: DocSyncInfo,
       existing: Issue,
       external: IssueExternalData,
-      update: DocumentUpdate<Issue>
+      update: IssueUpdate
     ) => Promise<boolean>,
     extraSyncUpdate?: DocumentUpdate<DocSyncInfo>
   ): Promise<void> {
@@ -354,13 +366,22 @@ export abstract class IssueSyncManagerBase {
       const lastModified = new Date().getTime()
 
       if (doc !== undefined && ((await verifyUpdate?.(syncData, doc, external, update)) ?? true)) {
+        const issueData: DocumentUpdate<Issue> = { ...update, description: doc.description }
         if (
           update.description !== undefined &&
           !areEqualMarkups(update.description, syncData.current?.description ?? '')
         ) {
           try {
-            const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(doc._id, 'description'))
+            const versionId = `${Date.now()}`
-            await this.collaborator.updateContent(collaborativeDoc, 'description', update.description)
+            issueData.description = await this.collaborator.updateContent(
+              doc.description,
+              { description: update.description },
+              {
+                versionId,
+                versionName: versionId,
+                createdBy: account
+              }
+            )
           } catch (err: any) {
             Analytics.handleError(err)
             this.ctx.error(err)
@@ -405,7 +426,12 @@ export abstract class IssueSyncManagerBase {
           },
           lastModified
         )
-        await this.client.diffUpdate(this.client.getHierarchy().as(doc, prj.mixinClass), update, lastModified, account)
+        await this.client.diffUpdate(
+          this.client.getHierarchy().as(doc, prj.mixinClass),
+          issueData,
+          lastModified,
+          account
+        )
         this.provider.sync()
       }
     }
@@ -649,7 +675,7 @@ export abstract class IssueSyncManagerBase {
 
   abstract performIssueFieldsUpdate (
     info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
     platformUpdate: DocumentUpdate<Issue>,
     issueData: GithubIssueData,
     container: ContainerFocus,
@@ -662,7 +688,7 @@ export abstract class IssueSyncManagerBase {
 
   async handleDiffUpdate (
     target: IssueSyncTarget,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
     info: DocSyncInfo,
     issueData: GithubIssueData,
     container: ContainerFocus,
@@ -894,21 +920,30 @@ export abstract class IssueSyncManagerBase {
       }
     }
 
-    if (Object.keys(update).length > 0) {
+    // Update collaborative description
-      // Update collaborative description
+    if (update.description !== undefined) {
       this.ctx.info(`<= perform ${issueExternal.url} update to collaborator`, {
         workspace: this.provider.getWorkspaceId().name
       })
-      if (update.description !== undefined) {
+      try {
-        try {
+        const versionId = `${Date.now()}`
-          issueData.description = update.description
+        issueData.description = update.description
-          const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existingIssue._id, 'description'))
+        update.description = await this.collaborator.updateContent(
-          await this.collaborator.updateContent(collaborativeDoc, 'description', update.description)
+          existingIssue.description,
-        } catch (err: any) {
+          { description: update.description },
-          Analytics.handleError(err)
+          {
-          this.ctx.error('error during description update', err)
+            versionId,
-        }
+            versionName: versionId,
+            createdBy: account
+          }
+        )
+      } catch (err: any) {
+        Analytics.handleError(err)
+        this.ctx.error('error during description update', err)
       }
+    }
+
+    if (Object.keys(update).length > 0) {
       // We have some fields to update of existing from external
       this.ctx.info(`<= perform ${issueExternal.url} update to platform`, {
         ...update,
@@ -930,7 +965,7 @@ export abstract class IssueSyncManagerBase {
   private async notifyConnected (
     container: ContainerFocus,
     info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
     issueExternal: IssueExternalData
   ): Promise<void> {
     const repo = container.repository.find((it) => it._id === info.repository) as GithubIntegrationRepository
@@ -948,9 +983,9 @@ export abstract class IssueSyncManagerBase {
 
   async collectIssueUpdate (
     info: DocSyncInfo,
-    doc: Issue,
+    doc: WithMarkup<Issue>,
     platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status'>,
     container: ContainerFocus,
     issueExternal: IssueExternalData,
     _class: Ref<Class<Issue>>

services/github/pod-github/src/sync/issues.ts

@@ -19,11 +19,9 @@ import core, {
   TxOperations,
   cutObjectArray,
   generateId,
-  getCollaborativeDoc,
+  makeCollaborativeDoc
-  getCollaborativeDocId
 } from '@hcengineering/core'
 import task, { TaskType, calcRank } from '@hcengineering/task'
-import { isEmptyMarkup } from '@hcengineering/text'
 import tracker, { Issue, IssuePriority } from '@hcengineering/tracker'
 import { Issue as GithubIssue, IssuesEvent, ProjectsV2ItemEvent } from '@octokit/webhooks-types'
 import github, {
@@ -47,7 +45,7 @@ import {
 } from '../types'
 import { IssueExternalData, issueDetails } from './githubTypes'
 import { appendGuestLink } from './guest'
-import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget } from './issueBase'
+import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget, IssueUpdate, WithMarkup } from './issueBase'
 import { syncConfig } from './syncConfig'
 import { getSince, gqlp, guessStatus, isGHWriteAllowed, syncRunner } from './utils'
 
@@ -214,7 +212,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         break
       }
       case 'edited': {
-        const update: DocumentUpdate<Issue> = {}
+        const update: IssueUpdate = {}
         const du: DocumentUpdate<DocSyncInfo> = {}
         if (event.changes.body !== undefined) {
           update.description = await this.provider.getMarkup(integration, event.issue.body, this.stripGuestLink)
@@ -373,6 +371,11 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         return { needSync: githubSyncVersion }
       }
 
+      const description = await this.ctx.withLog('query collaborative description', {}, async () => {
+        const content = await this.collaborator.getContent((existing as Issue).description)
+        return content.description ?? ''
+      })
+
       this.ctx.info('create github issue', {
         title: (existing as Issue).title,
         number: (existing as Issue).number,
@@ -382,7 +385,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
         'create github issue',
         {},
         async () => {
-          this.createPromise = this.createGithubIssue(container, existing as Issue, repository)
+          this.createPromise = this.createGithubIssue(container, { ...(existing as Issue), description }, repository)
           return await this.createPromise
         },
         { id: (existing as Issue).identifier, workspace: this.provider.getWorkspaceId().name }
@@ -612,13 +615,12 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
       }
     } else {
       try {
-        const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existing._id, 'description'))
         const description = await this.ctx.withLog(
           'query collaborative description',
           {},
           async () => {
-            const content = await this.collaborator.getContent(collaborativeDoc, 'description')
+            const content = await this.collaborator.getContent((existing as Issue).description)
-            return isEmptyMarkup(content) ? (existing as Issue).description : content
+            return content.description ?? ''
           },
           { url: issueExternal.url }
         )
@@ -657,9 +659,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
 
   async performIssueFieldsUpdate (
     info: DocSyncInfo,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
     platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
     container: ContainerFocus,
     issueExternal: IssueExternalData,
     okit: Octokit,
@@ -751,7 +753,7 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
 
   async createGithubIssue (
     container: ContainerFocus,
-    existing: Issue,
+    existing: WithMarkup<Issue>,
     repository: GithubIntegrationRepository
   ): Promise<IssueExternalData | undefined> {
     const existingIssue = existing
@@ -847,8 +849,13 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
 
     const number = (incResult as any).object.sequence
 
+    const issueId = info._id as unknown as Ref<Issue>
+
+    const { description, ...update } = issueData
+
     const value: AttachedData<Issue> = {
-      ...issueData,
+      ...update,
+      description: makeCollaborativeDoc(issueId, 'description'),
       kind: taskType,
       component: null,
       milestone: null,
@@ -867,7 +874,8 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
       childInfo: [],
       identifier: `${prj.identifier}-${number}`
     }
-    const issueId = info._id as unknown as Ref<Issue>
+
+    await this.collaborator.updateContent(value.description, { description })
 
     await this.client.addCollection(
       tracker.class.Issue,

services/github/pod-github/src/sync/pullrequests.ts

@@ -16,11 +16,9 @@ import core, {
   WithLookup,
   cutObjectArray,
   generateId,
-  getCollaborativeDoc,
+  makeCollaborativeDoc
-  getCollaborativeDocId
 } from '@hcengineering/core'
 import task, { TaskType, calcRank, makeRank } from '@hcengineering/task'
-import { isEmptyMarkup } from '@hcengineering/text'
 import time, { ToDo, ToDoPriority } from '@hcengineering/time'
 import tracker, { Issue, IssuePriority, IssueStatus, Project } from '@hcengineering/tracker'
 import { OctokitResponse } from '@octokit/types'
@@ -59,13 +57,15 @@ import {
   toReviewDecision,
   toReviewState
 } from './githubTypes'
-import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget } from './issueBase'
+import { GithubIssueData, IssueSyncManagerBase, IssueSyncTarget, WithMarkup } from './issueBase'
 import { syncConfig } from './syncConfig'
 import { errorToObj, getSinceRaw, gqlp, guessStatus, isGHWriteAllowed, syncDerivedDocuments, syncRunner } from './utils'
 
 type GithubPullRequestData = GithubIssueData &
 Omit<GithubPullRequest, keyof Issue | 'commits' | 'reviews' | 'reviewComments'>
 
+type GithubPullRequestUpdate = DocumentUpdate<WithMarkup<GithubPullRequest>>
+
 export class PullRequestSyncManager extends IssueSyncManagerBase implements DocSyncManager {
   externalDerivedSync = true
   async handleEvent<T>(integration: IntegrationContainer, derivedClient: TxOperations, evt: T): Promise<void> {
@@ -194,7 +194,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
         break
       }
       case 'edited': {
-        const update: DocumentUpdate<GithubPullRequest> = {}
+        const update: GithubPullRequestUpdate = {}
         const du: DocumentUpdate<DocSyncInfo> = {}
         if (event.changes.title !== undefined) {
           update.title = event.pull_request.title
@@ -584,9 +584,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
           'query collaborative pull request description',
           {},
           async () => {
-            const collaborativeDoc = getCollaborativeDoc(getCollaborativeDocId(existing._id, 'description'))
+            const content = await this.collaborator.getContent((existing as any).description)
-            const content = await this.collaborator.getContent(collaborativeDoc, 'description')
+            return content.description
-            return isEmptyMarkup(content) ? (existing as Issue).description : content
           },
           { url: pullRequestExternal.url }
         )
@@ -987,7 +986,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     info: DocSyncInfo,
     existing: Issue,
     platformUpdate: DocumentUpdate<Issue>,
-    issueData: Pick<Issue, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
+    issueData: Pick<WithMarkup<Issue>, 'title' | 'description' | 'assignee' | 'status' | 'remainingTime' | 'component'>,
     container: ContainerFocus,
     issueExternal: IssueExternalData,
     okit: Octokit,
@@ -1162,10 +1161,14 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
       account
     )
 
+    const prId = info._id as unknown as Ref<GithubPullRequest>
+
+    const { description, ...data } = pullRequestData
     const project = (incResult as any).object as Project
     const number = project.sequence
     const value: AttachedData<GithubPullRequest> = {
-      ...pullRequestData,
+      ...data,
+      description: makeCollaborativeDoc(prId, 'description'),
       kind: taskType,
       component: null,
       milestone: null,
@@ -1188,7 +1191,8 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
       reviews: 0
     }
 
-    const prId = info._id as unknown as Ref<GithubPullRequest>
+    await this.collaborator.updateContent(value.description, { description })
+
     await client.addCollection(
       github.class.GithubPullRequest,
       info.space,

services/github/pod-github/src/worker.ts

@@ -320,7 +320,7 @@ export class GithubWorker implements IntegrationManager {
 
     this.repositoryManager = new RepositorySyncMapper(this.ctx.newChild('repository', {}), this._client, this.app)
 
-    this.collaborator = createCollaboratorClient(this._client.getHierarchy(), this.workspace)
+    this.collaborator = createCollaboratorClient(this.workspace)
 
     this.personMapper = new UsersSyncManager(this.ctx.newChild('users', {}), this._client, this.liveQuery)
 

tests/docker-compose.yaml

@@ -74,7 +74,6 @@ services:
       - REKONI_URL=http://rekoni:4007
       - TELEGRAM_URL=http://localhost:8086
       - COLLABORATOR_URL=ws://localhost:3079
-      - COLLABORATOR_API_URL=http://localhost:3079
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - BRANDING_URL=http://localhost:8083/branding-test.json
   transactor:
@@ -102,6 +101,7 @@ services:
       - REKONI_URL=http://rekoni:7
       - FRONT_URL=http://localhost:8083
       - ACCOUNTS_URL=http://account:3003
+      - COLLABORATOR_URL=http://collaborator:3079
       - LAST_NAME_FIRST=true
       - ELASTIC_INDEX_NAME=local_storage_index
       - BRANDING_PATH=/var/cfg/branding.json
@@ -112,9 +112,9 @@ services:
       - minio
       - transactor
     ports:
-      - 3079:3078
+      - 3079:3079
     environment:
-      - COLLABORATOR_PORT=3078
+      - COLLABORATOR_PORT=3079
       - SECRET=secret
       - ACCOUNTS_URL=http://account:3003
       - MONGO_URL=mongodb://mongodb:27018

tests/sanity/tests/indexer.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from '@playwright/test'
+import { test } from '@playwright/test'
 import { NewDocument } from './model/documents/types'
 import { LeftSideMenuPage } from './model/left-side-menu-page'
 import { DocumentsPage } from './model/documents/documents-page'
@@ -17,8 +17,6 @@ test.use({
   storageState: PlatformSetting
 })
 
-const retryOptions = { intervals: [1000, 1500, 2500], timeout: 60000 }
-
 test.describe('Fulltext index', () => {
   test.beforeEach(async ({ page }) => {
     await (await page.goto(`${PlatformURI}/workbench/sanity-ws`))?.finished()
@@ -65,21 +63,17 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newDocument.title, 1)
-          await spotlight.checkSearchResult(newDocument.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by content', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(contentId)
-          await spotlight.fillSearchInput(contentId)
+        await spotlight.checkSearchResult(newDocument.title, 1)
-          await spotlight.checkSearchResult(newDocument.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
 
@@ -120,30 +114,24 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by old title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.checkSearchResult(newDocument.title, 0)
-          await spotlight.checkSearchResult(newDocument.title, 0)
+        await spotlight.checkSearchResult(updatedTitle, 0)
-          await spotlight.checkSearchResult(updatedTitle, 0)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(updatedTitleId)
-          await spotlight.fillSearchInput(updatedTitleId)
+        await spotlight.checkSearchResult(updatedTitle, 1)
-          await spotlight.checkSearchResult(updatedTitle, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by content', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(updatedContentId)
-          await spotlight.fillSearchInput(updatedContentId)
+        await spotlight.checkSearchResult(updatedTitle, 1)
-          await spotlight.checkSearchResult(updatedTitle, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
 
@@ -169,12 +157,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newDocument.title, 1)
-          await spotlight.checkSearchResult(newDocument.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('remove document', async () => {
@@ -185,12 +171,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newDocument.title, 0)
-          await spotlight.checkSearchResult(newDocument.title, 0)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
   })
@@ -224,21 +208,17 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 1)
-          await spotlight.checkSearchResult(newIssue.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by content', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(contentId)
-          await spotlight.fillSearchInput(contentId)
+        await spotlight.checkSearchResult(newIssue.title, 1)
-          await spotlight.checkSearchResult(newIssue.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
 
@@ -262,12 +242,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 1)
-          await spotlight.checkSearchResult(newIssue.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('update issue', async () => {
@@ -281,30 +259,24 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by old title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 0)
-          await spotlight.checkSearchResult(newIssue.title, 0)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(updatedTitleId)
-          await spotlight.fillSearchInput(updatedTitleId)
+        await spotlight.checkSearchResult(updatedTitle, 1)
-          await spotlight.checkSearchResult(updatedTitle, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('search by content', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(updatedContentId)
-          await spotlight.fillSearchInput(updatedContentId)
+        await spotlight.checkSearchResult(updatedTitle, 1)
-          await spotlight.checkSearchResult(updatedTitle, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
 
@@ -323,12 +295,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 1)
-          await spotlight.checkSearchResult(newIssue.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('remove issue', async () => {
@@ -341,12 +311,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 0)
-          await spotlight.checkSearchResult(newIssue.title, 0)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
   })
@@ -375,12 +343,10 @@ test.describe('Fulltext index', () => {
       })
 
       await test.step('search by title', async () => {
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 1)
-          await spotlight.checkSearchResult(newIssue.title, 1)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
 
       await test.step('create workspace', async () => {
@@ -401,12 +367,10 @@ test.describe('Fulltext index', () => {
 
       await test.step('search by title', async () => {
         await leftSideMenuPage.clickTracker()
-        await expect(async () => {
+        await spotlight.open()
-          await spotlight.open()
+        await spotlight.fillSearchInput(titleId)
-          await spotlight.fillSearchInput(titleId)
+        await spotlight.checkSearchResult(newIssue.title, 0)
-          await spotlight.checkSearchResult(newIssue.title, 0)
+        await spotlight.close()
-          await spotlight.close()
-        }).toPass(retryOptions)
       })
     })
   })