Support unified Huly data import format (#7191)

Support unified Huly data import format

Signed-off-by: Anna Khismatullina <anna.khismatullina@gmail.com>

.vscode/launch.json

@@ -582,7 +582,21 @@
       "name": "Debug ClickUp import",
       "type": "node",
       "request": "launch",
-      "args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws5"],
+      "args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws10"],
+      "env": {
+        "FRONT_URL": "http://localhost:8087",
+      },
+      "runtimeVersion": "20",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "sourceMaps": true,
+      "outputCapture": "std",
+      "cwd": "${workspaceRoot}/dev/import-tool"
+    },
+    {
+      "name": "Debug Huly import",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"],
       "env": {
         "FRONT_URL": "http://localhost:8087"
       },

common/config/rush/pnpm-lock.yaml

@@ -23868,7 +23868,7 @@ packages:
     dev: false
 
   file:projects/import-tool.tgz:
-    resolution: {integrity: sha512-KD9EB2XXo43QYnx8xp4j9z62JGValTBx2uTS4Egw11rsdAoENIxNxI+68CIKASfY8DMKMMuTmUZ6N6nK/6XSiw==, tarball: file:projects/import-tool.tgz}
+    resolution: {integrity: sha512-0Q1/hHZxEdYFPr2qqfovlVJRA8JyvWsKtY3ubHrffnaAMbbN6BNWt6Jf+GzwyGeIwImLI2Oud2x/WqOFb/USdg==, tarball: file:projects/import-tool.tgz}
     name: '@rush-temp/import-tool'
     version: 0.0.0
     dependencies:
@@ -23876,6 +23876,7 @@ packages:
       '@types/domhandler': 2.4.5
       '@types/htmlparser2': 3.10.7
       '@types/jest': 29.5.12
+      '@types/js-yaml': 4.0.9
       '@types/mime-types': 2.1.4
       '@types/minio': 7.0.18
       '@types/node': 20.11.19
@@ -23895,6 +23896,7 @@ packages:
       eslint-plugin-promise: 6.1.1(eslint@8.56.0)
       htmlparser2: 9.1.0
       jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
+      js-yaml: 4.1.0
       mammoth: 1.8.0
       mime-types: 2.1.35
       prettier: 3.2.5

dev/import-tool/README.md

@@ -1,5 +1,25 @@
-### Supported Import Options
+# Huly Import Tool
+
+Tool for importing data into Huly workspace.
+
+## Recommended Import Method
+
+### Unified Format Import
+The recommended way to import data into Huly is using our [Unified Import Format](./src/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure.
+
+See the [complete guide](./src/huly/README.md) and [example workspace](./src/huly/example-workspace) to get started.
+
+### Why Use Unified Format?
+- Simple, human-readable format using YAML and Markdown
+- Flexible structure that can represent data from any system
+- Easy to validate and fix data before import
+- Can be generated automatically by script or prepared manually
+
+## Direct Import Options
+
+We also support direct import from some platforms:
 
 1. **Notion**: see [Import from Notion Guide](./src/notion/README.md)
 2. **ClickUp**: see [Import from ClickUp Guide](./src/clickup/README.md)
 
+These direct imports are suitable for simple migrations, but for complex cases or systems not listed above, please use the Unified Format.

dev/import-tool/package.json

@@ -49,7 +49,8 @@
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
     "@types/jest": "^29.5.5",
-    "@types/csvtojson": "^2.0.0"
+    "@types/csvtojson": "^2.0.0",
+    "@types/js-yaml": "^4.0.9"
   },
   "dependencies": {
     "@hcengineering/attachment": "^0.6.14",
@@ -69,6 +70,7 @@
     "csvtojson": "^2.0.10",
     "@hcengineering/task": "^0.6.20",
     "@hcengineering/contact": "^0.6.24",
-    "@hcengineering/chunter": "^0.6.20"
+    "@hcengineering/chunter": "^0.6.20",
+    "js-yaml": "^4.1.0"
   }
 }

dev/import-tool/src/clickup/clickup.ts

@@ -15,6 +15,7 @@
 import contact, { type Person, type PersonAccount } from '@hcengineering/contact'
 import { type Ref, type Timestamp, type TxOperations } from '@hcengineering/core'
 import { MarkupNodeType, traverseNode, type MarkupNode } from '@hcengineering/text'
+import tracker from '@hcengineering/tracker'
 import csv from 'csvtojson'
 import { download } from '../importer/dowloader'
 import {
@@ -22,10 +23,10 @@ import {
   type ImportComment,
   type ImportIssue,
   type ImportProject,
-  type ImportProjectType,
+  type ImportProjectType
-  type MarkdownPreprocessor
 } from '../importer/importer'
 import { type FileUploader } from '../importer/uploader'
+import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
 
 interface ClickupTask {
   'Task ID': string
@@ -61,77 +62,15 @@ interface ImportIssueEx extends ImportIssue {
   clickupProjectName?: string
 }
 
-class ClickupMarkdownPreprocessor implements MarkdownPreprocessor {
+class ClickupMarkdownPreprocessor extends BaseMarkdownPreprocessor {
-  private readonly MENTION_REGEX = /@([\p{L}\p{M}]+ [\p{L}\p{M}]+)/gu
-  constructor (private readonly personsByName: Map<string, Ref<Person>>) {}
-
   process (json: MarkupNode): MarkupNode {
     traverseNode(json, (node) => {
-      if (node.type === MarkupNodeType.paragraph && node.content !== undefined) {
+      if (node.type === MarkupNodeType.paragraph) {
-        const newContent: MarkupNode[] = []
+        this.processMentions(node)
-        for (const childNode of node.content) {
-          if (childNode.type === MarkupNodeType.text && childNode.text !== undefined) {
-            let match
-            let lastIndex = 0
-            let hasMentions = false
-
-            while ((match = this.MENTION_REGEX.exec(childNode.text)) !== null) {
-              hasMentions = true
-              if (match.index > lastIndex) {
-                newContent.push({
-                  type: MarkupNodeType.text,
-                  text: childNode.text.slice(lastIndex, match.index),
-                  marks: childNode.marks,
-                  attrs: childNode.attrs
-                })
-              }
-
-              const name = match[1]
-              const personRef = this.personsByName.get(name)
-              if (personRef !== undefined) {
-                newContent.push({
-                  type: MarkupNodeType.reference,
-                  attrs: {
-                    id: personRef,
-                    label: name,
-                    objectclass: contact.class.Person
-                  }
-                })
-              } else {
-                newContent.push({
-                  type: MarkupNodeType.text,
-                  text: match[0],
-                  marks: childNode.marks,
-                  attrs: childNode.attrs
-                })
-              }
-
-              lastIndex = this.MENTION_REGEX.lastIndex
-            }
-
-            if (hasMentions) {
-              if (lastIndex < childNode.text.length) {
-                newContent.push({
-                  type: MarkupNodeType.text,
-                  text: childNode.text.slice(lastIndex),
-                  marks: childNode.marks,
-                  attrs: childNode.attrs
-                })
-              }
-            } else {
-              newContent.push(childNode)
-            }
-          } else {
-            newContent.push(childNode)
-          }
-        }
-
-        node.content = newContent
         return false
       }
       return true
     })
-
     return json
   }
 }
@@ -206,8 +145,8 @@ class ClickupImporter {
     for (const projectName of projects) {
       const identifier = this.getProjectIdentifier(projectName)
       importProjectsByName.set(projectName, {
-        class: 'tracker.class.Project',
+        class: tracker.class.Project,
-        name: projectName,
+        title: projectName,
         identifier,
         private: false,
         autoJoin: false,
@@ -280,7 +219,7 @@ class ClickupImporter {
     }
 
     return {
-      class: 'tracker.class.Issue',
+      class: tracker.class.Issue,
       title: clickup['Task Name'],
       descrProvider: () => {
         return Promise.resolve(description)

dev/import-tool/src/huly/README.md

@@ -0,0 +1,127 @@
+## Import from Unified Format Guide
+
+### Overview
+The unified format represents workspace data in a hierarchical folder structure where:
+* Root directory contains space configurations (*.yaml) and their corresponding folders
+* Each space folder contains documents/issues (*.md) and their subdocuments/subissues
+* Documents/issues can have child items in similarly-named folders
+* File named `settings.yaml` is reserved and should not be used for spaces configuration
+* Files without required `class` property in frontmatter will be skipped
+
+See the complete working example in the [example workspace](./example-workspace).
+
+### File Structure Example
+```
+workspace/
+├── Documentation/
+│   ├── Getting Started.md        # Standalone document
+│   ├── User Guide.md             # Document with children
+│   ├── User Guide/               # Child documents folder
+│   │   ├── Installation.md
+│   │   └── Configuration.md
+│   └── files/                    # Attachments
+│       └── architecture.png
+├── Documentation.yaml            # Space configuration
+├── Project Alpha/
+│   ├── 1.Project Setup.md       # Issue with subtasks
+│   ├── 1.Project Setup/         # Subtasks folder
+│   │   ├── 2.Configure CI.md
+│   │   └── 3.Setup Tests.md
+│   ├── 4.Update Docs.md         # Standalone issue
+│   └── files/
+│       └── diagram.png          # Can be referenced in markdown content
+└── Project Alpha.yaml           # Project configuration
+```
+
+### File Format Requirements
+
+#### Space Configuration (*.yaml)
+Project space (`Project Alpha.yaml`):
+```yaml
+class: tracker:class:Project       # Required
+title: Project Alpha               # Required
+identifier: ALPHA                  # Required, max 5 uppercase letters/numbers, must start with a letter
+private: false                     # Optional, default: false
+autoJoin: true                     # Optional, default: true
+owners:                            # Optional, list of email addresses
+  - john.doe@example.com
+members:                           # Optional, list of email addresses
+  - joe.shmoe@example.com
+description: string                # Optional
+defaultIssueStatus: Todo           # Optional
+```
+
+Teamspace (`Documentation.yaml`):
+```yaml
+class: document:class:Teamspace   # Required
+title: Documentation              # Required
+private: false                    # Optional, default: false
+autoJoin: true                    # Optional, default: true
+owners:                           # Optional, list of email addresses
+  - john.doe@example.com
+members:                          # Optional, list of email addresses
+  - joe.shmoe@example.com
+description: string               # Optional
+```
+
+#### Documents and Issues (*.md)
+All files must include YAML frontmatter followed by Markdown content:
+
+Document (`Getting Started.md`):
+```yaml
+---
+class: document:class:Document    # Required
+title: Getting Started Guide      # Required
+---
+# Content in Markdown format
+```
+
+Issue (`1.Project Setup.md`):
+```yaml
+---
+class: tracker:class:Issue        # Required
+title: Project Setup              # Required
+status: In Progress               # Required
+priority: High                    # Optional
+assignee: John Smith              # Optional
+estimation: 8                     # Optional, in hours
+remainingTime: 4                  # Optional, in hours
+---
+Task description in Markdown...
+```
+
+### Task Identification
+* Human-readable task ID is formed by combining project's identifier and task number from filename
+* Example: For project with identifier "ALPHA" and task "1.Setup Project.md", the task ID will be "ALPHA-1"
+
+### Allowed Values
+
+Issue status values:
+* `Backlog`
+* `Todo`
+* `In Progress`
+* `Done`
+* `Canceled`
+
+Issue priority values:
+* `Low`
+* `Medium`
+* `High`
+* `Urgent`
+
+### Run Import Tool
+```bash
+docker run \
+  -e FRONT_URL="https://huly.app" \
+  -v /path/to/workspace:/data \
+  hardcoreeng/import-tool:latest \
+  -- bundle.js import /data \
+  --user your.email@company.com \
+  --password yourpassword \
+  --workspace workspace-id
+```
+
+### Limitations
+* All users must exist in the system before import
+* Assignees are mapped by full name
+* Files in space directories can be used as attachments when referenced in markdown content

dev/import-tool/src/huly/example-workspace/Documentation.yaml

@@ -0,0 +1,9 @@
+class: document:class:Teamspace
+title: Documentation
+private: false
+autoJoin: true
+owners:
+  - john.doe@example.com
+members:
+  - joe.shmoe@example.com
+description: Technical documentation and guides 

dev/import-tool/src/huly/example-workspace/Documentation/Getting Started.md

@@ -0,0 +1,19 @@
+---
+class: document:class:Document
+title: Getting Started Guide
+---
+# Getting Started
+
+Welcome to our project! This guide will help you get started with development.
+
+## Setup Steps
+
+1. Clone the repository
+2. Install dependencies
+3. Set up your environment
+
+## Project Communication
+We use Huly for all project communication:
+- Team discussions in Virtual Office
+- Technical discussions in issue comments
+- Documentation in Huly Documents

dev/import-tool/src/huly/example-workspace/Documentation/User Guide.md

@@ -0,0 +1,16 @@
+---
+class: document:class:Document
+title: User Guide
+---
+# User Guide
+
+Our platform architecture and key components.
+
+## System Overview
+
+<img src="./files/architecture.png" width="800"/>
+
+## Development Workflow
+- Code reviews via GitHub integration
+- CI/CD status in Huly Activity Feed
+- Team sync-ups in Huly Virtual Office 

dev/import-tool/src/huly/example-workspace/Documentation/User Guide/Installation.md

@@ -0,0 +1,19 @@
+---
+class: document:class:Document
+title: Installation Guide
+---
+# Installation
+
+## System Requirements
+- Node.js 18 or higher
+- Docker Desktop
+- Git
+
+## Setup Steps
+
+1. Clone the repository
+2. Install dependencies
+3. Configure your environment
+
+## Need Help?
+Contact @Joe Shmoe for technical support 

dev/import-tool/src/huly/example-workspace/Project Alpha.yaml

@@ -0,0 +1,11 @@
+class: tracker:class:Project
+title: Project Alpha
+identifier: ALPHA
+private: false
+autoJoin: true
+owners:
+  - john.doe@example.com
+members:
+  - joe.shmoe@example.com
+description: Main development project
+defaultIssueStatus: Todo 

dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup.md

@@ -0,0 +1,28 @@
+---
+class: tracker:class:Issue
+title: Project Setup
+status: In Progress
+priority: High
+assignee: John Doe
+estimation: 8
+remainingTime: 4
+comments:
+  - author: john.doe@example.com
+    text: |
+      Initial infrastructure is ready!
+  - author: joe.shmoe@example.com
+    text: |
+      Great! I'll start working on [Configure CI](./1.Project%20Setup/2.Configure%20CI.md) task now.
+  - author: john.doe@example.com
+    text: |
+      Perfect, don't forget to update [documentation](../Documentation/User%20Guide/Installation.md) when you're done.
+---
+**Initial project infrastructure setup.**
+
+Tasks:
+1. Repository setup
+2. Configure GitHub integration with Huly
+3. Set up CI/CD pipeline using [configuration template](./files/config.yaml)
+4. Configure team access and permissions
+
+Daily sync-ups in Huly Virtual Office at 10:00 AM UTC. 

dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md

@@ -0,0 +1,13 @@
+---
+class: tracker:class:Issue
+title: Configure CI Pipeline
+status: Todo
+priority: High
+assignee: Joe Shmoe
+estimation: 4
+---
+Set up CI pipeline with GitHub integration:
+- Configure GitHub Actions
+- Set up test automation
+- Add build status notifications to Huly
+- Configure deployment workflow 

dev/import-tool/src/huly/example-workspace/Project Alpha/4.Update Docs.md

@@ -0,0 +1,11 @@
+---
+class: tracker:class:Issue
+title: Update Documentation
+status: Backlog
+priority: Medium
+---
+Update project documentation in Huly:
+- Complete installation guide
+- Add troubleshooting section
+- Document GitHub integration setup
+- Review in next team meeting 

dev/import-tool/src/huly/example-workspace/Project Alpha/files/config.yaml

@@ -0,0 +1,18 @@
+# Development environment configuration
+server:
+  port: 3000
+  host: localhost
+
+database:
+  host: localhost
+  port: 5432
+  name: project_alpha
+  user: postgres
+
+github:
+  webhook_url: https://api.github.com/webhooks
+  branch: master
+
+logging:
+  level: debug
+  format: json 

dev/import-tool/src/huly/unified.ts

@@ -0,0 +1,625 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { type Attachment } from '@hcengineering/attachment'
+import contact, { type Person, type PersonAccount } from '@hcengineering/contact'
+import { type Class, type Doc, generateId, type Ref, type Space, type TxOperations } from '@hcengineering/core'
+import document, { type Document } from '@hcengineering/document'
+import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text'
+import tracker, { type Issue } from '@hcengineering/tracker'
+import * as fs from 'fs'
+import * as yaml from 'js-yaml'
+import { contentType } from 'mime-types'
+import * as path from 'path'
+import { ImportWorkspaceBuilder } from '../importer/builder'
+import {
+  type ImportAttachment,
+  type ImportComment,
+  type ImportDocument,
+  type ImportIssue,
+  type ImportProject,
+  type ImportProjectType,
+  type ImportTeamspace,
+  type ImportWorkspace,
+  WorkspaceImporter
+} from '../importer/importer'
+import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
+import { type FileUploader } from '../importer/uploader'
+
+interface UnifiedComment {
+  author: string
+  text: string
+}
+
+interface UnifiedIssueHeader {
+  class: 'tracker:class:Issue'
+  title: string
+  status: string
+  assignee?: string
+  priority?: string
+  estimation?: number // in hours
+  remainingTime?: number // in hours
+  comments?: UnifiedComment[]
+}
+
+interface UnifiedSpaceSettings {
+  class: 'tracker:class:Project' | 'document:class:Teamspace'
+  title: string
+  private?: boolean
+  autoJoin?: boolean
+  owners?: string[]
+  members?: string[]
+  description?: string
+}
+
+interface UnifiedProjectSettings extends UnifiedSpaceSettings {
+  class: 'tracker:class:Project'
+  identifier: string
+  projectType?: string
+  defaultIssueStatus?: string
+}
+
+interface UnifiedTeamspaceSettings extends UnifiedSpaceSettings {
+  class: 'document:class:Teamspace'
+}
+
+interface UnifiedDocumentHeader {
+  class: 'document:class:Document'
+  title: string
+}
+
+interface UnifiedWorkspaceSettings {
+  projectTypes?: Array<{
+    name: string
+    taskTypes?: Array<{
+      name: string
+      description?: string
+      statuses: Array<{
+        name: string
+        description: string
+      }>
+    }>
+  }>
+}
+
+class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor {
+  constructor (
+    private readonly urlProvider: (id: string) => string,
+    private readonly metadataByFilePath: Map<string, DocMetadata>,
+    private readonly metadataById: Map<Ref<Doc>, DocMetadata>,
+    private readonly attachMetadataByPath: Map<string, AttachmentMetadata>,
+    personsByName: Map<string, Ref<Person>>
+  ) {
+    super(personsByName)
+  }
+
+  process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
+    traverseNode(json, (node) => {
+      if (node.type === MarkupNodeType.image) {
+        this.processImageNode(node, id, spaceId)
+      } else {
+        this.processLinkMarks(node, id, spaceId)
+        this.processMentions(node)
+      }
+      return true
+    })
+    return json
+  }
+
+  private processImageNode (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
+    const src = node.attrs?.src
+    if (src === undefined) return
+
+    const sourceMeta = this.getSourceMetadata(id)
+    if (sourceMeta == null) return
+
+    const href = decodeURI(src as string)
+    const fullPath = path.resolve(path.dirname(sourceMeta.path), href)
+    const attachmentMeta = this.attachMetadataByPath.get(fullPath)
+
+    if (attachmentMeta === undefined) {
+      console.warn(`Attachment image not found for ${fullPath}`)
+      return
+    }
+
+    this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
+    this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name)
+  }
+
+  private processLinkMarks (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
+    traverseNodeMarks(node, (mark) => {
+      if (mark.type !== MarkupMarkType.link) return
+
+      const sourceMeta = this.getSourceMetadata(id)
+      if (sourceMeta == null) return
+
+      const href = decodeURI(mark.attrs.href)
+      const fullPath = path.resolve(path.dirname(sourceMeta.path), href)
+
+      if (this.metadataByFilePath.has(fullPath)) {
+        const targetDocMeta = this.metadataByFilePath.get(fullPath)
+        if (targetDocMeta !== undefined) {
+          this.alterInternalLinkNode(node, targetDocMeta)
+        }
+      } else if (this.attachMetadataByPath.has(fullPath)) {
+        const attachmentMeta = this.attachMetadataByPath.get(fullPath)
+        if (attachmentMeta !== undefined) {
+          this.alterAttachmentLinkNode(node, attachmentMeta)
+          this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
+        }
+      } else {
+        console.log('Unknown link type, leave it as is:', href)
+      }
+    })
+  }
+
+  private alterImageNode (node: MarkupNode, id: string, name: string): void {
+    node.type = MarkupNodeType.image
+    if (node.attrs !== undefined) {
+      node.attrs = {
+        'file-id': id,
+        src: this.urlProvider(id),
+        width: node.attrs.width ?? null,
+        height: node.attrs.height ?? null,
+        align: node.attrs.align ?? null,
+        alt: name,
+        title: name
+      }
+      const mimeType = this.getContentType(name)
+      if (mimeType !== undefined) {
+        node.attrs['data-file-type'] = mimeType
+      }
+    }
+  }
+
+  private alterInternalLinkNode (node: MarkupNode, targetMeta: DocMetadata): void {
+    node.type = MarkupNodeType.reference
+    node.attrs = {
+      id: targetMeta.id,
+      label: targetMeta.refTitle,
+      objectclass: targetMeta.class,
+      text: '',
+      content: ''
+    }
+  }
+
+  private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void {
+    const stats = fs.statSync(targetMeta.path)
+    node.type = MarkupNodeType.file
+    node.attrs = {
+      'file-id': targetMeta.id,
+      'data-file-name': targetMeta.name,
+      'data-file-size': stats.size,
+      'data-file-href': targetMeta.path
+    }
+    const mimeType = this.getContentType(targetMeta.name)
+    if (mimeType !== undefined) {
+      node.attrs['data-file-type'] = mimeType
+    }
+  }
+
+  private getContentType (fileName: string): string | undefined {
+    const mimeType = contentType(fileName)
+    return mimeType !== false ? mimeType : undefined
+  }
+
+  private getSourceMetadata (id: Ref<Doc>): DocMetadata | null {
+    const sourceMeta = this.metadataById.get(id)
+    if (sourceMeta == null) {
+      console.warn(`Source metadata not found for ${id}`)
+      return null
+    }
+    return sourceMeta
+  }
+
+  private updateAttachmentMetadata (
+    fullPath: string,
+    attachmentMeta: AttachmentMetadata,
+    id: Ref<Doc>,
+    spaceId: Ref<Space>,
+    sourceMeta: DocMetadata
+  ): void {
+    this.attachMetadataByPath.set(fullPath, {
+      ...attachmentMeta,
+      spaceId,
+      parentId: id,
+      parentClass: sourceMeta.class as Ref<Class<Doc<Space>>>
+    })
+  }
+}
+
+interface DocMetadata {
+  id: Ref<Doc>
+  class: string
+  path: string
+  refTitle: string
+}
+
+interface AttachmentMetadata {
+  id: Ref<Attachment>
+  name: string
+  path: string
+  parentId?: Ref<Doc>
+  parentClass?: Ref<Class<Doc<Space>>>
+  spaceId?: Ref<Space>
+}
+
+export class UnifiedFormatImporter {
+  private readonly metadataById = new Map<Ref<Doc>, DocMetadata>()
+  private readonly metadataByFilePath = new Map<string, DocMetadata>()
+  private readonly attachMetadataByPath = new Map<string, AttachmentMetadata>()
+
+  private personsByName = new Map<string, Ref<Person>>()
+  private accountsByEmail = new Map<string, Ref<PersonAccount>>()
+
+  constructor (
+    private readonly client: TxOperations,
+    private readonly fileUploader: FileUploader
+  ) {}
+
+  async importFolder (folderPath: string): Promise<void> {
+    await this.cachePersonsByNames()
+    await this.cacheAccountsByEmails()
+
+    const workspaceData = await this.processImportFolder(folderPath)
+
+    console.log('========================================')
+    console.log('IMPORT DATA STRUCTURE: ', JSON.stringify(workspaceData, null, 4))
+    console.log('========================================')
+
+    console.log('Importing documents...')
+    const preprocessor = new HulyMarkdownPreprocessor(
+      this.fileUploader.getFileUrl,
+      this.metadataByFilePath,
+      this.metadataById,
+      this.attachMetadataByPath,
+      this.personsByName
+    )
+    await new WorkspaceImporter(this.client, this.fileUploader, workspaceData, preprocessor).performImport()
+
+    console.log('Importing attachments...')
+    const attachments: ImportAttachment[] = Array.from(this.attachMetadataByPath.values())
+      .filter((attachment) => attachment.parentId !== undefined)
+      .map((attachment) => {
+        return {
+          id: attachment.id,
+          title: path.basename(attachment.path),
+          blobProvider: async () => {
+            const data = fs.readFileSync(attachment.path)
+            return new Blob([data])
+          },
+          parentId: attachment.parentId,
+          parentClass: attachment.parentClass,
+          spaceId: attachment.spaceId
+        }
+      })
+    await new WorkspaceImporter(this.client, this.fileUploader, { attachments }).performImport()
+
+    console.log('========================================')
+    console.log('IMPORT SUCCESS')
+  }
+
+  private async processImportFolder (folderPath: string): Promise<ImportWorkspace> {
+    const builder = new ImportWorkspaceBuilder(this.client)
+    await builder.initCache()
+
+    // Load workspace settings if exists
+    const wsSettingsPath = path.join(folderPath, 'settings.yaml')
+    if (fs.existsSync(wsSettingsPath)) {
+      const wsSettingsFile = fs.readFileSync(wsSettingsPath, 'utf8')
+      const wsSettings = yaml.load(wsSettingsFile) as UnifiedWorkspaceSettings
+
+      // Add project types
+      for (const pt of this.processProjectTypes(wsSettings)) {
+        builder.addProjectType(pt)
+      }
+    }
+
+    // Process all yaml files first
+    const yamlFiles = fs.readdirSync(folderPath).filter((f) => f.endsWith('.yaml') && f !== 'settings.yaml')
+
+    for (const yamlFile of yamlFiles) {
+      const yamlPath = path.join(folderPath, yamlFile)
+      const spaceName = path.basename(yamlFile, '.yaml')
+      const spacePath = path.join(folderPath, spaceName)
+
+      try {
+        console.log(`Processing ${spaceName}...`)
+        const spaceConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as UnifiedSpaceSettings
+
+        if (spaceConfig.class === undefined) {
+          console.warn(`Skipping ${spaceName}: not a space - no class specified`)
+          continue
+        }
+
+        switch (spaceConfig.class) {
+          case tracker.class.Project: {
+            const project = await this.processProject(spaceConfig as UnifiedProjectSettings)
+            builder.addProject(spacePath, project)
+            if (fs.existsSync(spacePath) && fs.statSync(spacePath).isDirectory()) {
+              await this.processIssuesRecursively(builder, project.identifier, spacePath, spacePath)
+            }
+            break
+          }
+
+          case document.class.Teamspace: {
+            const teamspace = await this.processTeamspace(spaceConfig as UnifiedTeamspaceSettings)
+            builder.addTeamspace(spacePath, teamspace)
+            if (fs.existsSync(spacePath) && fs.statSync(spacePath).isDirectory()) {
+              await this.processDocumentsRecursively(builder, spacePath, spacePath)
+            }
+            break
+          }
+
+          default: {
+            throw new Error(`Unknown space class ${spaceConfig.class} in ${spaceName}`)
+          }
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error)
+        throw new Error(`Invalid space configuration in ${spaceName}: ${message}`)
+      }
+    }
+
+    await this.processAttachments(folderPath)
+
+    return builder.build()
+  }
+
+  private async processIssuesRecursively (
+    builder: ImportWorkspaceBuilder,
+    projectIdentifier: string,
+    projectPath: string,
+    currentPath: string,
+    parentIssuePath?: string
+  ): Promise<void> {
+    const issueFiles = fs.readdirSync(currentPath).filter((f) => f.endsWith('.md'))
+
+    for (const issueFile of issueFiles) {
+      const issuePath = path.join(currentPath, issueFile)
+      const issueHeader = (await this.readYamlHeader(issuePath)) as UnifiedIssueHeader
+
+      if (issueHeader.class === undefined) {
+        console.warn(`Skipping ${issueFile}: not an issue`)
+        continue
+      }
+
+      if (issueHeader.class === tracker.class.Issue) {
+        const numberMatch = issueFile.match(/^(\d+)\./)
+        const issueNumber = numberMatch?.[1]
+
+        const meta: DocMetadata = {
+          id: generateId<Issue>(),
+          class: tracker.class.Issue,
+          path: issuePath,
+          refTitle: `${projectIdentifier}-${issueNumber}`
+        }
+
+        this.metadataById.set(meta.id, meta)
+        this.metadataByFilePath.set(issuePath, meta)
+
+        const issue: ImportIssue = {
+          id: meta.id as Ref<Issue>,
+          class: tracker.class.Issue,
+          title: issueHeader.title,
+          number: parseInt(issueNumber ?? 'NaN'),
+          descrProvider: async () => await this.readMarkdownContent(issuePath),
+          status: { name: issueHeader.status },
+          priority: issueHeader.priority,
+          estimation: issueHeader.estimation,
+          remainingTime: issueHeader.remainingTime,
+          comments: this.processComments(issueHeader.comments),
+          subdocs: [], // Will be added via builder
+          assignee: this.findPersonByName(issueHeader.assignee)
+        }
+
+        builder.addIssue(projectPath, issuePath, issue, parentIssuePath)
+
+        // Process sub-issues if they exist
+        const subDir = path.join(currentPath, issueFile.replace('.md', ''))
+        if (fs.existsSync(subDir) && fs.statSync(subDir).isDirectory()) {
+          await this.processIssuesRecursively(builder, projectIdentifier, projectPath, subDir, issuePath)
+        }
+      } else {
+        throw new Error(`Unknown issue class ${issueHeader.class} in ${issueFile}`)
+      }
+    }
+  }
+
+  private findPersonByName (name?: string): Ref<Person> | undefined {
+    if (name === undefined) {
+      return undefined
+    }
+    const person = this.personsByName.get(name)
+    if (person === undefined) {
+      throw new Error(`Person not found: ${name}`)
+    }
+    return person
+  }
+
+  private findAccountByEmail (email: string): Ref<PersonAccount> {
+    const account = this.accountsByEmail.get(email)
+    if (account === undefined) {
+      throw new Error(`Account not found: ${email}`)
+    }
+    return account
+  }
+
+  private async processDocumentsRecursively (
+    builder: ImportWorkspaceBuilder,
+    teamspacePath: string,
+    currentPath: string,
+    parentDocPath?: string
+  ): Promise<void> {
+    const docFiles = fs.readdirSync(currentPath).filter((f) => f.endsWith('.md'))
+
+    for (const docFile of docFiles) {
+      const docPath = path.join(currentPath, docFile)
+      const docHeader = (await this.readYamlHeader(docPath)) as UnifiedDocumentHeader
+
+      if (docHeader.class === undefined) {
+        console.warn(`Skipping ${docFile}: not a document`)
+        continue
+      }
+
+      if (docHeader.class === document.class.Document) {
+        const docMeta: DocMetadata = {
+          id: generateId<Document>(),
+          class: document.class.Document,
+          path: docPath,
+          refTitle: docHeader.title
+        }
+
+        this.metadataById.set(docMeta.id, docMeta)
+        this.metadataByFilePath.set(docPath, docMeta)
+
+        const doc: ImportDocument = {
+          id: docMeta.id as Ref<Document>,
+          class: document.class.Document,
+          title: docHeader.title,
+          descrProvider: async () => await this.readMarkdownContent(docPath),
+          subdocs: [] // Will be added via builder
+        }
+
+        builder.addDocument(teamspacePath, docPath, doc, parentDocPath)
+
+        // Process subdocuments if they exist
+        const subDir = path.join(currentPath, docFile.replace('.md', ''))
+        if (fs.existsSync(subDir) && fs.statSync(subDir).isDirectory()) {
+          await this.processDocumentsRecursively(builder, teamspacePath, subDir, docPath)
+        }
+      } else {
+        throw new Error(`Unknown document class ${docHeader.class} in ${docFile}`)
+      }
+    }
+  }
+
+  private processComments (comments: UnifiedComment[] = []): ImportComment[] {
+    return comments.map((comment) => {
+      return {
+        text: comment.text,
+        author: this.findAccountByEmail(comment.author)
+      }
+    })
+  }
+
+  private processProjectTypes (wsHeader: UnifiedWorkspaceSettings): ImportProjectType[] {
+    return (
+      wsHeader.projectTypes?.map((pt) => ({
+        name: pt.name,
+        taskTypes: pt.taskTypes?.map((tt) => ({
+          name: tt.name,
+          description: tt.description,
+          statuses: tt.statuses.map((st) => ({
+            name: st.name,
+            description: st.description
+          }))
+        }))
+      })) ?? []
+    )
+  }
+
+  private async processProject (projectHeader: UnifiedProjectSettings): Promise<ImportProject> {
+    return {
+      class: tracker.class.Project,
+      title: projectHeader.title,
+      identifier: projectHeader.identifier,
+      private: projectHeader.private ?? false,
+      autoJoin: projectHeader.autoJoin ?? true,
+      description: projectHeader.description,
+      defaultIssueStatus:
+        projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined,
+      owners:
+        projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
+      members:
+        projectHeader.members !== undefined ? projectHeader.members.map((email) => this.findAccountByEmail(email)) : [],
+      docs: []
+    }
+  }
+
+  private async processTeamspace (spaceHeader: UnifiedTeamspaceSettings): Promise<ImportTeamspace> {
+    return {
+      class: document.class.Teamspace,
+      title: spaceHeader.title,
+      private: spaceHeader.private ?? false,
+      autoJoin: spaceHeader.autoJoin ?? true,
+      description: spaceHeader.description,
+      owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
+      members:
+        spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [],
+      docs: []
+    }
+  }
+
+  private async readYamlHeader (filePath: string): Promise<any> {
+    console.log('Read YAML header from: ', filePath)
+    const content = fs.readFileSync(filePath, 'utf8')
+    const match = content.match(/^---\n([\s\S]*?)\n---/)
+    if (match != null) {
+      return yaml.load(match[1])
+    }
+    return {}
+  }
+
+  private async readMarkdownContent (filePath: string): Promise<string> {
+    const content = fs.readFileSync(filePath, 'utf8')
+    const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s)
+    return match != null ? match[1] : content
+  }
+
+  private async cachePersonsByNames (): Promise<void> {
+    this.personsByName = (await this.client.findAll(contact.class.Person, {}))
+      .map((person) => {
+        return {
+          _id: person._id,
+          name: person.name.split(',').reverse().join(' ')
+        }
+      })
+      .reduce((refByName, person) => {
+        refByName.set(person.name, person._id)
+        return refByName
+      }, new Map())
+  }
+
+  private async cacheAccountsByEmails (): Promise<void> {
+    const accounts = await this.client.findAll(contact.class.PersonAccount, {})
+    this.accountsByEmail = accounts.reduce((map, account) => {
+      map.set(account.email, account._id)
+      return map
+    }, new Map())
+  }
+
+  private async processAttachments (folderPath: string): Promise<void> {
+    const processDir = async (dir: string): Promise<void> => {
+      const entries = fs.readdirSync(dir, { withFileTypes: true })
+
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name)
+
+        if (entry.isDirectory()) {
+          await processDir(fullPath)
+        } else if (entry.isFile()) {
+          // Skip files that are already processed as documents or issues
+          if (!this.metadataByFilePath.has(fullPath)) {
+            const attachmentId = generateId<Attachment>()
+            this.attachMetadataByPath.set(fullPath, { id: attachmentId, name: entry.name, path: fullPath })
+          }
+        }
+      }
+    }
+
+    await processDir(folderPath)
+  }
+}

dev/import-tool/src/importer/builder.ts

@@ -0,0 +1,499 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import { type DocumentQuery, type Ref, type Status, type TxOperations } from '@hcengineering/core'
+import document from '@hcengineering/document'
+import tracker, { IssuePriority, type IssueStatus } from '@hcengineering/tracker'
+import {
+  type ImportDocument,
+  type ImportIssue,
+  type ImportProject,
+  type ImportProjectType,
+  type ImportTeamspace,
+  type ImportWorkspace
+} from './importer'
+
+export interface ValidationError {
+  path: string
+  error: string
+}
+
+export interface ValidationResult {
+  isValid: boolean
+  errors: Map<string, ValidationError>
+}
+
+const MAX_PROJECT_IDENTIFIER_LENGTH = 5
+const PROJECT_IDENTIFIER_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/
+
+export class ImportWorkspaceBuilder {
+  private readonly projects = new Map<string, ImportProject>()
+  private readonly teamspaces = new Map<string, ImportTeamspace>()
+  private readonly projectTypes = new Map<string, ImportProjectType>()
+  private readonly issuesByProject = new Map<string, Map<string, ImportIssue>>()
+  private readonly issueParents = new Map<string, string>()
+  private readonly documentsByTeamspace = new Map<string, Map<string, ImportDocument>>()
+  private readonly documentParents = new Map<string, string>()
+  private readonly errors = new Map<string, ValidationError>()
+
+  private readonly issueStatusCache = new Map<string, Ref<IssueStatus>>()
+
+  constructor (
+    private readonly client: TxOperations,
+    private readonly strictMode: boolean = true
+  ) {}
+
+  async initCache (): Promise<this> {
+    await this.cacheIssueStatuses()
+    return this
+  }
+
+  addProjectType (projectType: ImportProjectType): this {
+    this.validateAndAdd(
+      'projectType',
+      projectType.name,
+      projectType,
+      (pt) => this.validateProjectType(pt),
+      this.projectTypes
+    )
+    return this
+  }
+
+  addProject (path: string, project: ImportProject): this {
+    this.validateAndAdd('project', path, project, (p) => this.validateProject(p), this.projects, path)
+    return this
+  }
+
+  addTeamspace (path: string, teamspace: ImportTeamspace): this {
+    this.validateAndAdd('teamspace', path, teamspace, (t) => this.validateTeamspace(t), this.teamspaces, path)
+    return this
+  }
+
+  addIssue (projectPath: string, issuePath: string, issue: ImportIssue, parentIssuePath?: string): this {
+    if (!this.issuesByProject.has(projectPath)) {
+      this.issuesByProject.set(projectPath, new Map())
+    }
+
+    const projectIssues = this.issuesByProject.get(projectPath)
+    if (projectIssues === undefined) {
+      throw new Error(`Project ${projectPath} not found`)
+    }
+
+    const duplicateIssue = Array.from(projectIssues.values()).find(
+      (existingIssue) => existingIssue.number === issue.number
+    )
+
+    if (duplicateIssue !== undefined) {
+      this.addError(issuePath, `Duplicate issue number ${issue.number} in project ${projectPath}`)
+    } else {
+      this.validateAndAdd('issue', issuePath, issue, (i) => this.validateIssue(i), projectIssues, issuePath)
+
+      if (parentIssuePath !== undefined) {
+        this.issueParents.set(issuePath, parentIssuePath)
+      }
+    }
+    return this
+  }
+
+  addDocument (teamspacePath: string, docPath: string, doc: ImportDocument, parentDocPath?: string): this {
+    if (!this.documentsByTeamspace.has(teamspacePath)) {
+      this.documentsByTeamspace.set(teamspacePath, new Map())
+    }
+
+    const docs = this.documentsByTeamspace.get(teamspacePath)
+    if (docs === undefined) {
+      throw new Error(`Teamspace ${teamspacePath} not found`)
+    }
+
+    this.validateAndAdd('document', docPath, doc, (d) => this.validateDocument(d), docs, docPath)
+
+    if (parentDocPath !== undefined) {
+      this.documentParents.set(docPath, parentDocPath)
+    }
+
+    return this
+  }
+
+  validate (): ValidationResult {
+    // Perform cross-entity validation
+    this.validateProjectReferences()
+    this.validateSpaceDocuments()
+
+    return {
+      isValid: this.errors.size === 0,
+      errors: this.errors
+    }
+  }
+
+  build (): ImportWorkspace {
+    const validation = this.validate()
+    if (this.strictMode && !validation.isValid) {
+      throw new Error(
+        'Invalid workspace: \n' +
+          Array.from(validation.errors.values())
+            .map((e) => `    * ${e.path}: ${e.error}`)
+            .join(';\n')
+      )
+    }
+
+    for (const [teamspacePath, docs] of this.documentsByTeamspace) {
+      const teamspace = this.teamspaces.get(teamspacePath)
+      if (teamspace !== undefined) {
+        const rootDocPaths = Array.from(docs.keys()).filter((docPath) => !this.documentParents.has(docPath))
+
+        for (const rootPath of rootDocPaths) {
+          this.buildDocumentHierarchy(rootPath, docs)
+        }
+
+        teamspace.docs = rootDocPaths.map((path) => docs.get(path)).filter(Boolean) as ImportDocument[]
+      }
+    }
+
+    for (const [projectPath, issues] of this.issuesByProject) {
+      const project = this.projects.get(projectPath)
+      if (project !== undefined) {
+        const rootIssuePaths = Array.from(issues.keys()).filter((issuePath) => !this.issueParents.has(issuePath))
+
+        for (const rootPath of rootIssuePaths) {
+          this.buildIssueHierarchy(rootPath, issues)
+        }
+
+        project.docs = rootIssuePaths.map((path) => issues.get(path)).filter(Boolean) as ImportIssue[]
+      }
+    }
+
+    return {
+      projectTypes: Array.from(this.projectTypes.values()),
+      spaces: [...Array.from(this.projects.values()), ...Array.from(this.teamspaces.values())]
+    }
+  }
+
+  async cacheIssueStatuses (): Promise<void> {
+    const query: DocumentQuery<Status> = {
+      ofAttribute: tracker.attribute.IssueStatus
+    }
+
+    const statuses = await this.client.findAll(tracker.class.IssueStatus, query)
+    for (const status of statuses) {
+      this.issueStatusCache.set(status.name, status._id)
+    }
+  }
+
+  private validateAndAdd<T, K>(
+    type: string,
+    path: string,
+    item: T,
+    validator: (item: T) => string[],
+    collection: Map<K, T>,
+    key?: K
+  ): void {
+    const errors = validator(item)
+    if (errors.length > 0) {
+      this.addError(path, `Invalid ${type} at ${path}: \n${errors.map((e) => `    * ${e}`).join('\n')}`)
+      if (this.strictMode) {
+        throw new Error(`Invalid ${type} at ${path}: \n${errors.map((e) => `    * ${e}`).join('\n')}`)
+      }
+    } else {
+      collection.set((key ?? path) as K, item)
+    }
+  }
+
+  private validateProjectType (projectType: ImportProjectType): string[] {
+    const errors: string[] = []
+    if (!this.validateStringDefined(projectType.name)) {
+      errors.push('name is required')
+    }
+    return errors
+  }
+
+  private validateProject (project: ImportProject): string[] {
+    const errors: string[] = []
+
+    errors.push(...this.validateType(project.title, 'string', 'title'))
+    errors.push(...this.validateType(project.identifier, 'string', 'identifier'))
+    errors.push(...this.validateType(project.class, 'string', 'class'))
+
+    if (project.private !== undefined) {
+      errors.push(...this.validateType(project.private, 'boolean', 'private'))
+    }
+
+    if (project.autoJoin !== undefined) {
+      errors.push(...this.validateType(project.autoJoin, 'boolean', 'autoJoin'))
+    }
+
+    if (project.owners !== undefined) {
+      errors.push(...this.validateArray(project.owners, 'string', 'owners'))
+    }
+
+    if (project.members !== undefined) {
+      errors.push(...this.validateArray(project.members, 'string', 'members'))
+    }
+
+    if (project.description !== undefined) {
+      errors.push(...this.validateType(project.description, 'string', 'description'))
+    }
+
+    if (!this.validateStringDefined(project.title)) {
+      errors.push('title is required')
+    }
+
+    if (project.class !== tracker.class.Project) {
+      errors.push('invalid class: ' + project.class)
+    }
+
+    if (project.defaultIssueStatus !== undefined && !this.issueStatusCache.has(project.defaultIssueStatus.name)) {
+      errors.push('defaultIssueStatus not found: ' + project.defaultIssueStatus.name)
+    }
+
+    errors.push(...this.validateProjectIdentifier(project.identifier))
+    return errors
+  }
+
+  private validateProjectIdentifier (identifier: string): string[] {
+    const errors: string[] = []
+    if (!this.validateStringDefined(identifier)) {
+      errors.push('identifier is required')
+      return errors
+    }
+    if (identifier.length > MAX_PROJECT_IDENTIFIER_LENGTH) {
+      errors.push(`identifier must be no longer than ${MAX_PROJECT_IDENTIFIER_LENGTH} characters`)
+    }
+    if (!PROJECT_IDENTIFIER_REGEX.test(identifier)) {
+      errors.push(
+        'identifier must contain only Latin letters, numbers, and underscores, and must not start with a number'
+      )
+    }
+    return errors
+  }
+
+  private validateTeamspace (teamspace: ImportTeamspace): string[] {
+    const errors: string[] = []
+
+    errors.push(...this.validateType(teamspace.title, 'string', 'title'))
+    errors.push(...this.validateType(teamspace.class, 'string', 'class'))
+
+    if (teamspace.private !== undefined) {
+      errors.push(...this.validateType(teamspace.private, 'boolean', 'private'))
+    }
+
+    if (teamspace.autoJoin !== undefined) {
+      errors.push(...this.validateType(teamspace.autoJoin, 'boolean', 'autoJoin'))
+    }
+
+    if (teamspace.owners !== undefined) {
+      errors.push(...this.validateArray(teamspace.owners, 'string', 'owners'))
+    }
+
+    if (teamspace.members !== undefined) {
+      errors.push(...this.validateArray(teamspace.members, 'string', 'members'))
+    }
+
+    if (teamspace.description !== undefined) {
+      errors.push(...this.validateType(teamspace.description, 'string', 'description'))
+    }
+
+    if (!this.validateStringDefined(teamspace.title)) {
+      errors.push('title is required')
+    }
+    if (teamspace.class !== document.class.Teamspace) {
+      errors.push('invalid class: ' + teamspace.class)
+    }
+    return errors
+  }
+
+  private validateIssue (issue: ImportIssue): string[] {
+    const errors: string[] = []
+
+    errors.push(...this.validateType(issue.title, 'string', 'title'))
+    errors.push(...this.validateType(issue.class, 'string', 'class'))
+
+    if (issue.number !== undefined) {
+      errors.push(...this.validateType(issue.number, 'number', 'number'))
+    }
+
+    if (issue.estimation !== undefined) {
+      errors.push(...this.validateType(issue.estimation, 'number', 'estimation'))
+    }
+
+    if (issue.remainingTime !== undefined) {
+      errors.push(...this.validateType(issue.remainingTime, 'number', 'remainingTime'))
+    }
+
+    if (issue.priority !== undefined) {
+      errors.push(...this.validateType(issue.priority, 'string', 'priority'))
+    }
+
+    if (issue.assignee !== undefined) {
+      errors.push(...this.validateType(issue.assignee, 'string', 'assignee'))
+    }
+
+    if (issue.status == null) {
+      errors.push('status is required: ')
+    } else if (!this.issueStatusCache.has(issue.status.name)) {
+      errors.push('status not found: ' + issue.status.name)
+    }
+    if (issue.priority != null && IssuePriority[issue.priority as keyof typeof IssuePriority] === undefined) {
+      errors.push('priority not found: ' + issue.priority)
+    }
+    if (issue.class !== tracker.class.Issue) {
+      errors.push('invalid class: ' + issue.class)
+    }
+    if (issue.number !== undefined && !this.validatePossitiveNumber(issue.number)) {
+      errors.push('invalid issue number: ' + issue.number)
+    }
+    if (issue.estimation != null && !this.validatePossitiveNumber(issue.estimation)) {
+      errors.push('invalid estimation: ' + issue.estimation)
+    }
+    if (issue.remainingTime != null && !this.validatePossitiveNumber(issue.remainingTime)) {
+      errors.push('invalid remaining time: ' + issue.remainingTime)
+    }
+    if (issue.comments != null && issue.comments.length > 0) {
+      for (const comment of issue.comments) {
+        if (comment.author == null) {
+          errors.push('comment author is required')
+        }
+        if (!this.validateStringDefined(comment.text)) {
+          errors.push('comment text is required')
+        }
+      }
+    }
+    return errors
+  }
+
+  private validatePossitiveNumber (value: any): boolean {
+    return typeof value === 'number' && !Number.isNaN(value) && value >= 0
+  }
+
+  private validateStringDefined (value: string | null | undefined): boolean {
+    return typeof value === 'string' && value !== '' && value !== null && value !== undefined
+  }
+
+  private validateDocument (doc: ImportDocument): string[] {
+    const errors: string[] = []
+    if (!this.validateStringDefined(doc.title)) {
+      errors.push('title is required')
+    }
+    if (doc.class !== document.class.Document) {
+      errors.push('invalid class: ' + doc.class)
+    }
+    return errors
+  }
+
+  private validateProjectReferences (): void {
+    // Validate project type references
+    for (const project of this.projects.values()) {
+      if (project.projectType !== undefined && !this.projectTypes.has(project.projectType.name)) {
+        this.addError(project.title, `Referenced project type ${project.projectType.name} not found`)
+      }
+    }
+  }
+
+  private validateSpaceDocuments (): void {
+    // Validate that issues belong to projects and documents to teamspaces
+    for (const projectPath of this.issuesByProject.keys()) {
+      if (!this.projects.has(projectPath)) {
+        this.addError(projectPath, 'Issues reference non-existent project')
+      }
+    }
+
+    for (const [teamspacePath] of this.documentsByTeamspace) {
+      if (!this.teamspaces.has(teamspacePath)) {
+        this.addError(teamspacePath, 'Documents reference non-existent teamspace')
+      }
+    }
+  }
+
+  private addError (path: string, error: string): void {
+    this.errors.set(path, { path, error })
+  }
+
+  private buildDocumentHierarchy (docPath: string, allDocs: Map<string, ImportDocument>): void {
+    const doc = allDocs.get(docPath)
+    if (doc === undefined) return
+
+    const childDocs = Array.from(allDocs.entries())
+      .filter(([childPath]) => this.documentParents.get(childPath) === docPath)
+      .map(([childPath, childDoc]) => {
+        this.buildDocumentHierarchy(childPath, allDocs)
+        return childDoc
+      })
+
+    doc.subdocs = childDocs
+  }
+
+  private buildIssueHierarchy (issuePath: string, allIssues: Map<string, ImportIssue>): void {
+    const issue = allIssues.get(issuePath)
+    if (issue === undefined) return
+
+    const childIssues = Array.from(allIssues.entries())
+      .filter(([childPath]) => this.issueParents.get(childPath) === issuePath)
+      .map(([childPath, childIssue]) => {
+        this.buildIssueHierarchy(childPath, allIssues)
+        return childIssue
+      })
+
+    issue.subdocs = childIssues
+  }
+
+  private validateType (value: unknown, type: 'string' | 'number' | 'boolean', fieldName: string): string[] {
+    const errors: string[] = []
+    switch (type) {
+      case 'string':
+        if (typeof value !== 'string') {
+          errors.push(`${fieldName} must be string, got ${typeof value}`)
+        }
+        break
+      case 'number':
+        if (typeof value !== 'number') {
+          errors.push(`${fieldName} must be number, got ${typeof value}`)
+        }
+        break
+      case 'boolean':
+        if (typeof value !== 'boolean') {
+          errors.push(`${fieldName} must be boolean, got ${typeof value}`)
+        }
+        break
+    }
+    return errors
+  }
+
+  private validateArray (value: unknown, itemType: 'string' | 'number' | 'boolean', fieldName: string): string[] {
+    const errors: string[] = []
+    if (!Array.isArray(value)) {
+      errors.push(`${fieldName} must be an array`)
+      return errors
+    }
+
+    for (let i = 0; i < value.length; i++) {
+      switch (itemType) {
+        case 'string':
+          if (typeof value[i] !== 'string') {
+            errors.push(`${fieldName}[${i}] must be string, got ${typeof value[i]}`)
+          }
+          break
+        case 'number':
+          if (typeof value[i] !== 'number') {
+            errors.push(`${fieldName}[${i}] must be number, got ${typeof value[i]}`)
+          }
+          break
+        case 'boolean':
+          if (typeof value[i] !== 'boolean') {
+            errors.push(`${fieldName}[${i}] must be boolean, got ${typeof value[i]}`)
+          }
+          break
+      }
+    }
+    return errors
+  }
+}

dev/import-tool/src/importer/dowloader.ts

@@ -1,3 +1,17 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 export async function download (url: string): Promise<Blob | null> {
   try {
     const response = await fetch(url)

dev/import-tool/src/importer/importer.ts

@@ -19,6 +19,7 @@ import { type Person } from '@hcengineering/contact'
 import core, {
   type Account,
   type AttachedData,
+  type Class,
   type CollaborativeDoc,
   type Data,
   type Doc,
@@ -28,6 +29,7 @@ import core, {
   type Mixin,
   type Ref,
   SortingOrder,
+  type Space,
   type Status,
   type Timestamp,
   type TxOperations
@@ -40,7 +42,7 @@ import task, {
   type TaskType,
   type TaskTypeWithFactory
 } from '@hcengineering/task'
-import { jsonToMarkup, jsonToYDocNoSchema, type MarkupNode, parseMessageMarkdown } from '@hcengineering/text'
+import { jsonToMarkup, jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text'
 import tracker, {
   type Issue,
   type IssueParentInfo,
@@ -49,17 +51,13 @@ import tracker, {
   type Project,
   TimeReportDayType
 } from '@hcengineering/tracker'
+import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor'
 import { type FileUploader, type UploadResult } from './uploader'
 
 export interface ImportWorkspace {
-  persons?: ImportPerson[]
   projectTypes?: ImportProjectType[]
   spaces?: ImportSpace<ImportDoc>[]
-}
+  attachments?: ImportAttachment[]
-
-export interface ImportPerson {
-  name: string
-  email: string
 }
 
 export interface ImportProjectType {
@@ -80,45 +78,47 @@ export interface ImportStatus {
 }
 
 export interface ImportSpace<T extends ImportDoc> {
-  class: string
+  class: Ref<Class<Space>>
-  name: string
+  title: string
+  private: boolean
+  autoJoin?: boolean
   description?: string
-
+  owners?: Ref<Account>[]
+  members?: Ref<Account>[]
   docs: T[]
 }
 export interface ImportDoc {
-  class: string
+  id?: Ref<Doc>
+  class: Ref<Class<Doc<Space>>>
   title: string
   descrProvider: () => Promise<string>
-
   subdocs: ImportDoc[]
 }
 
 export interface ImportTeamspace extends ImportSpace<ImportDocument> {
-  class: 'document.class.TeamSpace'
+  class: Ref<Class<Teamspace>>
 }
 
 export interface ImportDocument extends ImportDoc {
-  class: 'document.class.Document'
+  id?: Ref<Document>
+  class: Ref<Class<Document>>
   subdocs: ImportDocument[]
 }
 
 export interface ImportProject extends ImportSpace<ImportIssue> {
-  class: 'tracker.class.Project'
+  class: Ref<Class<Project>>
   identifier: string
-  private: boolean
+  projectType?: ImportProjectType
-  autoJoin: boolean
-  projectType: ImportProjectType
-  defaultAssignee?: ImportPerson
   defaultIssueStatus?: ImportStatus
-  owners?: ImportPerson[]
-  members?: ImportPerson[]
   description?: string
 }
 
 export interface ImportIssue extends ImportDoc {
-  class: 'tracker.class.Issue'
+  id?: Ref<Issue>
+  class: Ref<Class<Issue>>
   status: ImportStatus
+  priority?: string
+  number?: number
   assignee?: Ref<Person>
   estimation?: number
   remainingTime?: number
@@ -133,16 +133,15 @@ export interface ImportComment {
 }
 
 export interface ImportAttachment {
+  id?: Ref<Attachment>
   title: string
   blobProvider: () => Promise<Blob | null>
-}
+  parentId?: Ref<Doc>
-
+  parentClass?: Ref<Class<Doc<Space>>>
-export interface MarkdownPreprocessor {
+  spaceId?: Ref<Space>
-  process: (json: MarkupNode) => MarkupNode
 }
 
 export class WorkspaceImporter {
-  private readonly personsByName = new Map<string, Ref<Person>>()
   private readonly issueStatusByName = new Map<string, Ref<IssueStatus>>()
   private readonly projectTypeByName = new Map<string, Ref<ProjectType>>()
 
@@ -150,23 +149,13 @@ export class WorkspaceImporter {
     private readonly client: TxOperations,
     private readonly fileUploader: FileUploader,
     private readonly workspaceData: ImportWorkspace,
-    private readonly preprocessor: MarkdownPreprocessor
+    private readonly preprocessor: MarkdownPreprocessor = new NoopMarkdownPreprocessor()
   ) {}
 
   public async performImport (): Promise<void> {
-    await this.importPersons()
     await this.importProjectTypes()
     await this.importSpaces()
-  }
+    await this.importAttachments()
-
-  private async importPersons (): Promise<void> {
-    if (this.workspaceData.persons === undefined) return
-
-    for (const person of this.workspaceData.persons) {
-      const personId = generateId<Person>()
-      this.personsByName.set(person.name, personId)
-      // TODO: Implement person creation
-    }
   }
 
   private async importProjectTypes (): Promise<void> {
@@ -182,14 +171,29 @@ export class WorkspaceImporter {
     if (this.workspaceData.spaces === undefined) return
 
     for (const space of this.workspaceData.spaces) {
-      if (space.class === 'document.class.TeamSpace') {
+      if (space.class === document.class.Teamspace) {
         await this.importTeamspace(space as ImportTeamspace)
-      } else if (space.class === 'tracker.class.Project') {
+      } else if (space.class === tracker.class.Project) {
         await this.importProject(space as ImportProject)
       }
     }
   }
 
+  private async importAttachments (): Promise<void> {
+    if (this.workspaceData.attachments === undefined) return
+
+    for (const attachment of this.workspaceData.attachments) {
+      if (
+        attachment.parentId === undefined ||
+        attachment.parentClass === undefined ||
+        attachment.spaceId === undefined
+      ) {
+        throw new Error('Attachment is missing parentId, parentClass or spaceId')
+      }
+      await this.importAttachment(attachment.parentId, attachment.parentClass, attachment, attachment.spaceId)
+    }
+  }
+
   async createProjectTypeWithTaskTypes (projectType: ImportProjectType): Promise<Ref<ProjectType>> {
     const taskTypes: TaskTypeWithFactory[] = []
     if (projectType.taskTypes !== undefined) {
@@ -230,7 +234,9 @@ export class WorkspaceImporter {
   }
 
   async importTeamspace (space: ImportTeamspace): Promise<Ref<Teamspace>> {
+    console.log('Creating teamspace: ', space.title)
     const teamspaceId = await this.createTeamspace(space)
+    console.log('Teamspace created: ', teamspaceId)
     for (const doc of space.docs) {
       await this.createDocumentWithSubdocs(doc, document.ids.NoParent, teamspaceId)
     }
@@ -242,7 +248,9 @@ export class WorkspaceImporter {
     parentId: Ref<Document>,
     teamspaceId: Ref<Teamspace>
   ): Promise<Ref<Document>> {
+    console.log('Creating document: ', doc.title)
     const documentId = await this.createDocument(doc, parentId, teamspaceId)
+    console.log('Document created: ', documentId)
     for (const child of doc.subdocs) {
       await this.createDocumentWithSubdocs(child, documentId, teamspaceId)
     }
@@ -254,12 +262,12 @@ export class WorkspaceImporter {
     const data = {
       type: document.spaceType.DefaultTeamspaceType,
       description: space.description ?? '',
-      title: space.name,
+      title: space.title,
-      name: space.name,
+      name: space.title,
-      private: false,
+      private: space.private,
-      members: [],
+      owners: space.owners ?? [],
-      owners: [],
+      members: space.members ?? [],
-      autoJoin: false,
+      autoJoin: space.autoJoin,
       archived: false
     }
     await this.client.createDoc(document.class.Teamspace, core.space.Space, data, teamspaceId)
@@ -271,9 +279,9 @@ export class WorkspaceImporter {
     parentId: Ref<Document>,
     teamspaceId: Ref<Teamspace>
   ): Promise<Ref<Document>> {
-    const id = generateId<Document>()
+    const id = doc.id ?? generateId<Document>()
     const content = await doc.descrProvider()
-    const collabId = await this.createCollaborativeContent(id, 'content', content)
+    const collabId = await this.createCollaborativeContent(id, 'content', content, teamspaceId)
 
     const lastRank = await getFirstRank(this.client, teamspaceId, parentId)
     const rank = makeRank(lastRank, undefined)
@@ -295,7 +303,7 @@ export class WorkspaceImporter {
   }
 
   async importProject (project: ImportProject): Promise<Ref<Project>> {
-    console.log('Create project: ', project.name)
+    console.log('Creating project: ', project.title)
     const projectId = await this.createProject(project)
     console.log('Project created: ' + projectId)
 
@@ -316,7 +324,7 @@ export class WorkspaceImporter {
     project: Project,
     parentsInfo: IssueParentInfo[]
   ): Promise<{ id: Ref<Issue>, identifier: string }> {
-    console.log('Create issue: ', issue.title)
+    console.log('Creating issue: ', issue.title)
     const issueResult = await this.createIssue(issue, project, parentId, parentsInfo)
     console.log('Issue created: ', issueResult)
 
@@ -341,23 +349,29 @@ export class WorkspaceImporter {
 
   async createProject (project: ImportProject): Promise<Ref<Project>> {
     const projectId = generateId<Project>()
-    const projectType = this.projectTypeByName.get(project.projectType.name)
+
+    const projectType =
+      project.projectType !== undefined
+        ? this.projectTypeByName.get(project.projectType.name)
+        : tracker.ids.ClassingProjectType
+
     const defaultIssueStatus =
       project.defaultIssueStatus !== undefined
         ? this.issueStatusByName.get(project.defaultIssueStatus.name)
         : tracker.status.Backlog
+
     const identifier = await this.uniqueProjectIdentifier(project.identifier)
     const projectData = {
-      name: project.name,
+      name: project.title,
       description: project.description ?? '',
       private: project.private,
-      members: [],
+      members: project.members ?? [],
-      owners: [],
+      owners: project.owners ?? [],
       archived: false,
       autoJoin: project.autoJoin,
       identifier,
       sequence: 0,
-      defaultIssueStatus: defaultIssueStatus ?? tracker.status.Backlog, // todo: test with no status
+      defaultIssueStatus: defaultIssueStatus ?? tracker.status.Backlog,
       defaultTimeReportDay: TimeReportDayType.PreviousWorkDay,
       type: projectType as Ref<ProjectType>
     }
@@ -375,14 +389,22 @@ export class WorkspaceImporter {
     parentId: Ref<Issue>,
     parentsInfo: IssueParentInfo[]
   ): Promise<{ id: Ref<Issue>, identifier: string }> {
-    const issueId = generateId<Issue>()
+    const issueId = issue.id ?? generateId<Issue>()
     const content = await issue.descrProvider()
-    const collabId = await this.createCollaborativeContent(issueId, 'description', content)
+    const collabId = await this.createCollaborativeContent(issueId, 'description', content, project._id)
+
+    const { number, identifier } =
+      issue.number !== undefined
+        ? { number: issue.number, identifier: `${project.identifier}-${issue.number}` }
+        : await this.getNextIssueIdentifier(project)
 
-    const { number, identifier } = await this.getNextIssueIdentifier(project)
     const kind = await this.getIssueKind(project)
     const rank = await this.getIssueRank(project)
     const status = await this.findIssueStatusByName(issue.status.name)
+    const priority =
+      issue.priority !== undefined
+        ? IssuePriority[issue.priority as keyof typeof IssuePriority]
+        : IssuePriority.NoPriority
 
     const estimation = issue.estimation ?? 0
     const remainingTime = issue.remainingTime ?? 0
@@ -395,7 +417,7 @@ export class WorkspaceImporter {
       component: null,
       number,
       status,
-      priority: IssuePriority.NoPriority,
+      priority,
       rank,
       comments: issue.comments?.length ?? 0,
       subIssues: issue.subdocs.length,
@@ -469,7 +491,7 @@ export class WorkspaceImporter {
 
   async createComment (issueId: Ref<Issue>, comment: ImportComment, projectId: Ref<Project>): Promise<void> {
     const json = parseMessageMarkdown(comment.text ?? '', 'image://')
-    const processedJson = this.preprocessor.process(json)
+    const processedJson = this.preprocessor.process(json, issueId, projectId)
     const markup = jsonToMarkup(processedJson)
 
     const value: AttachedData<ChatMessage> = {
@@ -491,71 +513,86 @@ export class WorkspaceImporter {
     )
 
     if (comment.attachments !== undefined) {
-      await this.importAttachments(commentId, comment.attachments, projectId)
+      for (const attachment of comment.attachments) {
+        await this.importAttachment(commentId, chunter.class.ChatMessage, attachment, projectId)
+      }
     }
   }
 
-  private async importAttachments (
+  private async importAttachment (
-    commentId: Ref<ChatMessage>,
+    parentId: Ref<Doc>,
-    attachments: ImportAttachment[],
+    parentClass: Ref<Class<Doc<Space>>>,
-    projectId: Ref<Project>
+    attachment: ImportAttachment,
+    spaceId: Ref<Space>
   ): Promise<void> {
-    for (const attach of attachments) {
+    const blob = await attachment.blobProvider()
-      const blob = await attach.blobProvider()
+    if (blob === null) {
-      if (blob === null) {
+      console.warn('Failed to read attachment file: ', attachment.title)
-        console.warn('Failed to download attachment file: ', attach.title)
+      return
-        continue
+    }
-      }
 
-      const attachmentId = await this.createAttachment(blob, attach, projectId, commentId)
+    const file = new File([blob], attachment.title)
-      if (attachmentId === null) {
+    const attachmentId = await this.createAttachment(
-        console.warn('Failed to upload attachment file: ', attach.title)
+      attachment.id ?? generateId<Attachment>(),
-      }
+      file,
+      spaceId,
+      parentId,
+      parentClass
+    )
+    if (attachmentId === null) {
+      console.warn('Failed to upload attachment file: ', attachment.title)
     }
   }
 
   private async createAttachment (
-    blob: Blob,
+    id: Ref<Attachment>,
-    attach: ImportAttachment,
+    file: File,
-    projectId: Ref<Project>,
+    spaceId: Ref<Space>,
-    commentId: Ref<ChatMessage>
+    parentId: Ref<Doc>,
+    parentClass: Ref<Class<Doc<Space>>>
   ): Promise<Ref<Attachment> | null> {
-    const attachmentId = generateId<Attachment>()
+    const response = await this.fileUploader.uploadFile(id, id, file)
-    const file = new File([blob], attach.title)
+    if (response.status !== 200) {
-
+      return null
-    const response = await this.fileUploader.uploadFile(attachmentId, attach.title, file)
-    if (response.status === 200) {
-      const responseText = await response.text()
-      if (responseText !== undefined) {
-        const uploadResult = JSON.parse(responseText) as UploadResult[]
-        if (!Array.isArray(uploadResult) || uploadResult.length === 0) {
-          return null
-        }
-
-        await this.client.addCollection(
-          attachment.class.Attachment,
-          projectId,
-          commentId,
-          chunter.class.ChatMessage,
-          'attachments',
-          {
-            file: uploadResult[0].id,
-            lastModified: Date.now(),
-            name: file.name,
-            size: file.size,
-            type: file.type
-          },
-          attachmentId
-        )
-      }
     }
-    return attachmentId
+
+    const responseText = await response.text()
+    if (responseText === undefined) {
+      return null
+    }
+
+    const uploadResult = JSON.parse(responseText) as UploadResult[]
+    if (!Array.isArray(uploadResult) || uploadResult.length === 0) {
+      return null
+    }
+
+    await this.client.addCollection(
+      attachment.class.Attachment,
+      spaceId,
+      parentId,
+      parentClass,
+      'attachments',
+      {
+        file: uploadResult[0].id,
+        lastModified: Date.now(),
+        name: file.name,
+        size: file.size,
+        type: file.type
+      },
+      id
+    )
+    return id
   }
 
   // Collaborative content handling
-  private async createCollaborativeContent (id: Ref<Doc>, field: string, content: string): Promise<CollaborativeDoc> {
+  private async createCollaborativeContent (
+    id: Ref<Doc>,
+    field: string,
+    content: string,
+    spaceId: Ref<Space>
+  ): Promise<CollaborativeDoc> {
     const json = parseMessageMarkdown(content ?? '', 'image://')
-    const processedJson = this.preprocessor.process(json)
+    const processedJson = this.preprocessor.process(json, id, spaceId)
     const collabId = makeCollaborativeDoc(id, 'description')
 
     const yDoc = jsonToYDocNoSchema(processedJson, field)
@@ -568,8 +605,7 @@ export class WorkspaceImporter {
   async findIssueStatusByName (name: string): Promise<Ref<IssueStatus>> {
     const query: DocumentQuery<Status> = {
       name,
-      ofAttribute: tracker.attribute.IssueStatus,
+      ofAttribute: tracker.attribute.IssueStatus
-      category: task.statusCategory.Active
     }
 
     const status = await this.client.findOne(tracker.class.IssueStatus, query)

dev/import-tool/src/importer/preprocessor.ts

@@ -0,0 +1,119 @@
+//
+// Copyright © 2024 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+import contact, { type Person } from '@hcengineering/contact'
+import { type Doc, type Space, type Ref } from '@hcengineering/core'
+import { type MarkupNode, MarkupNodeType } from '@hcengineering/text'
+
+export interface MarkdownPreprocessor {
+  process: (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>) => MarkupNode
+}
+
+export class NoopMarkdownPreprocessor implements MarkdownPreprocessor {
+  process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
+    return json
+  }
+}
+
+export abstract class BaseMarkdownPreprocessor implements MarkdownPreprocessor {
+  protected readonly MENTION_REGEX = /@([\p{L}\p{M}]+ [\p{L}\p{M}]+)/gu
+
+  constructor (protected readonly personsByName: Map<string, Ref<Person>>) {}
+
+  abstract process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode
+
+  protected processMentions (node: MarkupNode): void {
+    if (node.type !== MarkupNodeType.paragraph || node.content === undefined) return
+
+    const newContent: MarkupNode[] = []
+    for (const childNode of node.content) {
+      if (childNode.type === MarkupNodeType.text && childNode.text !== undefined) {
+        this.processMentionTextNode(childNode, newContent)
+      } else {
+        newContent.push(childNode)
+      }
+    }
+    node.content = newContent
+  }
+
+  protected processMentionTextNode (node: MarkupNode, newContent: MarkupNode[]): void {
+    if (node.text === undefined) return
+
+    let match
+    let lastIndex = 0
+    let hasMentions = false
+
+    while ((match = this.MENTION_REGEX.exec(node.text)) !== null) {
+      hasMentions = true
+      this.addTextBeforeMention(newContent, node, lastIndex, match.index)
+      this.addMentionNode(newContent, match[1], node)
+      lastIndex = this.MENTION_REGEX.lastIndex
+    }
+
+    if (hasMentions) {
+      this.addRemainingText(newContent, node, lastIndex)
+    } else {
+      newContent.push(node)
+    }
+  }
+
+  protected addTextBeforeMention (
+    newContent: MarkupNode[],
+    node: MarkupNode,
+    lastIndex: number,
+    matchIndex: number
+  ): void {
+    if (node.text === undefined) return
+    if (matchIndex > lastIndex) {
+      newContent.push({
+        type: MarkupNodeType.text,
+        text: node.text.slice(lastIndex, matchIndex),
+        marks: node.marks,
+        attrs: node.attrs
+      })
+    }
+  }
+
+  protected addMentionNode (newContent: MarkupNode[], name: string, originalNode: MarkupNode): void {
+    const personRef = this.personsByName.get(name)
+    if (personRef !== undefined) {
+      newContent.push({
+        type: MarkupNodeType.reference,
+        attrs: {
+          id: personRef,
+          label: name,
+          objectclass: contact.class.Person
+        }
+      })
+    } else {
+      newContent.push({
+        type: MarkupNodeType.text,
+        text: `@${name}`,
+        marks: originalNode.marks,
+        attrs: originalNode.attrs
+      })
+    }
+  }
+
+  protected addRemainingText (newContent: MarkupNode[], node: MarkupNode, lastIndex: number): void {
+    if (node.text !== undefined && lastIndex < node.text.length) {
+      newContent.push({
+        type: MarkupNodeType.text,
+        text: node.text.slice(lastIndex),
+        marks: node.marks,
+        attrs: node.attrs
+      })
+    }
+  }
+}

dev/import-tool/src/index.ts

@@ -24,6 +24,7 @@ import { importNotion } from './notion/notion'
 import { setMetadata } from '@hcengineering/platform'
 import { FrontFileUploader, type FileUploader } from './importer/uploader'
 import { ClickupImporter } from './clickup/clickup'
+import { UnifiedFormatImporter } from './huly/unified'
 
 /**
  * @public
@@ -131,5 +132,20 @@ export function importTool (): void {
       })
     })
 
+  // import /home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace --workspace ws1 --user user1 --password 1234
+  program
+    .command('import <dir>')
+    .description('import issues in Unified Huly Format')
+    .requiredOption('-u, --user <user>', 'user')
+    .requiredOption('-pw, --password <password>', 'password')
+    .requiredOption('-ws, --workspace <workspace>', 'workspace url where the documents should be imported to')
+    .action(async (dir: string, cmd) => {
+      const { workspace, user, password } = cmd
+      await authorize(user, password, workspace, async (client, uploader) => {
+        const importer = new UnifiedFormatImporter(client, uploader)
+        await importer.importFolder(dir)
+      })
+    })
+
   program.parse(process.argv)
 }

models/tracker/src/index.ts

@@ -772,7 +772,7 @@ function defineSpaceType (builder: Builder): void {
     task.class.TaskType,
     core.space.Model,
     {
-      parent: tracker.ids.ClassingProjectType,
+      parent: pluginState.ids.ClassingProjectType,
       statuses: classicStatuses,
       descriptor: tracker.descriptors.Issue,
       name: 'Issue',
@@ -800,6 +800,6 @@ function defineSpaceType (builder: Builder): void {
       statuses: classicStatuses.map((s) => ({ _id: s, taskType: tracker.taskTypes.Issue })),
       targetClass: tracker.mixin.ClassicProjectTypeData
     },
-    tracker.ids.ClassingProjectType
+    pluginState.ids.ClassingProjectType
   )
 }

models/tracker/src/migration.ts

@@ -39,11 +39,16 @@ import { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_TASK, migrateDefaultStatusesBase } from '@hcengineering/model-task'
 import tags from '@hcengineering/tags'
 import task from '@hcengineering/task'
-import { type Issue, type IssueStatus, type Project, TimeReportDayType, trackerId } from '@hcengineering/tracker'
+import tracker, {
+  type Issue,
+  type IssueStatus,
+  type Project,
+  TimeReportDayType,
+  trackerId
+} from '@hcengineering/tracker'
 
 import contact from '@hcengineering/model-contact'
 import { classicIssueTaskStatuses } from '.'
-import tracker from './plugin'
 
 async function createDefaultProject (tx: TxOperations): Promise<void> {
   const current = await tx.findOne(tracker.class.Project, {

models/tracker/src/plugin.ts

@@ -85,7 +85,6 @@ export default mergeIds(trackerId, tracker, {
     IssueTemplateChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
     ComponentChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
     MilestoneChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
-    ClassingProjectType: '' as Ref<ProjectType>,
     DefaultProjectType: '' as Ref<ProjectType>
   },
   completion: {

plugins/tracker/src/index.ts

@@ -36,6 +36,7 @@ import { Preference } from '@hcengineering/preference'
 import { TagCategory, TagElement, TagReference } from '@hcengineering/tags'
 import { ToDo } from '@hcengineering/time'
 import {
+  ProjectType,
   ProjectTypeDescriptor,
   Task,
   Project as TaskProject,
@@ -384,7 +385,8 @@ const pluginState = plugin(trackerId, {
   ids: {
     NoParent: '' as Ref<Issue>,
     IssueDraft: '',
-    IssueDraftChild: ''
+    IssueDraftChild: '',
+    ClassingProjectType: '' as Ref<ProjectType>
   },
   status: {
     Backlog: '' as Ref<Status>,