Support unified Huly data import format (#7191)
Some checks are pending
CI / build (push) Waiting to run
CI / svelte-check (push) Blocked by required conditions
CI / formatting (push) Blocked by required conditions
CI / test (push) Blocked by required conditions
CI / uitest (push) Waiting to run
CI / uitest-pg (push) Waiting to run
CI / uitest-qms (push) Waiting to run
CI / docker-build (push) Blocked by required conditions

Support unified Huly data import format

Signed-off-by: Anna Khismatullina <anna.khismatullina@gmail.com>
This commit is contained in:
Anna Khismatullina 2024-11-20 14:58:30 +07:00 committed by GitHub
parent b4bde9ceac
commit bcf7857477
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 1757 additions and 194 deletions

16
.vscode/launch.json vendored
View File

@ -582,7 +582,21 @@
"name": "Debug ClickUp import",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws5"],
"args": ["src/__start.ts", "import-clickup-tasks", "/home/anna/work/clickup/aleksandr/debug/mentions.csv", "-u", "user1", "-pw", "1234", "-ws", "ws10"],
"env": {
"FRONT_URL": "http://localhost:8087",
},
"runtimeVersion": "20",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"sourceMaps": true,
"outputCapture": "std",
"cwd": "${workspaceRoot}/dev/import-tool"
},
{
"name": "Debug Huly import",
"type": "node",
"request": "launch",
"args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"],
"env": {
"FRONT_URL": "http://localhost:8087"
},

View File

@ -23868,7 +23868,7 @@ packages:
dev: false
file:projects/import-tool.tgz:
resolution: {integrity: sha512-KD9EB2XXo43QYnx8xp4j9z62JGValTBx2uTS4Egw11rsdAoENIxNxI+68CIKASfY8DMKMMuTmUZ6N6nK/6XSiw==, tarball: file:projects/import-tool.tgz}
resolution: {integrity: sha512-0Q1/hHZxEdYFPr2qqfovlVJRA8JyvWsKtY3ubHrffnaAMbbN6BNWt6Jf+GzwyGeIwImLI2Oud2x/WqOFb/USdg==, tarball: file:projects/import-tool.tgz}
name: '@rush-temp/import-tool'
version: 0.0.0
dependencies:
@ -23876,6 +23876,7 @@ packages:
'@types/domhandler': 2.4.5
'@types/htmlparser2': 3.10.7
'@types/jest': 29.5.12
'@types/js-yaml': 4.0.9
'@types/mime-types': 2.1.4
'@types/minio': 7.0.18
'@types/node': 20.11.19
@ -23895,6 +23896,7 @@ packages:
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
htmlparser2: 9.1.0
jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2)
js-yaml: 4.1.0
mammoth: 1.8.0
mime-types: 2.1.35
prettier: 3.2.5

View File

@ -1,5 +1,25 @@
### Supported Import Options
# Huly Import Tool
Tool for importing data into Huly workspace.
## Recommended Import Method
### Unified Format Import
The recommended way to import data into Huly is using our [Unified Import Format](./src/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure.
See the [complete guide](./src/huly/README.md) and [example workspace](./src/huly/example-workspace) to get started.
### Why Use Unified Format?
- Simple, human-readable format using YAML and Markdown
- Flexible structure that can represent data from any system
- Easy to validate and fix data before import
- Can be generated automatically by script or prepared manually
## Direct Import Options
We also support direct import from some platforms:
1. **Notion**: see [Import from Notion Guide](./src/notion/README.md)
2. **ClickUp**: see [Import from ClickUp Guide](./src/clickup/README.md)
These direct imports are suitable for simple migrations, but for complex cases or systems not listed above, please use the Unified Format.

View File

@ -49,7 +49,8 @@
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"@types/jest": "^29.5.5",
"@types/csvtojson": "^2.0.0"
"@types/csvtojson": "^2.0.0",
"@types/js-yaml": "^4.0.9"
},
"dependencies": {
"@hcengineering/attachment": "^0.6.14",
@ -69,6 +70,7 @@
"csvtojson": "^2.0.10",
"@hcengineering/task": "^0.6.20",
"@hcengineering/contact": "^0.6.24",
"@hcengineering/chunter": "^0.6.20"
"@hcengineering/chunter": "^0.6.20",
"js-yaml": "^4.1.0"
}
}

View File

@ -15,6 +15,7 @@
import contact, { type Person, type PersonAccount } from '@hcengineering/contact'
import { type Ref, type Timestamp, type TxOperations } from '@hcengineering/core'
import { MarkupNodeType, traverseNode, type MarkupNode } from '@hcengineering/text'
import tracker from '@hcengineering/tracker'
import csv from 'csvtojson'
import { download } from '../importer/dowloader'
import {
@ -22,10 +23,10 @@ import {
type ImportComment,
type ImportIssue,
type ImportProject,
type ImportProjectType,
type MarkdownPreprocessor
type ImportProjectType
} from '../importer/importer'
import { type FileUploader } from '../importer/uploader'
import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
interface ClickupTask {
'Task ID': string
@ -61,77 +62,15 @@ interface ImportIssueEx extends ImportIssue {
clickupProjectName?: string
}
class ClickupMarkdownPreprocessor implements MarkdownPreprocessor {
private readonly MENTION_REGEX = /@([\p{L}\p{M}]+ [\p{L}\p{M}]+)/gu
constructor (private readonly personsByName: Map<string, Ref<Person>>) {}
class ClickupMarkdownPreprocessor extends BaseMarkdownPreprocessor {
process (json: MarkupNode): MarkupNode {
traverseNode(json, (node) => {
if (node.type === MarkupNodeType.paragraph && node.content !== undefined) {
const newContent: MarkupNode[] = []
for (const childNode of node.content) {
if (childNode.type === MarkupNodeType.text && childNode.text !== undefined) {
let match
let lastIndex = 0
let hasMentions = false
while ((match = this.MENTION_REGEX.exec(childNode.text)) !== null) {
hasMentions = true
if (match.index > lastIndex) {
newContent.push({
type: MarkupNodeType.text,
text: childNode.text.slice(lastIndex, match.index),
marks: childNode.marks,
attrs: childNode.attrs
})
}
const name = match[1]
const personRef = this.personsByName.get(name)
if (personRef !== undefined) {
newContent.push({
type: MarkupNodeType.reference,
attrs: {
id: personRef,
label: name,
objectclass: contact.class.Person
}
})
} else {
newContent.push({
type: MarkupNodeType.text,
text: match[0],
marks: childNode.marks,
attrs: childNode.attrs
})
}
lastIndex = this.MENTION_REGEX.lastIndex
}
if (hasMentions) {
if (lastIndex < childNode.text.length) {
newContent.push({
type: MarkupNodeType.text,
text: childNode.text.slice(lastIndex),
marks: childNode.marks,
attrs: childNode.attrs
})
}
} else {
newContent.push(childNode)
}
} else {
newContent.push(childNode)
}
}
node.content = newContent
if (node.type === MarkupNodeType.paragraph) {
this.processMentions(node)
return false
}
return true
})
return json
}
}
@ -206,8 +145,8 @@ class ClickupImporter {
for (const projectName of projects) {
const identifier = this.getProjectIdentifier(projectName)
importProjectsByName.set(projectName, {
class: 'tracker.class.Project',
name: projectName,
class: tracker.class.Project,
title: projectName,
identifier,
private: false,
autoJoin: false,
@ -280,7 +219,7 @@ class ClickupImporter {
}
return {
class: 'tracker.class.Issue',
class: tracker.class.Issue,
title: clickup['Task Name'],
descrProvider: () => {
return Promise.resolve(description)

View File

@ -0,0 +1,127 @@
## Import from Unified Format Guide
### Overview
The unified format represents workspace data in a hierarchical folder structure where:
* Root directory contains space configurations (*.yaml) and their corresponding folders
* Each space folder contains documents/issues (*.md) and their subdocuments/subissues
* Documents/issues can have child items in similarly-named folders
* File named `settings.yaml` is reserved and should not be used for spaces configuration
* Files without required `class` property in frontmatter will be skipped
See the complete working example in the [example workspace](./example-workspace).
### File Structure Example
```
workspace/
├── Documentation/
│ ├── Getting Started.md # Standalone document
│ ├── User Guide.md # Document with children
│ ├── User Guide/ # Child documents folder
│ │ ├── Installation.md
│ │ └── Configuration.md
│ └── files/ # Attachments
│ └── architecture.png
├── Documentation.yaml # Space configuration
├── Project Alpha/
│ ├── 1.Project Setup.md # Issue with subtasks
│ ├── 1.Project Setup/ # Subtasks folder
│ │ ├── 2.Configure CI.md
│ │ └── 3.Setup Tests.md
│ ├── 4.Update Docs.md # Standalone issue
│ └── files/
│ └── diagram.png # Can be referenced in markdown content
└── Project Alpha.yaml # Project configuration
```
### File Format Requirements
#### Space Configuration (*.yaml)
Project space (`Project Alpha.yaml`):
```yaml
class: tracker:class:Project # Required
title: Project Alpha # Required
identifier: ALPHA # Required, max 5 uppercase letters/numbers, must start with a letter
private: false # Optional, default: false
autoJoin: true # Optional, default: true
owners: # Optional, list of email addresses
- john.doe@example.com
members: # Optional, list of email addresses
- joe.shmoe@example.com
description: string # Optional
defaultIssueStatus: Todo # Optional
```
Teamspace (`Documentation.yaml`):
```yaml
class: document:class:Teamspace # Required
title: Documentation # Required
private: false # Optional, default: false
autoJoin: true # Optional, default: true
owners: # Optional, list of email addresses
- john.doe@example.com
members: # Optional, list of email addresses
- joe.shmoe@example.com
description: string # Optional
```
#### Documents and Issues (*.md)
All files must include YAML frontmatter followed by Markdown content:
Document (`Getting Started.md`):
```yaml
---
class: document:class:Document # Required
title: Getting Started Guide # Required
---
# Content in Markdown format
```
Issue (`1.Project Setup.md`):
```yaml
---
class: tracker:class:Issue # Required
title: Project Setup # Required
status: In Progress # Required
priority: High # Optional
assignee: John Smith # Optional
estimation: 8 # Optional, in hours
remainingTime: 4 # Optional, in hours
---
Task description in Markdown...
```
### Task Identification
* Human-readable task ID is formed by combining project's identifier and task number from filename
* Example: For project with identifier "ALPHA" and task "1.Setup Project.md", the task ID will be "ALPHA-1"
### Allowed Values
Issue status values:
* `Backlog`
* `Todo`
* `In Progress`
* `Done`
* `Canceled`
Issue priority values:
* `Low`
* `Medium`
* `High`
* `Urgent`
### Run Import Tool
```bash
docker run \
-e FRONT_URL="https://huly.app" \
-v /path/to/workspace:/data \
hardcoreeng/import-tool:latest \
-- bundle.js import /data \
--user your.email@company.com \
--password yourpassword \
--workspace workspace-id
```
### Limitations
* All users must exist in the system before import
* Assignees are mapped by full name
* Files in space directories can be used as attachments when referenced in markdown content

View File

@ -0,0 +1,9 @@
class: document:class:Teamspace
title: Documentation
private: false
autoJoin: true
owners:
- john.doe@example.com
members:
- joe.shmoe@example.com
description: Technical documentation and guides

View File

@ -0,0 +1,19 @@
---
class: document:class:Document
title: Getting Started Guide
---
# Getting Started
Welcome to our project! This guide will help you get started with development.
## Setup Steps
1. Clone the repository
2. Install dependencies
3. Set up your environment
## Project Communication
We use Huly for all project communication:
- Team discussions in Virtual Office
- Technical discussions in issue comments
- Documentation in Huly Documents

View File

@ -0,0 +1,16 @@
---
class: document:class:Document
title: User Guide
---
# User Guide
Our platform architecture and key components.
## System Overview
<img src="./files/architecture.png" width="800"/>
## Development Workflow
- Code reviews via GitHub integration
- CI/CD status in Huly Activity Feed
- Team sync-ups in Huly Virtual Office

View File

@ -0,0 +1,19 @@
---
class: document:class:Document
title: Installation Guide
---
# Installation
## System Requirements
- Node.js 18 or higher
- Docker Desktop
- Git
## Setup Steps
1. Clone the repository
2. Install dependencies
3. Configure your environment
## Need Help?
Contact @Joe Shmoe for technical support

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

View File

@ -0,0 +1,11 @@
class: tracker:class:Project
title: Project Alpha
identifier: ALPHA
private: false
autoJoin: true
owners:
- john.doe@example.com
members:
- joe.shmoe@example.com
description: Main development project
defaultIssueStatus: Todo

View File

@ -0,0 +1,28 @@
---
class: tracker:class:Issue
title: Project Setup
status: In Progress
priority: High
assignee: John Doe
estimation: 8
remainingTime: 4
comments:
- author: john.doe@example.com
text: |
Initial infrastructure is ready!
- author: joe.shmoe@example.com
text: |
Great! I'll start working on [Configure CI](./1.Project%20Setup/2.Configure%20CI.md) task now.
- author: john.doe@example.com
text: |
Perfect, don't forget to update [documentation](../Documentation/User%20Guide/Installation.md) when you're done.
---
**Initial project infrastructure setup.**
Tasks:
1. Repository setup
2. Configure GitHub integration with Huly
3. Set up CI/CD pipeline using [configuration template](./files/config.yaml)
4. Configure team access and permissions
Daily sync-ups in Huly Virtual Office at 10:00 AM UTC.

View File

@ -0,0 +1,13 @@
---
class: tracker:class:Issue
title: Configure CI Pipeline
status: Todo
priority: High
assignee: Joe Shmoe
estimation: 4
---
Set up CI pipeline with GitHub integration:
- Configure GitHub Actions
- Set up test automation
- Add build status notifications to Huly
- Configure deployment workflow

View File

@ -0,0 +1,11 @@
---
class: tracker:class:Issue
title: Update Documentation
status: Backlog
priority: Medium
---
Update project documentation in Huly:
- Complete installation guide
- Add troubleshooting section
- Document GitHub integration setup
- Review in next team meeting

View File

@ -0,0 +1,18 @@
# Development environment configuration
server:
port: 3000
host: localhost
database:
host: localhost
port: 5432
name: project_alpha
user: postgres
github:
webhook_url: https://api.github.com/webhooks
branch: master
logging:
level: debug
format: json

View File

@ -0,0 +1,625 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Attachment } from '@hcengineering/attachment'
import contact, { type Person, type PersonAccount } from '@hcengineering/contact'
import { type Class, type Doc, generateId, type Ref, type Space, type TxOperations } from '@hcengineering/core'
import document, { type Document } from '@hcengineering/document'
import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text'
import tracker, { type Issue } from '@hcengineering/tracker'
import * as fs from 'fs'
import * as yaml from 'js-yaml'
import { contentType } from 'mime-types'
import * as path from 'path'
import { ImportWorkspaceBuilder } from '../importer/builder'
import {
type ImportAttachment,
type ImportComment,
type ImportDocument,
type ImportIssue,
type ImportProject,
type ImportProjectType,
type ImportTeamspace,
type ImportWorkspace,
WorkspaceImporter
} from '../importer/importer'
import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
import { type FileUploader } from '../importer/uploader'
interface UnifiedComment {
author: string
text: string
}
interface UnifiedIssueHeader {
class: 'tracker:class:Issue'
title: string
status: string
assignee?: string
priority?: string
estimation?: number // in hours
remainingTime?: number // in hours
comments?: UnifiedComment[]
}
interface UnifiedSpaceSettings {
class: 'tracker:class:Project' | 'document:class:Teamspace'
title: string
private?: boolean
autoJoin?: boolean
owners?: string[]
members?: string[]
description?: string
}
interface UnifiedProjectSettings extends UnifiedSpaceSettings {
class: 'tracker:class:Project'
identifier: string
projectType?: string
defaultIssueStatus?: string
}
interface UnifiedTeamspaceSettings extends UnifiedSpaceSettings {
class: 'document:class:Teamspace'
}
interface UnifiedDocumentHeader {
class: 'document:class:Document'
title: string
}
interface UnifiedWorkspaceSettings {
projectTypes?: Array<{
name: string
taskTypes?: Array<{
name: string
description?: string
statuses: Array<{
name: string
description: string
}>
}>
}>
}
class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor {
constructor (
private readonly urlProvider: (id: string) => string,
private readonly metadataByFilePath: Map<string, DocMetadata>,
private readonly metadataById: Map<Ref<Doc>, DocMetadata>,
private readonly attachMetadataByPath: Map<string, AttachmentMetadata>,
personsByName: Map<string, Ref<Person>>
) {
super(personsByName)
}
process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
traverseNode(json, (node) => {
if (node.type === MarkupNodeType.image) {
this.processImageNode(node, id, spaceId)
} else {
this.processLinkMarks(node, id, spaceId)
this.processMentions(node)
}
return true
})
return json
}
private processImageNode (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
const src = node.attrs?.src
if (src === undefined) return
const sourceMeta = this.getSourceMetadata(id)
if (sourceMeta == null) return
const href = decodeURI(src as string)
const fullPath = path.resolve(path.dirname(sourceMeta.path), href)
const attachmentMeta = this.attachMetadataByPath.get(fullPath)
if (attachmentMeta === undefined) {
console.warn(`Attachment image not found for ${fullPath}`)
return
}
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name)
}
private processLinkMarks (node: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): void {
traverseNodeMarks(node, (mark) => {
if (mark.type !== MarkupMarkType.link) return
const sourceMeta = this.getSourceMetadata(id)
if (sourceMeta == null) return
const href = decodeURI(mark.attrs.href)
const fullPath = path.resolve(path.dirname(sourceMeta.path), href)
if (this.metadataByFilePath.has(fullPath)) {
const targetDocMeta = this.metadataByFilePath.get(fullPath)
if (targetDocMeta !== undefined) {
this.alterInternalLinkNode(node, targetDocMeta)
}
} else if (this.attachMetadataByPath.has(fullPath)) {
const attachmentMeta = this.attachMetadataByPath.get(fullPath)
if (attachmentMeta !== undefined) {
this.alterAttachmentLinkNode(node, attachmentMeta)
this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta)
}
} else {
console.log('Unknown link type, leave it as is:', href)
}
})
}
private alterImageNode (node: MarkupNode, id: string, name: string): void {
node.type = MarkupNodeType.image
if (node.attrs !== undefined) {
node.attrs = {
'file-id': id,
src: this.urlProvider(id),
width: node.attrs.width ?? null,
height: node.attrs.height ?? null,
align: node.attrs.align ?? null,
alt: name,
title: name
}
const mimeType = this.getContentType(name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
}
private alterInternalLinkNode (node: MarkupNode, targetMeta: DocMetadata): void {
node.type = MarkupNodeType.reference
node.attrs = {
id: targetMeta.id,
label: targetMeta.refTitle,
objectclass: targetMeta.class,
text: '',
content: ''
}
}
private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void {
const stats = fs.statSync(targetMeta.path)
node.type = MarkupNodeType.file
node.attrs = {
'file-id': targetMeta.id,
'data-file-name': targetMeta.name,
'data-file-size': stats.size,
'data-file-href': targetMeta.path
}
const mimeType = this.getContentType(targetMeta.name)
if (mimeType !== undefined) {
node.attrs['data-file-type'] = mimeType
}
}
private getContentType (fileName: string): string | undefined {
const mimeType = contentType(fileName)
return mimeType !== false ? mimeType : undefined
}
private getSourceMetadata (id: Ref<Doc>): DocMetadata | null {
const sourceMeta = this.metadataById.get(id)
if (sourceMeta == null) {
console.warn(`Source metadata not found for ${id}`)
return null
}
return sourceMeta
}
private updateAttachmentMetadata (
fullPath: string,
attachmentMeta: AttachmentMetadata,
id: Ref<Doc>,
spaceId: Ref<Space>,
sourceMeta: DocMetadata
): void {
this.attachMetadataByPath.set(fullPath, {
...attachmentMeta,
spaceId,
parentId: id,
parentClass: sourceMeta.class as Ref<Class<Doc<Space>>>
})
}
}
interface DocMetadata {
id: Ref<Doc>
class: string
path: string
refTitle: string
}
interface AttachmentMetadata {
id: Ref<Attachment>
name: string
path: string
parentId?: Ref<Doc>
parentClass?: Ref<Class<Doc<Space>>>
spaceId?: Ref<Space>
}
export class UnifiedFormatImporter {
private readonly metadataById = new Map<Ref<Doc>, DocMetadata>()
private readonly metadataByFilePath = new Map<string, DocMetadata>()
private readonly attachMetadataByPath = new Map<string, AttachmentMetadata>()
private personsByName = new Map<string, Ref<Person>>()
private accountsByEmail = new Map<string, Ref<PersonAccount>>()
constructor (
private readonly client: TxOperations,
private readonly fileUploader: FileUploader
) {}
async importFolder (folderPath: string): Promise<void> {
await this.cachePersonsByNames()
await this.cacheAccountsByEmails()
const workspaceData = await this.processImportFolder(folderPath)
console.log('========================================')
console.log('IMPORT DATA STRUCTURE: ', JSON.stringify(workspaceData, null, 4))
console.log('========================================')
console.log('Importing documents...')
const preprocessor = new HulyMarkdownPreprocessor(
this.fileUploader.getFileUrl,
this.metadataByFilePath,
this.metadataById,
this.attachMetadataByPath,
this.personsByName
)
await new WorkspaceImporter(this.client, this.fileUploader, workspaceData, preprocessor).performImport()
console.log('Importing attachments...')
const attachments: ImportAttachment[] = Array.from(this.attachMetadataByPath.values())
.filter((attachment) => attachment.parentId !== undefined)
.map((attachment) => {
return {
id: attachment.id,
title: path.basename(attachment.path),
blobProvider: async () => {
const data = fs.readFileSync(attachment.path)
return new Blob([data])
},
parentId: attachment.parentId,
parentClass: attachment.parentClass,
spaceId: attachment.spaceId
}
})
await new WorkspaceImporter(this.client, this.fileUploader, { attachments }).performImport()
console.log('========================================')
console.log('IMPORT SUCCESS')
}
private async processImportFolder (folderPath: string): Promise<ImportWorkspace> {
const builder = new ImportWorkspaceBuilder(this.client)
await builder.initCache()
// Load workspace settings if exists
const wsSettingsPath = path.join(folderPath, 'settings.yaml')
if (fs.existsSync(wsSettingsPath)) {
const wsSettingsFile = fs.readFileSync(wsSettingsPath, 'utf8')
const wsSettings = yaml.load(wsSettingsFile) as UnifiedWorkspaceSettings
// Add project types
for (const pt of this.processProjectTypes(wsSettings)) {
builder.addProjectType(pt)
}
}
// Process all yaml files first
const yamlFiles = fs.readdirSync(folderPath).filter((f) => f.endsWith('.yaml') && f !== 'settings.yaml')
for (const yamlFile of yamlFiles) {
const yamlPath = path.join(folderPath, yamlFile)
const spaceName = path.basename(yamlFile, '.yaml')
const spacePath = path.join(folderPath, spaceName)
try {
console.log(`Processing ${spaceName}...`)
const spaceConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as UnifiedSpaceSettings
if (spaceConfig.class === undefined) {
console.warn(`Skipping ${spaceName}: not a space - no class specified`)
continue
}
switch (spaceConfig.class) {
case tracker.class.Project: {
const project = await this.processProject(spaceConfig as UnifiedProjectSettings)
builder.addProject(spacePath, project)
if (fs.existsSync(spacePath) && fs.statSync(spacePath).isDirectory()) {
await this.processIssuesRecursively(builder, project.identifier, spacePath, spacePath)
}
break
}
case document.class.Teamspace: {
const teamspace = await this.processTeamspace(spaceConfig as UnifiedTeamspaceSettings)
builder.addTeamspace(spacePath, teamspace)
if (fs.existsSync(spacePath) && fs.statSync(spacePath).isDirectory()) {
await this.processDocumentsRecursively(builder, spacePath, spacePath)
}
break
}
default: {
throw new Error(`Unknown space class ${spaceConfig.class} in ${spaceName}`)
}
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
throw new Error(`Invalid space configuration in ${spaceName}: ${message}`)
}
}
await this.processAttachments(folderPath)
return builder.build()
}
private async processIssuesRecursively (
builder: ImportWorkspaceBuilder,
projectIdentifier: string,
projectPath: string,
currentPath: string,
parentIssuePath?: string
): Promise<void> {
const issueFiles = fs.readdirSync(currentPath).filter((f) => f.endsWith('.md'))
for (const issueFile of issueFiles) {
const issuePath = path.join(currentPath, issueFile)
const issueHeader = (await this.readYamlHeader(issuePath)) as UnifiedIssueHeader
if (issueHeader.class === undefined) {
console.warn(`Skipping ${issueFile}: not an issue`)
continue
}
if (issueHeader.class === tracker.class.Issue) {
const numberMatch = issueFile.match(/^(\d+)\./)
const issueNumber = numberMatch?.[1]
const meta: DocMetadata = {
id: generateId<Issue>(),
class: tracker.class.Issue,
path: issuePath,
refTitle: `${projectIdentifier}-${issueNumber}`
}
this.metadataById.set(meta.id, meta)
this.metadataByFilePath.set(issuePath, meta)
const issue: ImportIssue = {
id: meta.id as Ref<Issue>,
class: tracker.class.Issue,
title: issueHeader.title,
number: parseInt(issueNumber ?? 'NaN'),
descrProvider: async () => await this.readMarkdownContent(issuePath),
status: { name: issueHeader.status },
priority: issueHeader.priority,
estimation: issueHeader.estimation,
remainingTime: issueHeader.remainingTime,
comments: this.processComments(issueHeader.comments),
subdocs: [], // Will be added via builder
assignee: this.findPersonByName(issueHeader.assignee)
}
builder.addIssue(projectPath, issuePath, issue, parentIssuePath)
// Process sub-issues if they exist
const subDir = path.join(currentPath, issueFile.replace('.md', ''))
if (fs.existsSync(subDir) && fs.statSync(subDir).isDirectory()) {
await this.processIssuesRecursively(builder, projectIdentifier, projectPath, subDir, issuePath)
}
} else {
throw new Error(`Unknown issue class ${issueHeader.class} in ${issueFile}`)
}
}
}
private findPersonByName (name?: string): Ref<Person> | undefined {
if (name === undefined) {
return undefined
}
const person = this.personsByName.get(name)
if (person === undefined) {
throw new Error(`Person not found: ${name}`)
}
return person
}
private findAccountByEmail (email: string): Ref<PersonAccount> {
const account = this.accountsByEmail.get(email)
if (account === undefined) {
throw new Error(`Account not found: ${email}`)
}
return account
}
private async processDocumentsRecursively (
builder: ImportWorkspaceBuilder,
teamspacePath: string,
currentPath: string,
parentDocPath?: string
): Promise<void> {
const docFiles = fs.readdirSync(currentPath).filter((f) => f.endsWith('.md'))
for (const docFile of docFiles) {
const docPath = path.join(currentPath, docFile)
const docHeader = (await this.readYamlHeader(docPath)) as UnifiedDocumentHeader
if (docHeader.class === undefined) {
console.warn(`Skipping ${docFile}: not a document`)
continue
}
if (docHeader.class === document.class.Document) {
const docMeta: DocMetadata = {
id: generateId<Document>(),
class: document.class.Document,
path: docPath,
refTitle: docHeader.title
}
this.metadataById.set(docMeta.id, docMeta)
this.metadataByFilePath.set(docPath, docMeta)
const doc: ImportDocument = {
id: docMeta.id as Ref<Document>,
class: document.class.Document,
title: docHeader.title,
descrProvider: async () => await this.readMarkdownContent(docPath),
subdocs: [] // Will be added via builder
}
builder.addDocument(teamspacePath, docPath, doc, parentDocPath)
// Process subdocuments if they exist
const subDir = path.join(currentPath, docFile.replace('.md', ''))
if (fs.existsSync(subDir) && fs.statSync(subDir).isDirectory()) {
await this.processDocumentsRecursively(builder, teamspacePath, subDir, docPath)
}
} else {
throw new Error(`Unknown document class ${docHeader.class} in ${docFile}`)
}
}
}
private processComments (comments: UnifiedComment[] = []): ImportComment[] {
return comments.map((comment) => {
return {
text: comment.text,
author: this.findAccountByEmail(comment.author)
}
})
}
private processProjectTypes (wsHeader: UnifiedWorkspaceSettings): ImportProjectType[] {
return (
wsHeader.projectTypes?.map((pt) => ({
name: pt.name,
taskTypes: pt.taskTypes?.map((tt) => ({
name: tt.name,
description: tt.description,
statuses: tt.statuses.map((st) => ({
name: st.name,
description: st.description
}))
}))
})) ?? []
)
}
private async processProject (projectHeader: UnifiedProjectSettings): Promise<ImportProject> {
return {
class: tracker.class.Project,
title: projectHeader.title,
identifier: projectHeader.identifier,
private: projectHeader.private ?? false,
autoJoin: projectHeader.autoJoin ?? true,
description: projectHeader.description,
defaultIssueStatus:
projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined,
owners:
projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
members:
projectHeader.members !== undefined ? projectHeader.members.map((email) => this.findAccountByEmail(email)) : [],
docs: []
}
}
private async processTeamspace (spaceHeader: UnifiedTeamspaceSettings): Promise<ImportTeamspace> {
return {
class: document.class.Teamspace,
title: spaceHeader.title,
private: spaceHeader.private ?? false,
autoJoin: spaceHeader.autoJoin ?? true,
description: spaceHeader.description,
owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
members:
spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [],
docs: []
}
}
private async readYamlHeader (filePath: string): Promise<any> {
console.log('Read YAML header from: ', filePath)
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(/^---\n([\s\S]*?)\n---/)
if (match != null) {
return yaml.load(match[1])
}
return {}
}
private async readMarkdownContent (filePath: string): Promise<string> {
const content = fs.readFileSync(filePath, 'utf8')
const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s)
return match != null ? match[1] : content
}
private async cachePersonsByNames (): Promise<void> {
this.personsByName = (await this.client.findAll(contact.class.Person, {}))
.map((person) => {
return {
_id: person._id,
name: person.name.split(',').reverse().join(' ')
}
})
.reduce((refByName, person) => {
refByName.set(person.name, person._id)
return refByName
}, new Map())
}
private async cacheAccountsByEmails (): Promise<void> {
const accounts = await this.client.findAll(contact.class.PersonAccount, {})
this.accountsByEmail = accounts.reduce((map, account) => {
map.set(account.email, account._id)
return map
}, new Map())
}
private async processAttachments (folderPath: string): Promise<void> {
const processDir = async (dir: string): Promise<void> => {
const entries = fs.readdirSync(dir, { withFileTypes: true })
for (const entry of entries) {
const fullPath = path.join(dir, entry.name)
if (entry.isDirectory()) {
await processDir(fullPath)
} else if (entry.isFile()) {
// Skip files that are already processed as documents or issues
if (!this.metadataByFilePath.has(fullPath)) {
const attachmentId = generateId<Attachment>()
this.attachMetadataByPath.set(fullPath, { id: attachmentId, name: entry.name, path: fullPath })
}
}
}
}
await processDir(folderPath)
}
}

View File

@ -0,0 +1,499 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type DocumentQuery, type Ref, type Status, type TxOperations } from '@hcengineering/core'
import document from '@hcengineering/document'
import tracker, { IssuePriority, type IssueStatus } from '@hcengineering/tracker'
import {
type ImportDocument,
type ImportIssue,
type ImportProject,
type ImportProjectType,
type ImportTeamspace,
type ImportWorkspace
} from './importer'
export interface ValidationError {
path: string
error: string
}
export interface ValidationResult {
isValid: boolean
errors: Map<string, ValidationError>
}
const MAX_PROJECT_IDENTIFIER_LENGTH = 5
const PROJECT_IDENTIFIER_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/
export class ImportWorkspaceBuilder {
private readonly projects = new Map<string, ImportProject>()
private readonly teamspaces = new Map<string, ImportTeamspace>()
private readonly projectTypes = new Map<string, ImportProjectType>()
private readonly issuesByProject = new Map<string, Map<string, ImportIssue>>()
private readonly issueParents = new Map<string, string>()
private readonly documentsByTeamspace = new Map<string, Map<string, ImportDocument>>()
private readonly documentParents = new Map<string, string>()
private readonly errors = new Map<string, ValidationError>()
private readonly issueStatusCache = new Map<string, Ref<IssueStatus>>()
constructor (
private readonly client: TxOperations,
private readonly strictMode: boolean = true
) {}
async initCache (): Promise<this> {
await this.cacheIssueStatuses()
return this
}
addProjectType (projectType: ImportProjectType): this {
this.validateAndAdd(
'projectType',
projectType.name,
projectType,
(pt) => this.validateProjectType(pt),
this.projectTypes
)
return this
}
addProject (path: string, project: ImportProject): this {
this.validateAndAdd('project', path, project, (p) => this.validateProject(p), this.projects, path)
return this
}
addTeamspace (path: string, teamspace: ImportTeamspace): this {
this.validateAndAdd('teamspace', path, teamspace, (t) => this.validateTeamspace(t), this.teamspaces, path)
return this
}
addIssue (projectPath: string, issuePath: string, issue: ImportIssue, parentIssuePath?: string): this {
if (!this.issuesByProject.has(projectPath)) {
this.issuesByProject.set(projectPath, new Map())
}
const projectIssues = this.issuesByProject.get(projectPath)
if (projectIssues === undefined) {
throw new Error(`Project ${projectPath} not found`)
}
const duplicateIssue = Array.from(projectIssues.values()).find(
(existingIssue) => existingIssue.number === issue.number
)
if (duplicateIssue !== undefined) {
this.addError(issuePath, `Duplicate issue number ${issue.number} in project ${projectPath}`)
} else {
this.validateAndAdd('issue', issuePath, issue, (i) => this.validateIssue(i), projectIssues, issuePath)
if (parentIssuePath !== undefined) {
this.issueParents.set(issuePath, parentIssuePath)
}
}
return this
}
addDocument (teamspacePath: string, docPath: string, doc: ImportDocument, parentDocPath?: string): this {
if (!this.documentsByTeamspace.has(teamspacePath)) {
this.documentsByTeamspace.set(teamspacePath, new Map())
}
const docs = this.documentsByTeamspace.get(teamspacePath)
if (docs === undefined) {
throw new Error(`Teamspace ${teamspacePath} not found`)
}
this.validateAndAdd('document', docPath, doc, (d) => this.validateDocument(d), docs, docPath)
if (parentDocPath !== undefined) {
this.documentParents.set(docPath, parentDocPath)
}
return this
}
validate (): ValidationResult {
// Perform cross-entity validation
this.validateProjectReferences()
this.validateSpaceDocuments()
return {
isValid: this.errors.size === 0,
errors: this.errors
}
}
build (): ImportWorkspace {
const validation = this.validate()
if (this.strictMode && !validation.isValid) {
throw new Error(
'Invalid workspace: \n' +
Array.from(validation.errors.values())
.map((e) => ` * ${e.path}: ${e.error}`)
.join(';\n')
)
}
for (const [teamspacePath, docs] of this.documentsByTeamspace) {
const teamspace = this.teamspaces.get(teamspacePath)
if (teamspace !== undefined) {
const rootDocPaths = Array.from(docs.keys()).filter((docPath) => !this.documentParents.has(docPath))
for (const rootPath of rootDocPaths) {
this.buildDocumentHierarchy(rootPath, docs)
}
teamspace.docs = rootDocPaths.map((path) => docs.get(path)).filter(Boolean) as ImportDocument[]
}
}
for (const [projectPath, issues] of this.issuesByProject) {
const project = this.projects.get(projectPath)
if (project !== undefined) {
const rootIssuePaths = Array.from(issues.keys()).filter((issuePath) => !this.issueParents.has(issuePath))
for (const rootPath of rootIssuePaths) {
this.buildIssueHierarchy(rootPath, issues)
}
project.docs = rootIssuePaths.map((path) => issues.get(path)).filter(Boolean) as ImportIssue[]
}
}
return {
projectTypes: Array.from(this.projectTypes.values()),
spaces: [...Array.from(this.projects.values()), ...Array.from(this.teamspaces.values())]
}
}
async cacheIssueStatuses (): Promise<void> {
const query: DocumentQuery<Status> = {
ofAttribute: tracker.attribute.IssueStatus
}
const statuses = await this.client.findAll(tracker.class.IssueStatus, query)
for (const status of statuses) {
this.issueStatusCache.set(status.name, status._id)
}
}
private validateAndAdd<T, K>(
type: string,
path: string,
item: T,
validator: (item: T) => string[],
collection: Map<K, T>,
key?: K
): void {
const errors = validator(item)
if (errors.length > 0) {
this.addError(path, `Invalid ${type} at ${path}: \n${errors.map((e) => ` * ${e}`).join('\n')}`)
if (this.strictMode) {
throw new Error(`Invalid ${type} at ${path}: \n${errors.map((e) => ` * ${e}`).join('\n')}`)
}
} else {
collection.set((key ?? path) as K, item)
}
}
private validateProjectType (projectType: ImportProjectType): string[] {
const errors: string[] = []
if (!this.validateStringDefined(projectType.name)) {
errors.push('name is required')
}
return errors
}
private validateProject (project: ImportProject): string[] {
const errors: string[] = []
errors.push(...this.validateType(project.title, 'string', 'title'))
errors.push(...this.validateType(project.identifier, 'string', 'identifier'))
errors.push(...this.validateType(project.class, 'string', 'class'))
if (project.private !== undefined) {
errors.push(...this.validateType(project.private, 'boolean', 'private'))
}
if (project.autoJoin !== undefined) {
errors.push(...this.validateType(project.autoJoin, 'boolean', 'autoJoin'))
}
if (project.owners !== undefined) {
errors.push(...this.validateArray(project.owners, 'string', 'owners'))
}
if (project.members !== undefined) {
errors.push(...this.validateArray(project.members, 'string', 'members'))
}
if (project.description !== undefined) {
errors.push(...this.validateType(project.description, 'string', 'description'))
}
if (!this.validateStringDefined(project.title)) {
errors.push('title is required')
}
if (project.class !== tracker.class.Project) {
errors.push('invalid class: ' + project.class)
}
if (project.defaultIssueStatus !== undefined && !this.issueStatusCache.has(project.defaultIssueStatus.name)) {
errors.push('defaultIssueStatus not found: ' + project.defaultIssueStatus.name)
}
errors.push(...this.validateProjectIdentifier(project.identifier))
return errors
}
private validateProjectIdentifier (identifier: string): string[] {
const errors: string[] = []
if (!this.validateStringDefined(identifier)) {
errors.push('identifier is required')
return errors
}
if (identifier.length > MAX_PROJECT_IDENTIFIER_LENGTH) {
errors.push(`identifier must be no longer than ${MAX_PROJECT_IDENTIFIER_LENGTH} characters`)
}
if (!PROJECT_IDENTIFIER_REGEX.test(identifier)) {
errors.push(
'identifier must contain only Latin letters, numbers, and underscores, and must not start with a number'
)
}
return errors
}
private validateTeamspace (teamspace: ImportTeamspace): string[] {
const errors: string[] = []
errors.push(...this.validateType(teamspace.title, 'string', 'title'))
errors.push(...this.validateType(teamspace.class, 'string', 'class'))
if (teamspace.private !== undefined) {
errors.push(...this.validateType(teamspace.private, 'boolean', 'private'))
}
if (teamspace.autoJoin !== undefined) {
errors.push(...this.validateType(teamspace.autoJoin, 'boolean', 'autoJoin'))
}
if (teamspace.owners !== undefined) {
errors.push(...this.validateArray(teamspace.owners, 'string', 'owners'))
}
if (teamspace.members !== undefined) {
errors.push(...this.validateArray(teamspace.members, 'string', 'members'))
}
if (teamspace.description !== undefined) {
errors.push(...this.validateType(teamspace.description, 'string', 'description'))
}
if (!this.validateStringDefined(teamspace.title)) {
errors.push('title is required')
}
if (teamspace.class !== document.class.Teamspace) {
errors.push('invalid class: ' + teamspace.class)
}
return errors
}
private validateIssue (issue: ImportIssue): string[] {
const errors: string[] = []
errors.push(...this.validateType(issue.title, 'string', 'title'))
errors.push(...this.validateType(issue.class, 'string', 'class'))
if (issue.number !== undefined) {
errors.push(...this.validateType(issue.number, 'number', 'number'))
}
if (issue.estimation !== undefined) {
errors.push(...this.validateType(issue.estimation, 'number', 'estimation'))
}
if (issue.remainingTime !== undefined) {
errors.push(...this.validateType(issue.remainingTime, 'number', 'remainingTime'))
}
if (issue.priority !== undefined) {
errors.push(...this.validateType(issue.priority, 'string', 'priority'))
}
if (issue.assignee !== undefined) {
errors.push(...this.validateType(issue.assignee, 'string', 'assignee'))
}
if (issue.status == null) {
errors.push('status is required: ')
} else if (!this.issueStatusCache.has(issue.status.name)) {
errors.push('status not found: ' + issue.status.name)
}
if (issue.priority != null && IssuePriority[issue.priority as keyof typeof IssuePriority] === undefined) {
errors.push('priority not found: ' + issue.priority)
}
if (issue.class !== tracker.class.Issue) {
errors.push('invalid class: ' + issue.class)
}
if (issue.number !== undefined && !this.validatePossitiveNumber(issue.number)) {
errors.push('invalid issue number: ' + issue.number)
}
if (issue.estimation != null && !this.validatePossitiveNumber(issue.estimation)) {
errors.push('invalid estimation: ' + issue.estimation)
}
if (issue.remainingTime != null && !this.validatePossitiveNumber(issue.remainingTime)) {
errors.push('invalid remaining time: ' + issue.remainingTime)
}
if (issue.comments != null && issue.comments.length > 0) {
for (const comment of issue.comments) {
if (comment.author == null) {
errors.push('comment author is required')
}
if (!this.validateStringDefined(comment.text)) {
errors.push('comment text is required')
}
}
}
return errors
}
private validatePossitiveNumber (value: any): boolean {
return typeof value === 'number' && !Number.isNaN(value) && value >= 0
}
private validateStringDefined (value: string | null | undefined): boolean {
return typeof value === 'string' && value !== '' && value !== null && value !== undefined
}
private validateDocument (doc: ImportDocument): string[] {
const errors: string[] = []
if (!this.validateStringDefined(doc.title)) {
errors.push('title is required')
}
if (doc.class !== document.class.Document) {
errors.push('invalid class: ' + doc.class)
}
return errors
}
private validateProjectReferences (): void {
// Validate project type references
for (const project of this.projects.values()) {
if (project.projectType !== undefined && !this.projectTypes.has(project.projectType.name)) {
this.addError(project.title, `Referenced project type ${project.projectType.name} not found`)
}
}
}
private validateSpaceDocuments (): void {
// Validate that issues belong to projects and documents to teamspaces
for (const projectPath of this.issuesByProject.keys()) {
if (!this.projects.has(projectPath)) {
this.addError(projectPath, 'Issues reference non-existent project')
}
}
for (const [teamspacePath] of this.documentsByTeamspace) {
if (!this.teamspaces.has(teamspacePath)) {
this.addError(teamspacePath, 'Documents reference non-existent teamspace')
}
}
}
private addError (path: string, error: string): void {
this.errors.set(path, { path, error })
}
private buildDocumentHierarchy (docPath: string, allDocs: Map<string, ImportDocument>): void {
const doc = allDocs.get(docPath)
if (doc === undefined) return
const childDocs = Array.from(allDocs.entries())
.filter(([childPath]) => this.documentParents.get(childPath) === docPath)
.map(([childPath, childDoc]) => {
this.buildDocumentHierarchy(childPath, allDocs)
return childDoc
})
doc.subdocs = childDocs
}
private buildIssueHierarchy (issuePath: string, allIssues: Map<string, ImportIssue>): void {
const issue = allIssues.get(issuePath)
if (issue === undefined) return
const childIssues = Array.from(allIssues.entries())
.filter(([childPath]) => this.issueParents.get(childPath) === issuePath)
.map(([childPath, childIssue]) => {
this.buildIssueHierarchy(childPath, allIssues)
return childIssue
})
issue.subdocs = childIssues
}
private validateType (value: unknown, type: 'string' | 'number' | 'boolean', fieldName: string): string[] {
const errors: string[] = []
switch (type) {
case 'string':
if (typeof value !== 'string') {
errors.push(`${fieldName} must be string, got ${typeof value}`)
}
break
case 'number':
if (typeof value !== 'number') {
errors.push(`${fieldName} must be number, got ${typeof value}`)
}
break
case 'boolean':
if (typeof value !== 'boolean') {
errors.push(`${fieldName} must be boolean, got ${typeof value}`)
}
break
}
return errors
}
private validateArray (value: unknown, itemType: 'string' | 'number' | 'boolean', fieldName: string): string[] {
const errors: string[] = []
if (!Array.isArray(value)) {
errors.push(`${fieldName} must be an array`)
return errors
}
for (let i = 0; i < value.length; i++) {
switch (itemType) {
case 'string':
if (typeof value[i] !== 'string') {
errors.push(`${fieldName}[${i}] must be string, got ${typeof value[i]}`)
}
break
case 'number':
if (typeof value[i] !== 'number') {
errors.push(`${fieldName}[${i}] must be number, got ${typeof value[i]}`)
}
break
case 'boolean':
if (typeof value[i] !== 'boolean') {
errors.push(`${fieldName}[${i}] must be boolean, got ${typeof value[i]}`)
}
break
}
}
return errors
}
}

View File

@ -1,3 +1,17 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export async function download (url: string): Promise<Blob | null> {
try {
const response = await fetch(url)

View File

@ -19,6 +19,7 @@ import { type Person } from '@hcengineering/contact'
import core, {
type Account,
type AttachedData,
type Class,
type CollaborativeDoc,
type Data,
type Doc,
@ -28,6 +29,7 @@ import core, {
type Mixin,
type Ref,
SortingOrder,
type Space,
type Status,
type Timestamp,
type TxOperations
@ -40,7 +42,7 @@ import task, {
type TaskType,
type TaskTypeWithFactory
} from '@hcengineering/task'
import { jsonToMarkup, jsonToYDocNoSchema, type MarkupNode, parseMessageMarkdown } from '@hcengineering/text'
import { jsonToMarkup, jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text'
import tracker, {
type Issue,
type IssueParentInfo,
@ -49,17 +51,13 @@ import tracker, {
type Project,
TimeReportDayType
} from '@hcengineering/tracker'
import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor'
import { type FileUploader, type UploadResult } from './uploader'
export interface ImportWorkspace {
persons?: ImportPerson[]
projectTypes?: ImportProjectType[]
spaces?: ImportSpace<ImportDoc>[]
}
export interface ImportPerson {
name: string
email: string
attachments?: ImportAttachment[]
}
export interface ImportProjectType {
@ -80,45 +78,47 @@ export interface ImportStatus {
}
export interface ImportSpace<T extends ImportDoc> {
class: string
name: string
class: Ref<Class<Space>>
title: string
private: boolean
autoJoin?: boolean
description?: string
owners?: Ref<Account>[]
members?: Ref<Account>[]
docs: T[]
}
export interface ImportDoc {
class: string
id?: Ref<Doc>
class: Ref<Class<Doc<Space>>>
title: string
descrProvider: () => Promise<string>
subdocs: ImportDoc[]
}
export interface ImportTeamspace extends ImportSpace<ImportDocument> {
class: 'document.class.TeamSpace'
class: Ref<Class<Teamspace>>
}
export interface ImportDocument extends ImportDoc {
class: 'document.class.Document'
id?: Ref<Document>
class: Ref<Class<Document>>
subdocs: ImportDocument[]
}
export interface ImportProject extends ImportSpace<ImportIssue> {
class: 'tracker.class.Project'
class: Ref<Class<Project>>
identifier: string
private: boolean
autoJoin: boolean
projectType: ImportProjectType
defaultAssignee?: ImportPerson
projectType?: ImportProjectType
defaultIssueStatus?: ImportStatus
owners?: ImportPerson[]
members?: ImportPerson[]
description?: string
}
export interface ImportIssue extends ImportDoc {
class: 'tracker.class.Issue'
id?: Ref<Issue>
class: Ref<Class<Issue>>
status: ImportStatus
priority?: string
number?: number
assignee?: Ref<Person>
estimation?: number
remainingTime?: number
@ -133,16 +133,15 @@ export interface ImportComment {
}
export interface ImportAttachment {
id?: Ref<Attachment>
title: string
blobProvider: () => Promise<Blob | null>
}
export interface MarkdownPreprocessor {
process: (json: MarkupNode) => MarkupNode
parentId?: Ref<Doc>
parentClass?: Ref<Class<Doc<Space>>>
spaceId?: Ref<Space>
}
export class WorkspaceImporter {
private readonly personsByName = new Map<string, Ref<Person>>()
private readonly issueStatusByName = new Map<string, Ref<IssueStatus>>()
private readonly projectTypeByName = new Map<string, Ref<ProjectType>>()
@ -150,23 +149,13 @@ export class WorkspaceImporter {
private readonly client: TxOperations,
private readonly fileUploader: FileUploader,
private readonly workspaceData: ImportWorkspace,
private readonly preprocessor: MarkdownPreprocessor
private readonly preprocessor: MarkdownPreprocessor = new NoopMarkdownPreprocessor()
) {}
public async performImport (): Promise<void> {
await this.importPersons()
await this.importProjectTypes()
await this.importSpaces()
}
private async importPersons (): Promise<void> {
if (this.workspaceData.persons === undefined) return
for (const person of this.workspaceData.persons) {
const personId = generateId<Person>()
this.personsByName.set(person.name, personId)
// TODO: Implement person creation
}
await this.importAttachments()
}
private async importProjectTypes (): Promise<void> {
@ -182,14 +171,29 @@ export class WorkspaceImporter {
if (this.workspaceData.spaces === undefined) return
for (const space of this.workspaceData.spaces) {
if (space.class === 'document.class.TeamSpace') {
if (space.class === document.class.Teamspace) {
await this.importTeamspace(space as ImportTeamspace)
} else if (space.class === 'tracker.class.Project') {
} else if (space.class === tracker.class.Project) {
await this.importProject(space as ImportProject)
}
}
}
private async importAttachments (): Promise<void> {
if (this.workspaceData.attachments === undefined) return
for (const attachment of this.workspaceData.attachments) {
if (
attachment.parentId === undefined ||
attachment.parentClass === undefined ||
attachment.spaceId === undefined
) {
throw new Error('Attachment is missing parentId, parentClass or spaceId')
}
await this.importAttachment(attachment.parentId, attachment.parentClass, attachment, attachment.spaceId)
}
}
async createProjectTypeWithTaskTypes (projectType: ImportProjectType): Promise<Ref<ProjectType>> {
const taskTypes: TaskTypeWithFactory[] = []
if (projectType.taskTypes !== undefined) {
@ -230,7 +234,9 @@ export class WorkspaceImporter {
}
async importTeamspace (space: ImportTeamspace): Promise<Ref<Teamspace>> {
console.log('Creating teamspace: ', space.title)
const teamspaceId = await this.createTeamspace(space)
console.log('Teamspace created: ', teamspaceId)
for (const doc of space.docs) {
await this.createDocumentWithSubdocs(doc, document.ids.NoParent, teamspaceId)
}
@ -242,7 +248,9 @@ export class WorkspaceImporter {
parentId: Ref<Document>,
teamspaceId: Ref<Teamspace>
): Promise<Ref<Document>> {
console.log('Creating document: ', doc.title)
const documentId = await this.createDocument(doc, parentId, teamspaceId)
console.log('Document created: ', documentId)
for (const child of doc.subdocs) {
await this.createDocumentWithSubdocs(child, documentId, teamspaceId)
}
@ -254,12 +262,12 @@ export class WorkspaceImporter {
const data = {
type: document.spaceType.DefaultTeamspaceType,
description: space.description ?? '',
title: space.name,
name: space.name,
private: false,
members: [],
owners: [],
autoJoin: false,
title: space.title,
name: space.title,
private: space.private,
owners: space.owners ?? [],
members: space.members ?? [],
autoJoin: space.autoJoin,
archived: false
}
await this.client.createDoc(document.class.Teamspace, core.space.Space, data, teamspaceId)
@ -271,9 +279,9 @@ export class WorkspaceImporter {
parentId: Ref<Document>,
teamspaceId: Ref<Teamspace>
): Promise<Ref<Document>> {
const id = generateId<Document>()
const id = doc.id ?? generateId<Document>()
const content = await doc.descrProvider()
const collabId = await this.createCollaborativeContent(id, 'content', content)
const collabId = await this.createCollaborativeContent(id, 'content', content, teamspaceId)
const lastRank = await getFirstRank(this.client, teamspaceId, parentId)
const rank = makeRank(lastRank, undefined)
@ -295,7 +303,7 @@ export class WorkspaceImporter {
}
async importProject (project: ImportProject): Promise<Ref<Project>> {
console.log('Create project: ', project.name)
console.log('Creating project: ', project.title)
const projectId = await this.createProject(project)
console.log('Project created: ' + projectId)
@ -316,7 +324,7 @@ export class WorkspaceImporter {
project: Project,
parentsInfo: IssueParentInfo[]
): Promise<{ id: Ref<Issue>, identifier: string }> {
console.log('Create issue: ', issue.title)
console.log('Creating issue: ', issue.title)
const issueResult = await this.createIssue(issue, project, parentId, parentsInfo)
console.log('Issue created: ', issueResult)
@ -341,23 +349,29 @@ export class WorkspaceImporter {
async createProject (project: ImportProject): Promise<Ref<Project>> {
const projectId = generateId<Project>()
const projectType = this.projectTypeByName.get(project.projectType.name)
const projectType =
project.projectType !== undefined
? this.projectTypeByName.get(project.projectType.name)
: tracker.ids.ClassingProjectType
const defaultIssueStatus =
project.defaultIssueStatus !== undefined
? this.issueStatusByName.get(project.defaultIssueStatus.name)
: tracker.status.Backlog
const identifier = await this.uniqueProjectIdentifier(project.identifier)
const projectData = {
name: project.name,
name: project.title,
description: project.description ?? '',
private: project.private,
members: [],
owners: [],
members: project.members ?? [],
owners: project.owners ?? [],
archived: false,
autoJoin: project.autoJoin,
identifier,
sequence: 0,
defaultIssueStatus: defaultIssueStatus ?? tracker.status.Backlog, // todo: test with no status
defaultIssueStatus: defaultIssueStatus ?? tracker.status.Backlog,
defaultTimeReportDay: TimeReportDayType.PreviousWorkDay,
type: projectType as Ref<ProjectType>
}
@ -375,14 +389,22 @@ export class WorkspaceImporter {
parentId: Ref<Issue>,
parentsInfo: IssueParentInfo[]
): Promise<{ id: Ref<Issue>, identifier: string }> {
const issueId = generateId<Issue>()
const issueId = issue.id ?? generateId<Issue>()
const content = await issue.descrProvider()
const collabId = await this.createCollaborativeContent(issueId, 'description', content)
const collabId = await this.createCollaborativeContent(issueId, 'description', content, project._id)
const { number, identifier } =
issue.number !== undefined
? { number: issue.number, identifier: `${project.identifier}-${issue.number}` }
: await this.getNextIssueIdentifier(project)
const { number, identifier } = await this.getNextIssueIdentifier(project)
const kind = await this.getIssueKind(project)
const rank = await this.getIssueRank(project)
const status = await this.findIssueStatusByName(issue.status.name)
const priority =
issue.priority !== undefined
? IssuePriority[issue.priority as keyof typeof IssuePriority]
: IssuePriority.NoPriority
const estimation = issue.estimation ?? 0
const remainingTime = issue.remainingTime ?? 0
@ -395,7 +417,7 @@ export class WorkspaceImporter {
component: null,
number,
status,
priority: IssuePriority.NoPriority,
priority,
rank,
comments: issue.comments?.length ?? 0,
subIssues: issue.subdocs.length,
@ -469,7 +491,7 @@ export class WorkspaceImporter {
async createComment (issueId: Ref<Issue>, comment: ImportComment, projectId: Ref<Project>): Promise<void> {
const json = parseMessageMarkdown(comment.text ?? '', 'image://')
const processedJson = this.preprocessor.process(json)
const processedJson = this.preprocessor.process(json, issueId, projectId)
const markup = jsonToMarkup(processedJson)
const value: AttachedData<ChatMessage> = {
@ -491,71 +513,86 @@ export class WorkspaceImporter {
)
if (comment.attachments !== undefined) {
await this.importAttachments(commentId, comment.attachments, projectId)
for (const attachment of comment.attachments) {
await this.importAttachment(commentId, chunter.class.ChatMessage, attachment, projectId)
}
}
}
private async importAttachments (
commentId: Ref<ChatMessage>,
attachments: ImportAttachment[],
projectId: Ref<Project>
private async importAttachment (
parentId: Ref<Doc>,
parentClass: Ref<Class<Doc<Space>>>,
attachment: ImportAttachment,
spaceId: Ref<Space>
): Promise<void> {
for (const attach of attachments) {
const blob = await attach.blobProvider()
if (blob === null) {
console.warn('Failed to download attachment file: ', attach.title)
continue
}
const blob = await attachment.blobProvider()
if (blob === null) {
console.warn('Failed to read attachment file: ', attachment.title)
return
}
const attachmentId = await this.createAttachment(blob, attach, projectId, commentId)
if (attachmentId === null) {
console.warn('Failed to upload attachment file: ', attach.title)
}
const file = new File([blob], attachment.title)
const attachmentId = await this.createAttachment(
attachment.id ?? generateId<Attachment>(),
file,
spaceId,
parentId,
parentClass
)
if (attachmentId === null) {
console.warn('Failed to upload attachment file: ', attachment.title)
}
}
private async createAttachment (
blob: Blob,
attach: ImportAttachment,
projectId: Ref<Project>,
commentId: Ref<ChatMessage>
id: Ref<Attachment>,
file: File,
spaceId: Ref<Space>,
parentId: Ref<Doc>,
parentClass: Ref<Class<Doc<Space>>>
): Promise<Ref<Attachment> | null> {
const attachmentId = generateId<Attachment>()
const file = new File([blob], attach.title)
const response = await this.fileUploader.uploadFile(attachmentId, attach.title, file)
if (response.status === 200) {
const responseText = await response.text()
if (responseText !== undefined) {
const uploadResult = JSON.parse(responseText) as UploadResult[]
if (!Array.isArray(uploadResult) || uploadResult.length === 0) {
return null
}
await this.client.addCollection(
attachment.class.Attachment,
projectId,
commentId,
chunter.class.ChatMessage,
'attachments',
{
file: uploadResult[0].id,
lastModified: Date.now(),
name: file.name,
size: file.size,
type: file.type
},
attachmentId
)
}
const response = await this.fileUploader.uploadFile(id, id, file)
if (response.status !== 200) {
return null
}
return attachmentId
const responseText = await response.text()
if (responseText === undefined) {
return null
}
const uploadResult = JSON.parse(responseText) as UploadResult[]
if (!Array.isArray(uploadResult) || uploadResult.length === 0) {
return null
}
await this.client.addCollection(
attachment.class.Attachment,
spaceId,
parentId,
parentClass,
'attachments',
{
file: uploadResult[0].id,
lastModified: Date.now(),
name: file.name,
size: file.size,
type: file.type
},
id
)
return id
}
// Collaborative content handling
private async createCollaborativeContent (id: Ref<Doc>, field: string, content: string): Promise<CollaborativeDoc> {
private async createCollaborativeContent (
id: Ref<Doc>,
field: string,
content: string,
spaceId: Ref<Space>
): Promise<CollaborativeDoc> {
const json = parseMessageMarkdown(content ?? '', 'image://')
const processedJson = this.preprocessor.process(json)
const processedJson = this.preprocessor.process(json, id, spaceId)
const collabId = makeCollaborativeDoc(id, 'description')
const yDoc = jsonToYDocNoSchema(processedJson, field)
@ -568,8 +605,7 @@ export class WorkspaceImporter {
async findIssueStatusByName (name: string): Promise<Ref<IssueStatus>> {
const query: DocumentQuery<Status> = {
name,
ofAttribute: tracker.attribute.IssueStatus,
category: task.statusCategory.Active
ofAttribute: tracker.attribute.IssueStatus
}
const status = await this.client.findOne(tracker.class.IssueStatus, query)

View File

@ -0,0 +1,119 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import contact, { type Person } from '@hcengineering/contact'
import { type Doc, type Space, type Ref } from '@hcengineering/core'
import { type MarkupNode, MarkupNodeType } from '@hcengineering/text'
export interface MarkdownPreprocessor {
process: (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>) => MarkupNode
}
export class NoopMarkdownPreprocessor implements MarkdownPreprocessor {
process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode {
return json
}
}
export abstract class BaseMarkdownPreprocessor implements MarkdownPreprocessor {
protected readonly MENTION_REGEX = /@([\p{L}\p{M}]+ [\p{L}\p{M}]+)/gu
constructor (protected readonly personsByName: Map<string, Ref<Person>>) {}
abstract process (json: MarkupNode, id: Ref<Doc>, spaceId: Ref<Space>): MarkupNode
protected processMentions (node: MarkupNode): void {
if (node.type !== MarkupNodeType.paragraph || node.content === undefined) return
const newContent: MarkupNode[] = []
for (const childNode of node.content) {
if (childNode.type === MarkupNodeType.text && childNode.text !== undefined) {
this.processMentionTextNode(childNode, newContent)
} else {
newContent.push(childNode)
}
}
node.content = newContent
}
protected processMentionTextNode (node: MarkupNode, newContent: MarkupNode[]): void {
if (node.text === undefined) return
let match
let lastIndex = 0
let hasMentions = false
while ((match = this.MENTION_REGEX.exec(node.text)) !== null) {
hasMentions = true
this.addTextBeforeMention(newContent, node, lastIndex, match.index)
this.addMentionNode(newContent, match[1], node)
lastIndex = this.MENTION_REGEX.lastIndex
}
if (hasMentions) {
this.addRemainingText(newContent, node, lastIndex)
} else {
newContent.push(node)
}
}
protected addTextBeforeMention (
newContent: MarkupNode[],
node: MarkupNode,
lastIndex: number,
matchIndex: number
): void {
if (node.text === undefined) return
if (matchIndex > lastIndex) {
newContent.push({
type: MarkupNodeType.text,
text: node.text.slice(lastIndex, matchIndex),
marks: node.marks,
attrs: node.attrs
})
}
}
protected addMentionNode (newContent: MarkupNode[], name: string, originalNode: MarkupNode): void {
const personRef = this.personsByName.get(name)
if (personRef !== undefined) {
newContent.push({
type: MarkupNodeType.reference,
attrs: {
id: personRef,
label: name,
objectclass: contact.class.Person
}
})
} else {
newContent.push({
type: MarkupNodeType.text,
text: `@${name}`,
marks: originalNode.marks,
attrs: originalNode.attrs
})
}
}
protected addRemainingText (newContent: MarkupNode[], node: MarkupNode, lastIndex: number): void {
if (node.text !== undefined && lastIndex < node.text.length) {
newContent.push({
type: MarkupNodeType.text,
text: node.text.slice(lastIndex),
marks: node.marks,
attrs: node.attrs
})
}
}
}

View File

@ -24,6 +24,7 @@ import { importNotion } from './notion/notion'
import { setMetadata } from '@hcengineering/platform'
import { FrontFileUploader, type FileUploader } from './importer/uploader'
import { ClickupImporter } from './clickup/clickup'
import { UnifiedFormatImporter } from './huly/unified'
/**
* @public
@ -131,5 +132,20 @@ export function importTool (): void {
})
})
// import /home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace --workspace ws1 --user user1 --password 1234
program
.command('import <dir>')
.description('import issues in Unified Huly Format')
.requiredOption('-u, --user <user>', 'user')
.requiredOption('-pw, --password <password>', 'password')
.requiredOption('-ws, --workspace <workspace>', 'workspace url where the documents should be imported to')
.action(async (dir: string, cmd) => {
const { workspace, user, password } = cmd
await authorize(user, password, workspace, async (client, uploader) => {
const importer = new UnifiedFormatImporter(client, uploader)
await importer.importFolder(dir)
})
})
program.parse(process.argv)
}

View File

@ -772,7 +772,7 @@ function defineSpaceType (builder: Builder): void {
task.class.TaskType,
core.space.Model,
{
parent: tracker.ids.ClassingProjectType,
parent: pluginState.ids.ClassingProjectType,
statuses: classicStatuses,
descriptor: tracker.descriptors.Issue,
name: 'Issue',
@ -800,6 +800,6 @@ function defineSpaceType (builder: Builder): void {
statuses: classicStatuses.map((s) => ({ _id: s, taskType: tracker.taskTypes.Issue })),
targetClass: tracker.mixin.ClassicProjectTypeData
},
tracker.ids.ClassingProjectType
pluginState.ids.ClassingProjectType
)
}

View File

@ -39,11 +39,16 @@ import { DOMAIN_SPACE } from '@hcengineering/model-core'
import { DOMAIN_TASK, migrateDefaultStatusesBase } from '@hcengineering/model-task'
import tags from '@hcengineering/tags'
import task from '@hcengineering/task'
import { type Issue, type IssueStatus, type Project, TimeReportDayType, trackerId } from '@hcengineering/tracker'
import tracker, {
type Issue,
type IssueStatus,
type Project,
TimeReportDayType,
trackerId
} from '@hcengineering/tracker'
import contact from '@hcengineering/model-contact'
import { classicIssueTaskStatuses } from '.'
import tracker from './plugin'
async function createDefaultProject (tx: TxOperations): Promise<void> {
const current = await tx.findOne(tracker.class.Project, {

View File

@ -85,7 +85,6 @@ export default mergeIds(trackerId, tracker, {
IssueTemplateChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
ComponentChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
MilestoneChatMessageViewlet: '' as Ref<ChatMessageViewlet>,
ClassingProjectType: '' as Ref<ProjectType>,
DefaultProjectType: '' as Ref<ProjectType>
},
completion: {

View File

@ -36,6 +36,7 @@ import { Preference } from '@hcengineering/preference'
import { TagCategory, TagElement, TagReference } from '@hcengineering/tags'
import { ToDo } from '@hcengineering/time'
import {
ProjectType,
ProjectTypeDescriptor,
Task,
Project as TaskProject,
@ -384,7 +385,8 @@ const pluginState = plugin(trackerId, {
ids: {
NoParent: '' as Ref<Issue>,
IssueDraft: '',
IssueDraftChild: ''
IssueDraftChild: '',
ClassingProjectType: '' as Ref<ProjectType>
},
status: {
Backlog: '' as Ref<Status>,